yavih 1.0.0__tar.gz

yavih-1.0.0/.github/release.yml

@@ -0,0 +1,18 @@
+# Categories of the release notes GitHub generates (gh release create --generate-notes in workflows/release.yml).
+# A pull request lands in the first category whose labels it carries.
+changelog:
+  exclude:
+    labels: [skip-changelog]
+  categories:
+    - title: Breaking changes
+      labels: [breaking]
+    - title: Benchmarks
+      labels: [benchmark]
+    - title: New features
+      labels: [enhancement, feature]
+    - title: Bug fixes
+      labels: [bug, fix]
+    - title: Documentation
+      labels: [documentation, docs]
+    - title: Other changes
+      labels: ["*"]

yavih-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,109 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - run: python -m pip install ruff
+      - run: ruff check src tests
+
+  test:
+    name: pytest (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # setuptools-scm reads the version from the tags
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: System packages
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg
+      - name: Install (CPU torch)
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+          python -m pip install "setuptools>=77" "setuptools-scm>=10" wheel
+          python -m pip install -e ".[media,dev]"
+      - name: Versions
+        run: |
+          yavih --version
+          python -c "import torch, numpy; print('torch', torch.__version__, 'numpy', numpy.__version__)"
+      - name: Tests
+        # no site data here: tests that need staged benchmark data skip
+        run: pytest tests src/yavih/benchmarks -p no:cacheprovider -o addopts="--import-mode=importlib" -q -rfEs
+
+  package:
+    name: build and smoke-test the wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Build sdist and wheel
+        # --sdist --wheel builds both from the checkout, so the wheel records the source commit
+        run: |
+          python -m pip install --upgrade build twine
+          python -m build --sdist --wheel
+          ls -l dist
+      - run: python -m twine check --strict dist/*
+      - name: No maintainer notes, header images or repository docs in the wheel
+        run: |
+          python - <<'PY'
+          import glob, re, zipfile
+          (whl,) = glob.glob("dist/*.whl")
+          names = zipfile.ZipFile(whl).namelist()
+          bad = [n for n in names if "/notes/" in n or n.endswith(".pyc") or n.startswith("docs/") or "/docs/" in n
+                 or re.match(r"yavih/benchmarks/[^/]+/assets/", n)]
+          assert not bad, bad[:20]
+          assert "yavih/_version.py" in names
+          print(whl, len(names), "files")
+          PY
+      - name: Install the wheel into a clean venv (no torch)
+        run: |
+          python -m venv /tmp/venv
+          /tmp/venv/bin/python -m pip install numpy pyyaml
+          /tmp/venv/bin/python -m pip install --no-deps dist/*.whl
+      - name: Smoke-test from an empty directory
+        working-directory: ${{ runner.temp }}
+        run: |
+          /tmp/venv/bin/yavih --version
+          /tmp/venv/bin/yavih bench lint --all
+          /tmp/venv/bin/yavih bench files esc50
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          retention-days: 7

yavih-1.0.0/.github/workflows/headers.yml

@@ -0,0 +1,60 @@
+name: headers
+
+# Renders every benchmark folder's README header image for review and checks that the committed
+# assets/header.src.sha256 files (the sha256 of the HTML each committed PNG was rendered from) are current.
+on:
+  pull_request:
+    paths:
+      - "src/yavih/benchmarks/**"
+      - "assets/benchmark-header/**"
+      - "src/yavih/headers.py"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  render:
+    name: render the header images
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # setuptools-scm reads the version from the tags
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[media]" imagequant
+      - name: Chrome
+        run: google-chrome --version
+      - name: Render every header and the contact sheet
+        run: python -m yavih.headers --all --contact-sheet headers-contact-sheet.png
+      - name: The committed header sha files are current
+        run: |
+          python -m yavih.headers --all --html-only
+          changed="$(git status --porcelain -- 'src/yavih/benchmarks/*/assets/header.src.sha256')"
+          if [ -n "$changed" ]; then
+            echo "$changed"
+            git diff -- 'src/yavih/benchmarks/*/assets/header.src.sha256'
+            echo "::error::A header image is stale or missing its sha file: render it with 'python -m yavih.headers <name>' and commit assets/header.png and assets/header.src.sha256 (without Chrome: take the PNG from this run's 'headers' artifact and write the sha file with 'python -m yavih.headers <name> --html-only')."
+            exit 1
+          fi
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: headers
+          path: |
+            headers-contact-sheet.png
+            src/yavih/benchmarks/*/assets/header.png
+          if-no-files-found: warn
+          retention-days: 14

yavih-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,105 @@
+name: Release
+
+# Push a tag vX.Y.Z on main -> build, check, publish to PyPI (trusted publishing), GitHub Release with
+# generated notes. See docs/releasing.md.
+# Re-run for an existing tag: Actions -> Release -> Run workflow, tag = vX.Y.Z.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Existing tag to release (vX.Y.Z)"
+        required: true
+
+concurrency:
+  group: release-${{ github.event.inputs.tag || github.ref_name }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  TAG: ${{ github.event.inputs.tag || github.ref_name }}
+
+jobs:
+  build:
+    name: build ${{ github.event.inputs.tag || github.ref_name }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: refs/tags/${{ env.TAG }}
+          fetch-depth: 0  # setuptools-scm reads the version from the tag
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade build twine
+          python -m build --sdist --wheel
+          ls -l dist
+      - name: Built version must equal the tag
+        run: |
+          expected="${TAG#v}"
+          for f in dist/*; do
+            case "$(basename "$f")" in
+              "yavih-${expected}.tar.gz"|"yavih-${expected}-py3-none-any.whl") ;;
+              *) echo "::error::$(basename "$f") does not match tag ${TAG} (expected version ${expected})"; exit 1 ;;
+            esac
+          done
+      - run: python -m twine check --strict dist/*
+      - name: Smoke-test the wheel (no torch)
+        run: |
+          python -m venv /tmp/venv
+          /tmp/venv/bin/python -m pip install numpy pyyaml
+          /tmp/venv/bin/python -m pip install --no-deps dist/*.whl
+          cd "$RUNNER_TEMP"
+          test "$(/tmp/venv/bin/yavih --version)" = "yavih ${TAG#v}"
+          /tmp/venv/bin/yavih bench lint --all
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  pypi:
+    name: publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/yavih/
+    permissions:
+      id-token: write  # trusted publishing (OIDC); no API token
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true  # a re-run for an already published tag only (re)creates the GitHub Release
+
+  github-release:
+    name: GitHub Release
+    needs: [build, pypi]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create or update the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          # Notes are generated by GitHub from the pull requests merged since the previous tag, grouped by the
+          # categories in .github/release.yml. A re-run for an existing release only refreshes the files.
+          if gh release view "$TAG" >/dev/null 2>&1; then
+            gh release upload "$TAG" dist/* --clobber
+          else
+            gh release create "$TAG" dist/* --title "yavih ${TAG#v}" --generate-notes --verify-tag
+          fi

yavih-1.0.0/.gitignore

@@ -0,0 +1,32 @@
+# environments
+.venv/
+venv/
+
+# python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.ruff_cache/
+
+# version file written by setuptools-scm at build / install time
+src/yavih/_version.py
+
+# yavih artefacts
+runs/
+results/
+*.tmp
+_stale/
+*.tar
+*.npy
+*.log
+# python -m yavih.headers --html-only (the committed record is assets/header.src.sha256)
+src/yavih/benchmarks/*/assets/header.html
+
+# editors / OS / agents
+.DS_Store
+.idea/
+.vscode/
+.claude/

yavih-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Daniil Zverev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

yavih-1.0.0/PKG-INFO

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: yavih
+Version: 1.0.0
+Summary: Frozen-feature benchmark harness for video and audio models
+Author: Daniil Zverev
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Bizilizi/yavih
+Project-URL: Repository, https://github.com/Bizilizi/yavih
+Project-URL: Issues, https://github.com/Bizilizi/yavih/issues
+Project-URL: Documentation, https://github.com/Bizilizi/yavih/tree/main/docs
+Project-URL: Changelog, https://github.com/Bizilizi/yavih/releases
+Keywords: benchmark,evaluation,frozen features,linear probe,attentive probe,video,audio,audio-visual,representation learning,self-supervised learning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Multimedia :: Sound/Audio :: Analysis
+Classifier: Topic :: Multimedia :: Video
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyyaml>=6
+Provides-Extra: media
+Requires-Dist: av>=11; extra == "media"
+Requires-Dist: soundfile>=0.12; extra == "media"
+Requires-Dist: pillow>=9; extra == "media"
+Requires-Dist: scipy>=1.10; extra == "media"
+Provides-Extra: probe
+Requires-Dist: torch>=2.1; extra == "probe"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: scikit-learn>=1.3; extra == "dev"
+Requires-Dist: yavih[media,probe]; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/Bizilizi/yavih/main/assets/header.png" alt="yavih — frozen-feature benchmark harness for video and audio models" width="100%">
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/yavih/"><img alt="PyPI" src="https://img.shields.io/pypi/v/yavih?color=22d3ee"></a>
+  <a href="https://github.com/Bizilizi/yavih/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/Bizilizi/yavih/actions/workflows/ci.yml/badge.svg?branch=main"></a>
+  <a href="https://pypi.org/project/yavih/"><img alt="Python versions" src="https://img.shields.io/pypi/pyversions/yavih?logo=python&logoColor=white"></a>
+  <a href="https://github.com/Bizilizi/yavih/blob/main/LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-a78bfa"></a>
+</p>
+
+<h3 align="center">A growing library of video, audio and audio-visual benchmarks.<br>One harness, one protocol — and your coding agent does the wiring.</h3>
+
+**yavih** (Yet Another Video Harness) evaluates frozen video, audio and audio-visual encoders. You bring a model;
+yavih brings the benchmarks, the data recipes, the probes and the guarantees that your numbers mean the same thing
+as everyone else's.
+
+<p align="center"><img src="https://raw.githubusercontent.com/Bizilizi/yavih/main/assets/readme/agent.gif" alt="An agent benchmarks a model with yavih from one sentence" width="100%"></p>
+
+## Get started in one sentence
+
+```bash
+pip install yavih          # into the Python environment that runs your model
+yavih skill install        # teaches your coding agent (Claude Code, Codex, Cursor, …) how to use yavih
+```
+
+Then tell your agent:
+
+> **Benchmark my model with yavih.**
+
+That's it. The agent picks the default suite, prepares each benchmark's data, writes the small adapter that feeds
+your model, checks it, extracts features, trains the standard probes and hands you a report. What it cannot do on
+its own — a licence you have to accept, a download only a person can make — it lists for you at the end, while the
+rest runs.
+
+## Why yavih
+
+<table>
+<tr>
+<td width="50%" valign="top">
+
+### 🗂 Many benchmarks, one harness
+Classification, temporal, spatial, sync and dense tasks across audio, video and audio-visual data, all behind the
+same commands. Pick a ready-made suite or define your own.
+
+</td>
+<td width="50%" valign="top">
+
+### 🤖 One skill — your agent does the rest
+yavih is built to be driven by a coding agent. Every benchmark explains itself to the agent: where the data comes
+from, what the model receives, what it must return, and how to check each step.
+
+</td>
+</tr>
+<tr>
+<td width="50%" valign="top">
+
+### 🔒 Same protocol, same data — provably
+Every benchmark is frozen: the clips, the labels, the probe and the metric. A short hash names that exact protocol,
+and two numbers are compared only when their hashes match. No silent drift, no "our split".
+
+</td>
+<td width="50%" valign="top">
+
+### ➕ Add your own benchmark
+Ask your agent to *add &lt;your dataset&gt; to yavih*. It follows the contributor guide, builds the folder, runs a
+trial and opens the pull request — and everyone can run your benchmark the same way.
+
+</td>
+</tr>
+</table>
+
+## A growing library
+
+You rarely choose benchmarks one by one. A **suite** is a named set: `default` is what "benchmark my model" runs,
+`minimal` is a quick smoke pass, and `yavih suite new` makes your own.
+
+```bash
+yavih suite list               # default, minimal, and any suite you define
+yavih bench list               # every benchmark: task, size, how to get the data, what's ready on your machine
+```
+
+<p align="center"><img src="https://raw.githubusercontent.com/Bizilizi/yavih/main/assets/readme/library.gif" alt="The benchmark library grouped by capability" width="100%"></p>
+
+<!-- yavih:bench-table capabilities begin (generated from each folder's benchmark.yaml: python -m yavih.doctables --write) -->
+| capability | benchmarks | examples |
+|---|---|---|
+| `audio` | 2 | [esc50](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/esc50/README.md), [fsd50k](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/fsd50k/README.md) |
+| `visual` | 4 | [in1k](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/in1k/README.md), [k400](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/k400/README.md), [nyuv2_depth](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/nyuv2_depth/README.md), [ssv2](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/ssv2/README.md) |
+| `audio-visual` | 5 | [audioset20k](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/audioset20k/README.md), [ave](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/ave/README.md), [avsync](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/avsync/README.md), [vggsounder](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/vggsounder/README.md) + 1 more |
+| `temporal` | 2 | [ave](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/ave/README.md), [avsync](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/avsync/README.md) |
+| `spatial` | 2 | [nyuv2_depth](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/nyuv2_depth/README.md), [vggss_box](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/vggss_box/README.md) |
+
+[All 11 benchmarks → the index](https://github.com/Bizilizi/yavih/blob/main/src/yavih/benchmarks/README.md)
+<!-- yavih:bench-table capabilities end -->
+
+Each benchmark is one self-contained folder with its own page: the paper, how to get the data, and exactly how yavih
+runs it.
+
+## Numbers you can trust
+
+A benchmark result is only useful if someone else can get the same number the same way. yavih freezes everything
+that decides what a number means and leaves everything else to you:
+
+- **Frozen by yavih:** which clips, which labels, which raw files your model receives, which probe is trained on top,
+  how it is trained, and which metric is reported — with a floor (what a model that ignores its input would score)
+  next to every result.
+- **Yours to choose:** the model, how it reads the media (frames, crops, audio front end), its precision and batch
+  size. All of it is recorded with every number, none of it changes what the number is compared against.
+
+That frozen protocol is summarised in one short hash, the `eval_hash`. Same hash ⇔ same data, same protocol, same
+probe. The report only ever puts numbers with the same hash in one column; change a clip or a metric and you get a
+new hash, never a quietly different number.
+
+<p align="center"><img src="https://raw.githubusercontent.com/Bizilizi/yavih/main/assets/readme/hash.gif" alt="Matching hashes make results comparable; a changed protocol gets its own hash" width="100%"></p>
+
+## Add your benchmark
+
+Tell your agent:
+
+> **Add Diving48 to yavih.**
+
+It follows [contributing a benchmark](https://github.com/Bizilizi/yavih/blob/main/docs/contributing-a-benchmark.md)
+step by step — scaffold the folder, freeze the manifest from the public release, write the data recipe and the
+tests, render the header — then runs a synthetic trial and a fresh-agent
+[trial run](https://github.com/Bizilizi/yavih/blob/main/docs/trial-run.md) before opening the pull request.
+
+<p align="center"><img src="https://raw.githubusercontent.com/Bizilizi/yavih/main/assets/readme/contribute.gif" alt="An agent adds a new benchmark folder and opens a pull request" width="100%"></p>
+
+## How it works
+
+```
+prepare ──► extract ──► validate ──► probe ──► report
+ data         your model   yavih checks   standard     comparable
+ recipe       (a bridge)   the features   probe        tables
+```
+
+Your model connects through a **bridge**: a small Python file, written by your agent from a template, that turns a
+benchmark's raw files into features. It runs in your model's own environment — yavih installs without touching your
+torch. yavih checks the features, trains the same probe every model gets, and reports with floors and confidence
+intervals. Everything runs on a laptop, a workstation or as job arrays on any cluster.
+
+<details>
+<summary><b>Learn more</b></summary>
+
+- [Quickstart by hand](https://github.com/Bizilizi/yavih/blob/main/docs/quickstart.md) — every step without an agent, with a complete bridge.
+- [Design](https://github.com/Bizilizi/yavih/blob/main/docs/design.md) — why yavih works the way it does.
+- [The bridge contract](https://github.com/Bizilizi/yavih/blob/main/docs/bridge.md), [validation](https://github.com/Bizilizi/yavih/blob/main/docs/validation.md), [readouts](https://github.com/Bizilizi/yavih/blob/main/docs/readouts.md), [hashing](https://github.com/Bizilizi/yavih/blob/main/docs/hashing.md), [the CLI](https://github.com/Bizilizi/yavih/blob/main/docs/cli.md) — the reference.
+- [The agent skill](https://github.com/Bizilizi/yavih/blob/main/src/yavih/skill/yavih/SKILL.md) — what your agent reads.
+- [Developing yavih](https://github.com/Bizilizi/yavih/blob/main/docs/development.md) and the [releases](https://github.com/Bizilizi/yavih/releases).
+
+</details>
+
+## License
+
+yavih is released under the [MIT License](https://github.com/Bizilizi/yavih/blob/main/LICENSE). Benchmark datasets
+are not part of yavih and keep their own licences and terms of use (each benchmark's page says where they come from).