pyannote-metrics 4.0.0__tar.gz

pyannote_metrics-4.0.0/.github/workflows/doc.yml

@@ -0,0 +1,33 @@
+name: Documentation
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+    - name: Install the project
+      run: uv sync --extra doc
+  
+    - name: Build documentation
+      run: |
+        make --directory=doc html
+        touch ./doc/build/html/.nojekyll
+    - name: Deploy
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./doc/build/html

pyannote_metrics-4.0.0/.github/workflows/release.yml

@@ -0,0 +1,125 @@
+name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+    - name: Build
+      run: uv build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        cache-dependency-glob: uv.lock
+    - name: Publish distribution 📦 to PyPI
+      run: uv publish --trusted-publishing always  --publish-url https://upload.pypi.org/legacy/
+
+
+  github-release:
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        "$GITHUB_REF_NAME"
+        --repo "$GITHUB_REPOSITORY"
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        "$GITHUB_REF_NAME" dist/**
+        --repo "$GITHUB_REPOSITORY"
+
+#   publish-to-testpypi:
+#     name: Publish Python 🐍 distribution 📦 to TestPyPI
+#     needs:
+#     - build
+#     runs-on: ubuntu-latest
+#
+#     environment:
+#       name: testpypi
+#
+#     permissions:
+#       id-token: write  # IMPORTANT: mandatory for trusted publishing
+#
+#     steps:
+#     - name: Download all the dists
+#       uses: actions/download-artifact@v4
+#       with:
+#         name: python-package-distributions
+#         path: dist/
+#     - name: Install uv
+#       uses: astral-sh/setup-uv@v5
+#       with:
+#         enable-cache: true
+#         cache-dependency-glob: uv.lock
+#     - name: Publish distribution 📦 to PyPI
+#       run: uv publish --trusted-publishing always  --publish-url https://test.pypi.org/legacy/

pyannote_metrics-4.0.0/.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - develop
+  push:
+    branches:
+      - develop
+      - master
+      - release/*
+
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: 
+          - "3.10"
+          - "3.11"
+          - "3.12"
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install the project
+        run: uv sync --extra test
+
+      - name: Run tests
+        run: uv run pytest tests

pyannote_metrics-4.0.0/.gitignore

@@ -0,0 +1,58 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+MANIFEST
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+doc/.ipynb_checkpoints
+
+.mypy_cache/

pyannote_metrics-4.0.0/.python-version

@@ -0,0 +1 @@
+3.10

pyannote_metrics-4.0.0/ISSUE_TEMPLATE.md

@@ -0,0 +1,40 @@
+
+#### Description
+<!-- Example: DiarizationPurity does not return the expected value -->
+
+#### Steps/Code to Reproduce
+<!--
+Example:
+
+```
+from pyannote.core import Annotation, Segment
+reference = Annotation()
+reference[Segment(0, 10)] = 'A'
+reference[Segment(12, 20)] = 'B'
+reference[Segment(24, 27)] = 'A'
+reference[Segment(30, 40)] = 'C'
+hypothesis = Annotation()
+hypothesis[Segment(2, 13)] = 'a'
+hypothesis[Segment(13, 14)] = 'd'
+hypothesis[Segment(14, 20)] = 'b'
+hypothesis[Segment(22, 38)] = 'c'
+hypothesis[Segment(38, 40)] = 'd'
+
+from pyannote.metrics.diarization import DiarizationPurity
+purity = DiarizationPurity()
+print "Purity = {0:.3f}".format(purity(reference, hypothesis))
+```
+If the code is too long, feel free to put it in a public gist and link
+it in the issue: https://gist.github.com
+-->
+
+#### Expected Results
+<!-- Example: Output should be "Purity = 0.667". Please paste or describe the expected results.-->
+
+#### Actual Results
+<!-- Please paste or specifically describe the actual output or traceback. -->
+
+#### Versions
+<!-- Please paste the output of `pip freeze | grep pyannote` below -->
+
+<!-- Thanks for contributing! -->

pyannote_metrics-4.0.0/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 CNRS (Hervé BREDIN - http://herve.niderb.fr)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyannote_metrics-4.0.0/PKG-INFO

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: pyannote-metrics
+Version: 4.0.0
+Summary: A toolkit for reproducible evaluation, diagnostic, and error analysis of speaker diarization systems
+Author-email: Hervé BREDIN <herve@pyannote.ai>
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: numpy>=2.2.2
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pyannote-core>=6.0.0
+Requires-Dist: pyannote-database>=6.0.0
+Requires-Dist: scikit-learn>=1.6.1
+Requires-Dist: scipy>=1.15.1
+Provides-Extra: cli
+Requires-Dist: docopt>=0.6.2; extra == 'cli'
+Requires-Dist: sympy>=1.13.3; extra == 'cli'
+Requires-Dist: tabulate>=0.9.0; extra == 'cli'
+Provides-Extra: doc
+Requires-Dist: ipython>=8.32.0; extra == 'doc'
+Requires-Dist: matplotlib>=3.10.0; extra == 'doc'
+Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == 'doc'
+Requires-Dist: sphinx>=8.1.3; extra == 'doc'
+Provides-Extra: plot
+Requires-Dist: matplotlib>=3.10.0; extra == 'plot'
+Provides-Extra: test
+Requires-Dist: pytest>=8.3.4; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pyannote.metrics
+
+> a toolkit for reproducible evaluation, diagnostic, and error analysis of speaker diarization systems
+
+An overview of `pyannote.metrics` is available as an [InterSpeech 2017 paper](docs/pyannote-metrics.pdf): it is recommended to read it first, to quickly get an idea whether this tool is for you.
+
+## Installation
+
+```bash
+$ pip install pyannote.metrics
+```
+
+## Documentation
+
+The documentation is available at [http://pyannote.github.io/pyannote-metrics](http://pyannote.github.io/pyannote-metrics).
+
+Sample notebooks are available [here](http://nbviewer.ipython.org/github/pyannote/pyannote-metrics/blob/master/notebooks/index.ipynb).
+
+## Citation
+
+If you use `pyannote.metrics` in your research, please use the following citation:
+
+```bibtex
+@inproceedings{pyannote.metrics,
+  author = {Herv\'e Bredin},
+  title = {{pyannote.metrics: a toolkit for reproducible evaluation, diagnostic, and error analysis of speaker diarization systems}},
+  booktitle = {{Interspeech 2017, 18th Annual Conference of the International Speech Communication Association}},
+  year = {2017},
+  month = {August},
+  address = {Stockholm, Sweden},
+  url = {http://pyannote.github.io/pyannote-metrics},
+}
+```

pyannote_metrics-4.0.0/README.md

@@ -0,0 +1,33 @@
+# pyannote.metrics
+
+> a toolkit for reproducible evaluation, diagnostic, and error analysis of speaker diarization systems
+
+An overview of `pyannote.metrics` is available as an [InterSpeech 2017 paper](docs/pyannote-metrics.pdf): it is recommended to read it first, to quickly get an idea whether this tool is for you.
+
+## Installation
+
+```bash
+$ pip install pyannote.metrics
+```
+
+## Documentation
+
+The documentation is available at [http://pyannote.github.io/pyannote-metrics](http://pyannote.github.io/pyannote-metrics).
+
+Sample notebooks are available [here](http://nbviewer.ipython.org/github/pyannote/pyannote-metrics/blob/master/notebooks/index.ipynb).
+
+## Citation
+
+If you use `pyannote.metrics` in your research, please use the following citation:
+
+```bibtex
+@inproceedings{pyannote.metrics,
+  author = {Herv\'e Bredin},
+  title = {{pyannote.metrics: a toolkit for reproducible evaluation, diagnostic, and error analysis of speaker diarization systems}},
+  booktitle = {{Interspeech 2017, 18th Annual Conference of the International Speech Communication Association}},
+  year = {2017},
+  month = {August},
+  address = {Stockholm, Sweden},
+  url = {http://pyannote.github.io/pyannote-metrics},
+}
+```

pyannote_metrics-4.0.0/doc/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = pyannotemetrics
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	uv run --extra doc $(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	uv run --extra doc $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

pyannote_metrics-4.0.0/doc/source/api.rst

@@ -0,0 +1,8 @@
+Getting started
+===============
+
+.. toctree::
+   :maxdepth: 2
+
+   tutorial
+   basics

pyannote_metrics-4.0.0/doc/source/basics.rst

@@ -0,0 +1,186 @@
+Principles
+==========
+
+`pyannote.metrics` provides a set of classes to compare the output of speaker diarization (hereafter called `hypothesis`) systems to manual annotations (`reference`). Let us first instantiate a sample `reference` and `hypothesis`.
+
+.. ipython::
+
+   In [10]: from pyannote.core import Segment, Timeline, Annotation
+
+
+   In [11]: reference = Annotation(uri='file1')
+      ....: reference[Segment(0, 10)] = 'A'
+      ....: reference[Segment(12, 20)] = 'B'
+      ....: reference[Segment(24, 27)] = 'A'
+      ....: reference[Segment(30, 40)] = 'C'
+
+   In [12]: hypothesis = Annotation(uri='file1')
+      ....: hypothesis[Segment(2, 13)] = 'a'
+      ....: hypothesis[Segment(13, 14)] = 'd'
+      ....: hypothesis[Segment(14, 20)] = 'b'
+      ....: hypothesis[Segment(22, 38)] = 'c'
+      ....: hypothesis[Segment(38, 40)] = 'd'
+
+
+.. plot:: pyplots/tutorial.py
+
+This basically tells us that, according to the manual annotation, speaker `A` speaks in timeranges [0s, 10s] and [24s, 27s].
+
+
+.. note::
+
+    Overlapping segments are supported. See :mod:`pyannote.core` documentation for more details.
+
+`pyannote.metrics` follows an object-oriented paradigm.
+Most evaluation metrics (e.g. :class:`DiarizationErrorRate` below) inherit from :class:`BaseMetric`.
+As such, they share a common set of methods.
+
+For instance, once instantiated, they can be called directly to compute the value of the evaluation metric.
+
+.. ipython::
+   :okwarning:
+
+   In [10]: from pyannote.metrics.diarization import DiarizationErrorRate
+
+   In [1]: metric = DiarizationErrorRate()
+
+   In [1]: metric(reference, hypothesis)
+
+
+Accumulation & reporting
+------------------------
+
+The same metric instance can be used to evaluate multiple files.
+
+.. ipython::
+   :okwarning:
+
+   In [11]: other_reference = Annotation(uri='file2')
+      ....: other_reference[Segment(0, 5)] = 'A'
+      ....: other_reference[Segment(6, 10)] = 'B'
+      ....: other_reference[Segment(12, 13)] = 'B'
+      ....: other_reference[Segment(15, 20)] = 'A'
+
+   In [12]: other_hypothesis = Annotation(uri='file2')
+      ....: other_hypothesis[Segment(1, 6)] = 'a'
+      ....: other_hypothesis[Segment(6, 7)] = 'b'
+      ....: other_hypothesis[Segment(7, 10)] = 'c'
+      ....: other_hypothesis[Segment(11, 19)] = 'b'
+      ....: other_hypothesis[Segment(19, 20)] = 'a'
+
+   In [12]: metric = DiarizationErrorRate()
+
+   In [12]: metric(reference, hypothesis)
+
+   In [12]: metric(other_reference, other_hypothesis)
+
+
+You do not need to keep track of the result of each call yourself: this is done automatically.
+For instance, once you have evaluated all files, you can use the overriden :func:`~pyannote.metrics.base.BaseMetric.__abs__` operator to get the accumulated value:
+
+.. ipython::
+
+   In [12]: abs(metric)
+
+:func:`~pyannote.metrics.base.BaseMetric.report` provides a convenient summary of the result:
+
+.. ipython::
+
+   In [12]: report = metric.report(display=True)
+
+
+The internal accumulator can be reset using the :func:`~pyannote.metrics.base.BaseMetric.report` method:
+
+.. ipython::
+
+   In [12]: metric.reset()
+
+
+Evaluation map
+--------------
+
+Though audio files can always be processed entirely (from beginning to end), there are cases where reference annotations are only available for some regions of the audio files.
+All metrics support the provision of an evaluation map that indicate which part of the audio file should be evaluated.
+
+.. ipython::
+
+   In [2]: uem = Timeline([Segment(0, 10), Segment(15, 20)])
+
+   In [2]: metric(reference, hypothesis, uem=uem)
+
+
+Components
+----------
+
+Most metrics are computed as the combination of several components.
+For instance, the diarization error rate is the combination of false alarm (non-speech regions classified as speech), missed detection (speech regions classified as non-speech) and confusion between speakers.
+
+Using ``detailed=True`` will return the value of each component:
+
+.. ipython::
+   :okwarning:
+
+   In [13]: metric(reference, hypothesis, detailed=True)
+
+The accumulated value of each component can also be obtained using the overriden :func:`~pyannote.metrics.base.BaseMetric.__getitem__` operator:
+
+.. ipython::
+   :okwarning:
+
+   In [13]: metric(other_reference, other_hypothesis)
+
+   In [13]: metric['confusion']
+
+   In [13]: metric[:]
+
+
+Define your own metric
+----------------------
+
+It is possible (and encouraged) to develop and contribute new evaluation metrics.
+
+All you have to do is inherit from :class:`BaseMetric` and implement a few methods:
+``metric_name``, ``metric_components``, ``compute_components``, and ``compute_metric``:
+
+.. code-block:: python
+
+    def is_male(speaker_name):
+        # black magic that returns True if speaker is a man, False otherwise
+        pass
+
+    class MyMetric(BaseMetric):
+        # This dummy metric computes the ratio between male and female speakers.
+        # It does not actually use the reference annotation...
+
+        @classmethod
+        def metric_name(cls):
+            # Return human-readable name of the metric
+
+            return 'male / female ratio'
+
+        @classmethod:
+        def metric_components(cls):
+            # Return component names from which the metric is computed
+
+            return ['male', 'female']
+
+        def compute_components(self, reference, hypothesis, **kwargs):
+            # Actually compute the value of each component
+
+            components = {'male': 0., 'female': 0.}
+
+            for segment, _, speaker_name in hypothesis.itertracks(yield_label=True):
+                if is_male(speaker_name):
+                    components['male'] += segment.duration
+                else:
+                    components['female'] += segment.duration
+
+            return components
+
+        def compute_metric(self, components):
+            # Actually compute the metric based on the component values
+
+            return components['male'] / components['female']
+
+
+See :class:`pyannote.metrics.base.BaseMetric` for more details.