docling-eval 0.4.0__tar.gz

docling_eval-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 International Business Machines
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docling_eval-0.4.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: docling-eval
+Version: 0.4.0
+Summary: Evaluation of Docling
+Author-email: Peter Staar <taa@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Ahmed Nassar <ahn@zurich.ibm.com>, Nikos Livathinos <nli@zurich.ibm.com>, Maxim Lysak <mly@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>
+License-Expression: MIT
+Project-URL: homepage, https://github.com/docling-project/docling-eval
+Project-URL: repository, https://github.com/docling-project/docling-eval
+Project-URL: issues, https://github.com/docling-project/docling-eval/issues
+Project-URL: changelog, https://github.com/docling-project/docling-eval/blob/main/CHANGELOG.md
+Keywords: docling,evaluation,convert,document,pdf,docx,html,markdown,layout model,segmentation,table structure,table former
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <4.0,>=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: docling[vlm]<3.0.0,>=2.31.0
+Requires-Dist: docling-core<3.0.0,>=2.30.1
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: lxml<6.0.0,>=5.3.0
+Requires-Dist: datasets<4.0.0,>=3.2.0
+Requires-Dist: apted<2.0.0,>=1.0.3
+Requires-Dist: Distance<0.2.0,>=0.1.3
+Requires-Dist: matplotlib<4.0.0,>=3.10.0
+Requires-Dist: torch<3.0.0,>=2.5.1
+Requires-Dist: torchmetrics<2.0.0,>=1.6.0
+Requires-Dist: pycocotools<3.0.0,>=2.0.8
+Requires-Dist: tabulate<0.10.0,>=0.9.0
+Requires-Dist: tqdm<5.0.0,>=4.67.1
+Requires-Dist: pillow<11.0.0,>=10.3.0
+Requires-Dist: evaluate<0.5.0,>=0.4.3
+Requires-Dist: nltk<4.0.0,>=3.9.1
+Requires-Dist: ibm-cos-sdk<3.0.0,>=2.1.40
+Requires-Dist: beautifulsoup4<5.0.0,>=4.12.3
+Requires-Dist: jiwer<4.0.0,>=3.1.0
+Requires-Dist: urllib3<2.0.0,>=1.24.2
+Requires-Dist: pandas<3.0.0,>=2.2.3
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: pypdf2<4.0.0,>=3.0.1
+Requires-Dist: xmltodict<0.15.0,>=0.14.2
+Provides-Extra: hyperscalers
+Requires-Dist: azure-ai-documentintelligence<2.0.0,>=1.0.2; extra == "hyperscalers"
+Requires-Dist: azure-common<2.0.0,>=1.1.28; extra == "hyperscalers"
+Requires-Dist: azure-core<2.0.0,>=1.33.0; extra == "hyperscalers"
+Requires-Dist: boto3<2.0.0,>=1.37.8; extra == "hyperscalers"
+Requires-Dist: google-cloud-documentai<4.0.0,>=3.2.0; extra == "hyperscalers"
+Dynamic: license-file
+
+<p align="center">
+  <a href="https://github.com/docling-project/docling-eval">
+    <img loading="lazy" alt="Docling" src="docs/assets/docling-eval-pic.png" width="40%"/>
+  </a>
+</p>
+
+# Docling-eval
+
+
+[![arXiv](https://img.shields.io/badge/arXiv-2408.09869-b31b1b.svg)](https://arxiv.org/abs/2408.09869)
+[![PyPI version](https://img.shields.io/pypi/v/docling-eval)](https://pypi.org/project/docling-eval/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/docling-eval)](https://pypi.org/project/docling-eval/)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![License MIT](https://img.shields.io/github/license/docling-project/docling-eval)](https://opensource.org/licenses/MIT)
+
+Evaluate [Docling](https://github.com/docling-project/docling) on various datasets.
+
+## Features
+
+Evaluate docling on various datasets. You can use the cli
+
+```shell
+terminal %> docling-eval --help
+                                                                                                                                                                                                                                                
+ Usage: docling_eval [OPTIONS] COMMAND [ARGS]...                                                                                                                                                                                                
+                                                                                                                                                                                                                                                
+ Docling Evaluation CLI for benchmarking document processing tasks.                                                                                                                                                                             
+                                                                                                                                                                                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────────────╮
+│ --help          Show this message and exit.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────────────╮
+│ create        Create both ground truth and evaluation datasets in one step.          │
+│ create-eval   Create evaluation dataset from existing ground truth.                  │
+│ create-gt     Create ground truth dataset only.                                      │
+│ evaluate      Evaluate predictions against ground truth.                             │
+│ visualize     Visualize evaluation results.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────╯
+
+
+```
+
+## Benchmarks
+
+- General
+    - [DP-Bench benchmarks](docs/DP-Bench_benchmarks.md): Text, layout, reading order and table structure evaluation on the DP-Bench dataset.
+    - [OmniDocBench benchmarks](docs/OmniDocBench_benchmarks.md): Text, layout, reading order and table structure evaluation on the OmniDocBench dataset.
+- Layout
+    - [DocLayNetV1 Benchmarks](docs/DocLayNetv1_benchmarks.md): Text and layout evaluation on the DocLayNet v1.2 dataset.
+- Table-Structure
+    - [FinTabnet Benchmarks](docs/FinTabNet_benchmarks.md): Table structure evaluation on the FinTabNet dataset.
+    - [PubTabNet benchmarks](docs/PubTabNet_benchmarks.md): Table structure evaluation on the PubTabNet dataset.
+    - [Pub1M benchmarks](docs/P1M_benchmarks.md): Table structure evaluation on the Pub1M dataset.
+
+On our list for next benchmarks:
+
+- [OmniOCR](getomni-ai/ocr-benchmark)
+- Hyperscalers
+- [CoMix](https://github.com/emanuelevivoli/CoMix/tree/main/docs/datasets)
+- [DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA)
+- [rd-tablebench](https://huggingface.co/datasets/reducto/rd-tablebench)
+- [BigDocs-Bench](https://huggingface.co/datasets/ServiceNow/BigDocs-Bench)
+  
+## Contributing
+
+Please read [Contributing to Docling](https://github.com/docling-project/docling/blob/main/CONTRIBUTING.md) for details.
+
+
+## License
+
+The Docling codebase is under MIT license.
+For individual model usage, please refer to the model licenses found in the original packages.
+
+
+## IBM ❤️ Open Source AI
+
+Docling-eval has been brought to you by IBM.

docling_eval-0.4.0/README.md

@@ -0,0 +1,81 @@
+<p align="center">
+  <a href="https://github.com/docling-project/docling-eval">
+    <img loading="lazy" alt="Docling" src="docs/assets/docling-eval-pic.png" width="40%"/>
+  </a>
+</p>
+
+# Docling-eval
+
+
+[![arXiv](https://img.shields.io/badge/arXiv-2408.09869-b31b1b.svg)](https://arxiv.org/abs/2408.09869)
+[![PyPI version](https://img.shields.io/pypi/v/docling-eval)](https://pypi.org/project/docling-eval/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/docling-eval)](https://pypi.org/project/docling-eval/)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![License MIT](https://img.shields.io/github/license/docling-project/docling-eval)](https://opensource.org/licenses/MIT)
+
+Evaluate [Docling](https://github.com/docling-project/docling) on various datasets.
+
+## Features
+
+Evaluate docling on various datasets. You can use the cli
+
+```shell
+terminal %> docling-eval --help
+                                                                                                                                                                                                                                                
+ Usage: docling_eval [OPTIONS] COMMAND [ARGS]...                                                                                                                                                                                                
+                                                                                                                                                                                                                                                
+ Docling Evaluation CLI for benchmarking document processing tasks.                                                                                                                                                                             
+                                                                                                                                                                                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────────────╮
+│ --help          Show this message and exit.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────────────╮
+│ create        Create both ground truth and evaluation datasets in one step.          │
+│ create-eval   Create evaluation dataset from existing ground truth.                  │
+│ create-gt     Create ground truth dataset only.                                      │
+│ evaluate      Evaluate predictions against ground truth.                             │
+│ visualize     Visualize evaluation results.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────╯
+
+
+```
+
+## Benchmarks
+
+- General
+    - [DP-Bench benchmarks](docs/DP-Bench_benchmarks.md): Text, layout, reading order and table structure evaluation on the DP-Bench dataset.
+    - [OmniDocBench benchmarks](docs/OmniDocBench_benchmarks.md): Text, layout, reading order and table structure evaluation on the OmniDocBench dataset.
+- Layout
+    - [DocLayNetV1 Benchmarks](docs/DocLayNetv1_benchmarks.md): Text and layout evaluation on the DocLayNet v1.2 dataset.
+- Table-Structure
+    - [FinTabnet Benchmarks](docs/FinTabNet_benchmarks.md): Table structure evaluation on the FinTabNet dataset.
+    - [PubTabNet benchmarks](docs/PubTabNet_benchmarks.md): Table structure evaluation on the PubTabNet dataset.
+    - [Pub1M benchmarks](docs/P1M_benchmarks.md): Table structure evaluation on the Pub1M dataset.
+
+On our list for next benchmarks:
+
+- [OmniOCR](getomni-ai/ocr-benchmark)
+- Hyperscalers
+- [CoMix](https://github.com/emanuelevivoli/CoMix/tree/main/docs/datasets)
+- [DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA)
+- [rd-tablebench](https://huggingface.co/datasets/reducto/rd-tablebench)
+- [BigDocs-Bench](https://huggingface.co/datasets/ServiceNow/BigDocs-Bench)
+  
+## Contributing
+
+Please read [Contributing to Docling](https://github.com/docling-project/docling/blob/main/CONTRIBUTING.md) for details.
+
+
+## License
+
+The Docling codebase is under MIT license.
+For individual model usage, please refer to the model licenses found in the original packages.
+
+
+## IBM ❤️ Open Source AI
+
+Docling-eval has been brought to you by IBM.

docling_eval-0.4.0/docling_eval/aggregations/consolidator.py

@@ -0,0 +1,276 @@
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import openpyxl
+import pandas as pd
+from openpyxl import load_workbook
+from openpyxl.cell.cell import Cell
+from openpyxl.styles import Font
+from pandas import DataFrame
+
+from docling_eval.aggregations.multi_evalutor import MultiEvaluation
+from docling_eval.datamodels.types import ConsolidationFormats, EvaluationModality
+from docling_eval.evaluators.base_evaluator import EvaluationRejectionType
+from docling_eval.evaluators.bbox_text_evaluator import DatasetBoxesTextEvaluation
+from docling_eval.evaluators.layout_evaluator import DatasetLayoutEvaluation
+from docling_eval.evaluators.markdown_text_evaluator import DatasetMarkdownEvaluation
+from docling_eval.evaluators.readingorder_evaluator import DatasetReadingOrderEvaluation
+from docling_eval.evaluators.stats import DatasetStatistics
+from docling_eval.evaluators.table_evaluator import DatasetTableEvaluation
+
+_log = logging.getLogger(__name__)
+
+
+def export_value(val: Union[float, DatasetStatistics]) -> str:
+    r"""Get statistics value"""
+    if isinstance(val, DatasetStatistics):
+        fmt_val = f"{val.mean:.2f}±{val.std:.2f}"
+    else:
+        fmt_val = f"{val:.2f}"
+
+    return fmt_val
+
+
+class Consolidator:
+    r"""
+    Consolidate a MultiEvaluation into a comparison matrix
+
+    The comparison matrix has 3 dimensions:
+    - Benchmarks
+    - ConversionProviders
+    - Modalities
+    """
+
+    def __init__(self, output_path: Path):
+        r""" """
+        self._output_path = output_path
+        self._excel_engine = "openpyxl"
+        self._sheet_name = "matrix"
+        self._excel_filename = "consolidation_matrix.xlsx"
+
+        self._output_path.mkdir(parents=True, exist_ok=True)
+
+    def __call__(
+        self,
+        multi_evaluation: MultiEvaluation,
+        consolidation_format: Optional[
+            ConsolidationFormats
+        ] = ConsolidationFormats.EXCEL,
+    ) -> Tuple[Dict[EvaluationModality, DataFrame], Optional[Path]]:
+        r""" """
+        dfs = self._build_dataframes(multi_evaluation)
+
+        # Export dataframe
+        if consolidation_format == ConsolidationFormats.EXCEL:
+            produced_fn = self._to_excel(dfs)
+            _log.info("Produced excel: %s", str(produced_fn))
+        else:
+            _log.info("Unsupported consolidation format: %s", consolidation_format)
+
+        return dfs, produced_fn
+
+    def _to_excel(self, dfs: Dict[EvaluationModality, DataFrame]) -> Path:
+        r""" """
+        excel_fn = self._output_path / self._excel_filename
+        startrow = 0
+        header_rows: List[int] = []
+        with pd.ExcelWriter(excel_fn, engine=self._excel_engine) as writer:  # type: ignore
+            for modality, df in dfs.items():
+                if self._sheet_name in writer.book.sheetnames:
+                    sheet = writer.book[self._sheet_name]
+                    startrow = sheet.max_row + 2
+
+                # Add the modality as a "header" for the metrics subtable
+                header_df = DataFrame([modality.name])
+                header_rows.append(startrow + 1)
+                header_df.to_excel(
+                    writer,
+                    sheet_name=self._sheet_name,
+                    startrow=startrow,
+                    index=False,
+                    header=False,
+                )
+                startrow += 1
+
+                # Metrics subtable
+                df.to_excel(
+                    writer,
+                    sheet_name=self._sheet_name,
+                    startrow=startrow,
+                    index=False,
+                )
+        # Format the excel
+        self._format_excel(excel_fn, header_rows)
+
+        return excel_fn
+
+    def _format_excel(self, excel_fn: Path, header_rows: List[int]):
+        r"""Do some proper formatting of the generated excel"""
+        workbook = load_workbook(excel_fn)
+        sheet = workbook[self._sheet_name]
+
+        # Adjust the cell width
+        for col in sheet.columns:
+            # Find the maximum length of strings in this column (excluding empty cells)
+            max_length = 0
+            for cell in col:
+                try:
+                    if len(str(cell.value)) > max_length:
+                        max_length = len(str(cell.value))
+                except:
+                    pass
+            adjusted_width = max_length + 2  # Add some padding to make it look better
+            first_cell = col[0]
+            assert isinstance(first_cell, Cell)
+            sheet.column_dimensions[first_cell.column_letter].width = adjusted_width
+
+        # Iterate through each cell in the worksheet and remove borders
+        for row in sheet.iter_rows():
+            for cell in row:
+                cell.border = openpyxl.styles.Border()  # Remove borders
+
+        # Make bold the subtable headers
+        bold_font = Font(bold=True)
+        for header_row in header_rows:
+            cell = sheet.cell(row=header_row, column=1)
+            cell.font = bold_font
+            x = 0
+
+        # Save back the excel
+        workbook.save(excel_fn)
+
+    def _build_dataframes(
+        self,
+        multi_evaluation: MultiEvaluation,
+    ) -> Dict[EvaluationModality, DataFrame]:
+        r"""
+        Return a Dict with dataframes per modality
+        """
+        # Collect all data to build the dataframes
+        df_data: Dict[EvaluationModality, List[Dict[str, Union[str, float, int]]]] = {}
+
+        # Collect the dataframe data
+        for benchmark, exp_mod_eval in multi_evaluation.evaluations.items():
+            for experiment, mod_eval in exp_mod_eval.items():
+                for modality, single_evaluation in mod_eval.items():
+                    evaluation = single_evaluation.evaluation
+
+                    if modality == EvaluationModality.LAYOUT:
+                        metrics = self._layout_metrics(evaluation)
+                    elif modality == EvaluationModality.MARKDOWN_TEXT:
+                        metrics = self._markdowntext_metrics(evaluation)
+                    elif modality == EvaluationModality.TABLE_STRUCTURE:
+                        metrics = self._tablestructure_metrics(evaluation)
+                    elif modality == EvaluationModality.READING_ORDER:
+                        metrics = self._readingorder_metrics(evaluation)
+                    elif modality == EvaluationModality.BBOXES_TEXT:
+                        metrics = self._bboxestext_metrics(evaluation)
+                    else:
+                        _log.error(
+                            "Evaluation modality unsupported for export: %s", modality
+                        )
+                        continue
+
+                    # Gather the dataframe data
+                    provider = (
+                        single_evaluation.prediction_provider_type.value
+                        if single_evaluation.prediction_provider_type is not None
+                        else "Unkown"
+                    )
+                    data: Dict[str, Union[str, float]] = {
+                        "Benchmark": benchmark.value,
+                        "Provider": provider,
+                        "Experiment": experiment,
+                        "evaluated_samples": evaluation.evaluated_samples,
+                    }
+                    for rej_type in EvaluationRejectionType:
+                        if rej_type not in evaluation.rejected_samples:
+                            data[rej_type.value] = 0
+                        else:
+                            data[rej_type.value] = evaluation.rejected_samples[rej_type]
+
+                    data |= metrics
+                    if modality not in df_data:
+                        df_data[modality] = []
+                    df_data[modality].append(data)
+
+        # Build the dataframes
+        dfs: Dict[EvaluationModality, DataFrame] = {}
+        for modality, m_data in df_data.items():
+            df = DataFrame(m_data)
+            df = df.sort_values(by=["Benchmark", "Provider"], ascending=[True, True])
+            dfs[modality] = df
+
+        return dfs
+
+    def _layout_metrics(self, evaluation: DatasetLayoutEvaluation) -> Dict[str, str]:
+        r"""Get the metrics for the LayoutEvaluation"""
+        metrics = {
+            "mAP": export_value(evaluation.map_stats),
+            "mAP_50": export_value(evaluation.map_50_stats),
+            "mAP_75": export_value(evaluation.map_75_stats),
+            "weighted_mAP_50": export_value(evaluation.weighted_map_50_stats),
+            "weighted_mAP_75": export_value(evaluation.weighted_map_75_stats),
+            "weighted_mAP_90": export_value(evaluation.weighted_map_90_stats),
+            "weighted_mAP_95": export_value(evaluation.weighted_map_95_stats),
+        }
+        for class_evaluation in evaluation.evaluations_per_class:
+            key = f"class_{class_evaluation.label}"
+            metrics[key] = export_value(class_evaluation.value)
+
+        return metrics
+
+    def _markdowntext_metrics(
+        self,
+        evaluation: DatasetMarkdownEvaluation,
+    ) -> Dict[str, str]:
+        r""" """
+        metrics = {
+            "BLEU": export_value(evaluation.bleu_stats),
+            "F1": export_value(evaluation.f1_score_stats),
+            "Precision": export_value(evaluation.precision_stats),
+            "Recall": export_value(evaluation.recall_stats),
+            "Edit_Distance": export_value(evaluation.edit_distance_stats),
+            "METEOR": export_value(evaluation.meteor_stats),
+        }
+        return metrics
+
+    def _tablestructure_metrics(
+        self,
+        evaluation: DatasetTableEvaluation,
+    ) -> Dict[str, str]:
+        r""" """
+        metrics = {
+            "TEDS": export_value(evaluation.TEDS),
+            "TEDS_struct": export_value(evaluation.TEDS_struct),
+            "TEDS_simple": export_value(evaluation.TEDS_simple),
+            "TEDS_complex": export_value(evaluation.TEDS_complex),
+        }
+        return metrics
+
+    def _readingorder_metrics(
+        self,
+        evaluation: DatasetReadingOrderEvaluation,
+    ) -> Dict[str, str]:
+        r""" """
+        metrics = {
+            "ARD": export_value(evaluation.ard_stats),
+            "Weighted_ARD": export_value(evaluation.w_ard_stats),
+        }
+        return metrics
+
+    def _bboxestext_metrics(
+        self,
+        evaluation: DatasetBoxesTextEvaluation,
+    ) -> Dict[str, str]:
+        r""" """
+        metrics = {
+            "BLEU": export_value(evaluation.bleu_stats),
+            "F1": export_value(evaluation.f1_score_stats),
+            "Precision": export_value(evaluation.precision_stats),
+            "Recall": export_value(evaluation.recall_stats),
+            "Edit_Distance": export_value(evaluation.edit_distance_stats),
+            "METEOR": export_value(evaluation.meteor_stats),
+        }
+        return metrics