docling-metrics-core 0.1.0__tar.gz

docling_metrics_core-0.1.0/.gitignore

@@ -0,0 +1,446 @@
+*~
+scratch/
+.worktrees/
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,macos,virtualenv,pycharm,visualstudiocode,emacs,vim,jupyternotebooks
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,virtualenv,pycharm,visualstudiocode,emacs,vim,jupyternotebooks
+
+### Emacs ###
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data
+
+
+### JupyterNotebooks ###
+# gitignore template for Jupyter Notebooks
+# website: http://jupyter.org/
+
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+
+# IPython
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+
+### Visual Studio Code ###
+.vscode/
+
+### VirtualEnv ###
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+pyvenv.cfg
+pip-selfcheck.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+
+# Docs
+# docs/**/*.png
+# docs/**/*.svg
+
+
+# AI tools
+**/.bob/
+

docling_metrics_core-0.1.0/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: docling-metrics-core
+Version: 0.1.0
+Summary: Metric types and evaluation utilities for Docling document processing
+Project-URL: homepage, https://github.com/docling-project/docling-metrics
+Project-URL: repository, https://github.com/docling-project/docling-metrics
+Project-URL: issues, https://github.com/docling-project/docling-metrics/issues
+Author-email: Christoph Auer <cau@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Ahmed Nassar <ahn@zurich.ibm.com>, Nikos Livathinos <nli@zurich.ibm.com>, Maxim Lysak <mly@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
+License-Expression: MIT
+Keywords: docling,evaluation,metrics,types
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <4.0,>=3.10
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Description-Content-Type: text/markdown
+
+# docling-metrics-core
+
+A minimal interface for computing and aggregating metrics on paired data samples.
+
+## Overview
+
+This package provides base types for building metrics that:
+- Evaluate pairs of input samples (e.g., ground-truth vs. prediction)
+- Produce per-sample results traceable by sample ID
+- Aggregate results across multiple samples
+
+## Core Types
+
+- **`BaseInputSample`** — Input data with a unique `id` shared between sample pairs
+- **`BaseSampleResult`** — Output of evaluating a single sample pair
+- **`BaseAggregateResult`** — Output of aggregating multiple sample results
+- **`BaseMetric`** — Abstract interface defining `evaluate_sample`, `aggregate`, and `evaluate_dataset`
+
+## Installation
+
+```bash
+pip install docling-metrics-core
+```
+
+## Requirements
+
+- Python >=3.10,<4.0
+
+## Usage
+
+*Coming soon*
+
+## Development
+
+```bash
+# Install in development mode
+uv sync
+
+# Run tests
+uv run pytest
+```
+
+## License
+
+MIT

docling_metrics_core-0.1.0/README.md

@@ -0,0 +1,45 @@
+# docling-metrics-core
+
+A minimal interface for computing and aggregating metrics on paired data samples.
+
+## Overview
+
+This package provides base types for building metrics that:
+- Evaluate pairs of input samples (e.g., ground-truth vs. prediction)
+- Produce per-sample results traceable by sample ID
+- Aggregate results across multiple samples
+
+## Core Types
+
+- **`BaseInputSample`** — Input data with a unique `id` shared between sample pairs
+- **`BaseSampleResult`** — Output of evaluating a single sample pair
+- **`BaseAggregateResult`** — Output of aggregating multiple sample results
+- **`BaseMetric`** — Abstract interface defining `evaluate_sample`, `aggregate`, and `evaluate_dataset`
+
+## Installation
+
+```bash
+pip install docling-metrics-core
+```
+
+## Requirements
+
+- Python >=3.10,<4.0
+
+## Usage
+
+*Coming soon*
+
+## Development
+
+```bash
+# Install in development mode
+uv sync
+
+# Run tests
+uv run pytest
+```
+
+## License
+
+MIT

docling_metrics_core-0.1.0/docling_metrics_core/__init__.py

@@ -0,0 +1 @@
+"""Metric types and evaluation utilities for Docling document processing."""

docling_metrics_core-0.1.0/docling_metrics_core/base_types.py

@@ -0,0 +1,56 @@
+from abc import ABC, abstractmethod
+from typing import Annotated, Iterable, Optional
+
+from pydantic import BaseModel, Field
+
+
+class BaseInputSample(BaseModel):
+    """Base class for input samples to metrics."""
+
+    id: Annotated[
+        str,
+        Field(
+            description="Unique sample identifier, shared between ground-truth and predictions"
+        ),
+    ]
+
+
+class BaseSampleResult(BaseModel):
+    """Output of a single sample evaluation."""
+
+    id: Annotated[
+        str, Field(description="Sample identifier from the evaluated input pair")
+    ]
+
+
+class BaseAggregateResult(BaseModel):
+    """Output of aggregating multiple sample results."""
+
+    sample_count: Annotated[
+        int, Field(description="Number of samples that contributed to this result")
+    ]
+
+
+class BaseMetric(ABC):
+    """Abstract base class defining the interface for all metrics."""
+
+    @abstractmethod
+    def evaluate_sample(
+        self, sample_a: BaseInputSample, sample_b: BaseInputSample
+    ) -> BaseSampleResult:
+        """Evaluate a single sample pair."""
+        ...
+
+    @abstractmethod
+    def aggregate(
+        self, results: Iterable[BaseSampleResult]
+    ) -> Optional[BaseAggregateResult]:
+        """Aggregate multiple sample results."""
+        ...
+
+    @abstractmethod
+    def evaluate_dataset(
+        self, sample_pairs: Iterable[BaseInputSample]
+    ) -> BaseAggregateResult:
+        """Evaluate an entire dataset."""
+        ...

docling_metrics_core-0.1.0/pyproject.toml

@@ -0,0 +1,66 @@
+[project]
+name = "docling-metrics-core"
+version = "0.1.0" # DO NOT EDIT, updated automatically
+description = "Metric types and evaluation utilities for Docling document processing"
+license = "MIT"
+keywords = ["docling", "metrics", "evaluation", "types"]
+readme = "README.md"
+authors = [
+    { name = "Christoph Auer", email = "cau@zurich.ibm.com" },
+    { name = "Panos Vagenas", email = "pva@zurich.ibm.com" },
+    { name = "Ahmed Nassar", email = "ahn@zurich.ibm.com" },
+    { name = "Nikos Livathinos", email = "nli@zurich.ibm.com" },
+    { name = "Maxim Lysak", email = "mly@zurich.ibm.com" },
+    { name = "Michele Dolfi", email = "dol@zurich.ibm.com" },
+    { name = "Peter Staar", email = "taa@zurich.ibm.com" },
+]
+classifiers = [
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: POSIX :: Linux",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+]
+requires-python = '>=3.10,<4.0'
+dependencies = [
+    "pydantic>=2.0.0,<3.0.0",
+]
+
+[project.urls]
+homepage = "https://github.com/docling-project/docling-metrics"
+repository = "https://github.com/docling-project/docling-metrics"
+issues = "https://github.com/docling-project/docling-metrics/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["docling_metrics_core"]
+
+[tool.uv]
+package = true
+
+[dependency-groups]
+test = [
+    "coverage~=7.6",
+    "pytest~=8.3",
+    "pytest-cov>=6.1.1",
+    "pytest-dependency~=0.6",
+    "pytest-xdist~=3.3",
+]
+
+[tool.semantic_release]
+# for default values check:
+# https://github.com/python-semantic-release/python-semantic-release/blob/v7.32.2/semantic_release/defaults.cfg
+
+version_source = "tag_only"
+branch = "main"
+
+# configure types which should trigger minor and patch version bumps respectively
+# (note that they must be a subset of the configured allowed types):
+parser_angular_allowed_types = "build,chore,ci,docs,feat,fix,perf,style,refactor,test"
+parser_angular_minor_types = "feat"
+parser_angular_patch_types = "fix,perf"