KappaIOU 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kappaiou-1.0.0/KappaIOU.egg-info/PKG-INFO +41 -0
- kappaiou-1.0.0/KappaIOU.egg-info/SOURCES.txt +13 -0
- kappaiou-1.0.0/KappaIOU.egg-info/dependency_links.txt +1 -0
- kappaiou-1.0.0/KappaIOU.egg-info/entry_points.txt +2 -0
- kappaiou-1.0.0/KappaIOU.egg-info/requires.txt +2 -0
- kappaiou-1.0.0/KappaIOU.egg-info/top_level.txt +1 -0
- kappaiou-1.0.0/LICENSE +21 -0
- kappaiou-1.0.0/PKG-INFO +41 -0
- kappaiou-1.0.0/README.md +23 -0
- kappaiou-1.0.0/kappaiou/__init__.py +7 -0
- kappaiou-1.0.0/kappaiou/cli.py +125 -0
- kappaiou-1.0.0/kappaiou/core.py +134 -0
- kappaiou-1.0.0/pyproject.toml +41 -0
- kappaiou-1.0.0/setup.cfg +4 -0
- kappaiou-1.0.0/tests/test.py +98 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: KappaIOU
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A training-free, chance-corrected reliability library for continuous multi-object spatial annotations.
|
|
5
|
+
Author-email: "F. Trevor Rogers" <rogers72@hawaii.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/tr7200/KappaIOU
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/tr7200/KappaIOU/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy>=1.20.0
|
|
16
|
+
Requires-Dist: scipy>=1.7.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Kappa-IoU
|
|
20
|
+
|
|
21
|
+
This Python package implements the Kappa-IoU measure of Inter-annotator reliability from Rogers (2026):
|
|
22
|
+
|
|
23
|
+
Rogers, Trevor F. (2026). Kappa-IoU: Inter-Rater Reliability for Spatial Annotation
|
|
24
|
+
|
|
25
|
+
Large-scale annotation projects for which automated annotation solutions are insufficient require a metric that measures the quality of annotation between annotators.
|
|
26
|
+
[Intersection-over-Union](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Rezatofighi%2C%20H.%2C%20Tsoi%2C%20N.%2C%20Gwak%2C%20J.%2C%20Sadeghian%2C%20A.%2C%20Reid%2C%20I.%2C%20%26%20Savarese%2C%20S.%20(2019).%20Generalized%20intersection%20over%20union%3A%20A%20metric%20and%20a%20loss%20for%20object%20detection.%20Proceedings%20of%20the%20IEEE%2FCVF%20Conference%20on%20Computer%20Vision%20and%20Pattern%20Recognition%20(CVPR).), or IoU, is traditionally used for spatial data but requires a ground truth.
|
|
27
|
+
[Cohen's Kappa](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Cohen%2C%20J.%20(1960).%20A%20coefficient%20of%20agreement%20for%20nominal%20scales.%C2%A0Educational%20and%20psychological%20measurement%2C%C2%A020(1)%2C%2037-46.) is a psychometric measure that measures the amount of agreement between raters, but is traditionally used for categorical data.
|
|
28
|
+
|
|
29
|
+
This repository contains a Python package implementing the [Kappa-IoU](https://drive.google.com/file/d/1QW1dbFWX1-rxz7XcvPkBdlR0BFKoBIwC/view?usp=sharing) paper, which indexes IoU with Cohen's Kappa, and handles differences in object cardinality between raters by treating it as a linear assignment problem using [Kuhn-Munkres](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Munkres%2C%20J.%20(1957).%20Algorithms%20for%20the%20assignment%20and%20transportation%20problems.%20Journal%20of%20the%20Society%20for%20Industrial%20and%20Applied%20Mathematics%2C%205(1)%2C%2032-38.) assignment. Also included is an adjusted version of Kappa-IoU that uses a version of the [BLEU](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Papineni%2C%20K.%2C%20Roukos%2C%20S.%2C%20Ward%2C%20T.%2C%20%26%20Zhu%2C%20W.%20J.%20(2002).%20BLEU%3A%20a%20method%20for%20automatic%20evaluation%20of%20machine%20translation.%20In%20Proceedings%20of%20the%2040th%20Annual%20Meeting%20of%20the%20Association%20for%20Computational%20Linguistics%20(pp.%20311-318).) measure for machine translation which instead scores N-objects for multi-object annotation. Along with the Python package implementing this measure, this repository also contains `Section4.py`, code that recreates the examples from that section of the paper.
|
|
30
|
+
|
|
31
|
+
To install this package from inside the downloaded folder from Github:
|
|
32
|
+
|
|
33
|
+
python -m pip install -e .
|
|
34
|
+
|
|
35
|
+
To use, run the analysis engine straight from your terminal shell across your dataset directories using the console entrypoint:
|
|
36
|
+
|
|
37
|
+
kappaiou-run --baseline /data/author_baseline.csv --dir /data/annotator_tranches/ --tau 0.70
|
|
38
|
+
|
|
39
|
+
The author baseline and annotator tranche CSV files are expected to have the filenames in the first column, label in the second column, and then xywh-format annotations in the following four columns.
|
|
40
|
+
|
|
41
|
+
MIT License
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
KappaIOU.egg-info/PKG-INFO
|
|
5
|
+
KappaIOU.egg-info/SOURCES.txt
|
|
6
|
+
KappaIOU.egg-info/dependency_links.txt
|
|
7
|
+
KappaIOU.egg-info/entry_points.txt
|
|
8
|
+
KappaIOU.egg-info/requires.txt
|
|
9
|
+
KappaIOU.egg-info/top_level.txt
|
|
10
|
+
kappaiou/__init__.py
|
|
11
|
+
kappaiou/cli.py
|
|
12
|
+
kappaiou/core.py
|
|
13
|
+
tests/test.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
kappaiou
|
kappaiou-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 T. Rogers
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
kappaiou-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: KappaIOU
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A training-free, chance-corrected reliability library for continuous multi-object spatial annotations.
|
|
5
|
+
Author-email: "F. Trevor Rogers" <rogers72@hawaii.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/tr7200/KappaIOU
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/tr7200/KappaIOU/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy>=1.20.0
|
|
16
|
+
Requires-Dist: scipy>=1.7.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Kappa-IoU
|
|
20
|
+
|
|
21
|
+
This Python package implements the Kappa-IoU measure of Inter-annotator reliability from Rogers (2026):
|
|
22
|
+
|
|
23
|
+
Rogers, Trevor F. (2026). Kappa-IoU: Inter-Rater Reliability for Spatial Annotation
|
|
24
|
+
|
|
25
|
+
Large-scale annotation projects for which automated annotation solutions are insufficient require a metric that measures the quality of annotation between annotators.
|
|
26
|
+
[Intersection-over-Union](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Rezatofighi%2C%20H.%2C%20Tsoi%2C%20N.%2C%20Gwak%2C%20J.%2C%20Sadeghian%2C%20A.%2C%20Reid%2C%20I.%2C%20%26%20Savarese%2C%20S.%20(2019).%20Generalized%20intersection%20over%20union%3A%20A%20metric%20and%20a%20loss%20for%20object%20detection.%20Proceedings%20of%20the%20IEEE%2FCVF%20Conference%20on%20Computer%20Vision%20and%20Pattern%20Recognition%20(CVPR).), or IoU, is traditionally used for spatial data but requires a ground truth.
|
|
27
|
+
[Cohen's Kappa](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Cohen%2C%20J.%20(1960).%20A%20coefficient%20of%20agreement%20for%20nominal%20scales.%C2%A0Educational%20and%20psychological%20measurement%2C%C2%A020(1)%2C%2037-46.) is a psychometric measure that measures the amount of agreement between raters, but is traditionally used for categorical data.
|
|
28
|
+
|
|
29
|
+
This repository contains a Python package implementing the [Kappa-IoU](https://drive.google.com/file/d/1QW1dbFWX1-rxz7XcvPkBdlR0BFKoBIwC/view?usp=sharing) paper, which indexes IoU with Cohen's Kappa, and handles differences in object cardinality between raters by treating it as a linear assignment problem using [Kuhn-Munkres](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Munkres%2C%20J.%20(1957).%20Algorithms%20for%20the%20assignment%20and%20transportation%20problems.%20Journal%20of%20the%20Society%20for%20Industrial%20and%20Applied%20Mathematics%2C%205(1)%2C%2032-38.) assignment. Also included is an adjusted version of Kappa-IoU that uses a version of the [BLEU](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Papineni%2C%20K.%2C%20Roukos%2C%20S.%2C%20Ward%2C%20T.%2C%20%26%20Zhu%2C%20W.%20J.%20(2002).%20BLEU%3A%20a%20method%20for%20automatic%20evaluation%20of%20machine%20translation.%20In%20Proceedings%20of%20the%2040th%20Annual%20Meeting%20of%20the%20Association%20for%20Computational%20Linguistics%20(pp.%20311-318).) measure for machine translation which instead scores N-objects for multi-object annotation. Along with the Python package implementing this measure, this repository also contains `Section4.py`, code that recreates the examples from that section of the paper.
|
|
30
|
+
|
|
31
|
+
To install this package from inside the downloaded folder from Github:
|
|
32
|
+
|
|
33
|
+
python -m pip install -e .
|
|
34
|
+
|
|
35
|
+
To use, run the analysis engine straight from your terminal shell across your dataset directories using the console entrypoint:
|
|
36
|
+
|
|
37
|
+
kappaiou-run --baseline /data/author_baseline.csv --dir /data/annotator_tranches/ --tau 0.70
|
|
38
|
+
|
|
39
|
+
The author baseline and annotator tranche CSV files are expected to have the filenames in the first column, label in the second column, and then xywh-format annotations in the following four columns.
|
|
40
|
+
|
|
41
|
+
MIT License
|
kappaiou-1.0.0/README.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Kappa-IoU
|
|
2
|
+
|
|
3
|
+
This Python package implements the Kappa-IoU measure of Inter-annotator reliability from Rogers (2026):
|
|
4
|
+
|
|
5
|
+
Rogers, Trevor F. (2026). Kappa-IoU: Inter-Rater Reliability for Spatial Annotation
|
|
6
|
+
|
|
7
|
+
Large-scale annotation projects for which automated annotation solutions are insufficient require a metric that measures the quality of annotation between annotators.
|
|
8
|
+
[Intersection-over-Union](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Rezatofighi%2C%20H.%2C%20Tsoi%2C%20N.%2C%20Gwak%2C%20J.%2C%20Sadeghian%2C%20A.%2C%20Reid%2C%20I.%2C%20%26%20Savarese%2C%20S.%20(2019).%20Generalized%20intersection%20over%20union%3A%20A%20metric%20and%20a%20loss%20for%20object%20detection.%20Proceedings%20of%20the%20IEEE%2FCVF%20Conference%20on%20Computer%20Vision%20and%20Pattern%20Recognition%20(CVPR).), or IoU, is traditionally used for spatial data but requires a ground truth.
|
|
9
|
+
[Cohen's Kappa](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Cohen%2C%20J.%20(1960).%20A%20coefficient%20of%20agreement%20for%20nominal%20scales.%C2%A0Educational%20and%20psychological%20measurement%2C%C2%A020(1)%2C%2037-46.) is a psychometric measure that measures the amount of agreement between raters, but is traditionally used for categorical data.
|
|
10
|
+
|
|
11
|
+
This repository contains a Python package implementing the [Kappa-IoU](https://drive.google.com/file/d/1QW1dbFWX1-rxz7XcvPkBdlR0BFKoBIwC/view?usp=sharing) paper, which indexes IoU with Cohen's Kappa, and handles differences in object cardinality between raters by treating it as a linear assignment problem using [Kuhn-Munkres](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Munkres%2C%20J.%20(1957).%20Algorithms%20for%20the%20assignment%20and%20transportation%20problems.%20Journal%20of%20the%20Society%20for%20Industrial%20and%20Applied%20Mathematics%2C%205(1)%2C%2032-38.) assignment. Also included is an adjusted version of Kappa-IoU that uses a version of the [BLEU](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=Papineni%2C%20K.%2C%20Roukos%2C%20S.%2C%20Ward%2C%20T.%2C%20%26%20Zhu%2C%20W.%20J.%20(2002).%20BLEU%3A%20a%20method%20for%20automatic%20evaluation%20of%20machine%20translation.%20In%20Proceedings%20of%20the%2040th%20Annual%20Meeting%20of%20the%20Association%20for%20Computational%20Linguistics%20(pp.%20311-318).) measure for machine translation which instead scores N-objects for multi-object annotation. Along with the Python package implementing this measure, this repository also contains `Section4.py`, code that recreates the examples from that section of the paper.
|
|
12
|
+
|
|
13
|
+
To install this package from inside the downloaded folder from Github:
|
|
14
|
+
|
|
15
|
+
python -m pip install -e .
|
|
16
|
+
|
|
17
|
+
To use, run the analysis engine straight from your terminal shell across your dataset directories using the console entrypoint:
|
|
18
|
+
|
|
19
|
+
kappaiou-run --baseline /data/author_baseline.csv --dir /data/annotator_tranches/ --tau 0.70
|
|
20
|
+
|
|
21
|
+
The author baseline and annotator tranche CSV files are expected to have the filenames in the first column, label in the second column, and then xywh-format annotations in the following four columns.
|
|
22
|
+
|
|
23
|
+
MIT License
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
cli.py
|
|
4
|
+
|
|
5
|
+
backend input parsing
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
import argparse
|
|
12
|
+
import csv
|
|
13
|
+
from collections import defaultdict
|
|
14
|
+
from .core import calculate_adjusted_kappa_iou
|
|
15
|
+
|
|
16
|
+
def parse_csv_annotations(file_path):
|
|
17
|
+
"""
|
|
18
|
+
Parses your standard input structure:
|
|
19
|
+
Col 1: Filename | Col 2: Label (Object Type) | Col 3-6+: Spatial Data [x, y, w, h]
|
|
20
|
+
|
|
21
|
+
Input:
|
|
22
|
+
- file_path (str): path to annotations file
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
- data (defaultdict): parsed annotations file
|
|
26
|
+
"""
|
|
27
|
+
data = defaultdict(lambda: defaultdict(list))
|
|
28
|
+
|
|
29
|
+
if not os.path.exists(file_path):
|
|
30
|
+
print(f"Error: Target file not found at {file_path}")
|
|
31
|
+
sys.exit(1)
|
|
32
|
+
|
|
33
|
+
with open(file_path, mode='r', newline='', encoding='utf-8') as f:
|
|
34
|
+
reader = csv.reader(f)
|
|
35
|
+
# Skip optional headers if detected via string characters
|
|
36
|
+
first_row = next(reader, None)
|
|
37
|
+
|
|
38
|
+
if first_row:
|
|
39
|
+
try:
|
|
40
|
+
float(first_row[2])
|
|
41
|
+
row_start = first_row
|
|
42
|
+
except ValueError:
|
|
43
|
+
row_start = None # Row was a string header
|
|
44
|
+
|
|
45
|
+
def _process_row(row):
|
|
46
|
+
if len(row) < 6:
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
filename = row[0]
|
|
50
|
+
label = row[1]
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
coords = [float(c) for c in row[2:6]]
|
|
54
|
+
data[filename][label].append(coords)
|
|
55
|
+
except ValueError:
|
|
56
|
+
pass # Row logging exception safety
|
|
57
|
+
|
|
58
|
+
if row_start:
|
|
59
|
+
_process_row(row_start)
|
|
60
|
+
for row in reader:
|
|
61
|
+
_process_row(row)
|
|
62
|
+
|
|
63
|
+
return data
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main():
|
|
67
|
+
parser = argparse.ArgumentParser(description="Run Kappa-IoU verification against downstream annotator tranches.")
|
|
68
|
+
parser.add_argument("--baseline",
|
|
69
|
+
required=True,
|
|
70
|
+
help="Path to your primary baseline CSV file.")
|
|
71
|
+
parser.add_argument("--dir",
|
|
72
|
+
required=True,
|
|
73
|
+
help="Directory containing all other annotator CSV files.")
|
|
74
|
+
parser.add_argument("--tau",
|
|
75
|
+
type=float,
|
|
76
|
+
default=0.70,
|
|
77
|
+
help="Expected spatial project acceptance threshold.")
|
|
78
|
+
|
|
79
|
+
args = parser.parse_args()
|
|
80
|
+
|
|
81
|
+
# Parse Master/Baseline Dataset Template
|
|
82
|
+
baseline_data = parse_csv_annotations(args.baseline)
|
|
83
|
+
|
|
84
|
+
# Gather other target CSV logs in target path directory
|
|
85
|
+
all_files = os.listdir(args.dir)
|
|
86
|
+
annotator_files = [f for f in all_files if f.endswith('.csv') and os.path.abspath(os.path.join(args.dir, f)) != os.path.abspath(args.baseline)]
|
|
87
|
+
|
|
88
|
+
if not annotator_files:
|
|
89
|
+
print(f"No valid secondary annotator CSV files located in target directory: {args.dir}")
|
|
90
|
+
return
|
|
91
|
+
|
|
92
|
+
print(f"=========================================================================")
|
|
93
|
+
print(f"KAPPA-IoU COMPLIANCE LAB PIPELINE RUNTIME ENGINE")
|
|
94
|
+
print(f"=========================================================================")
|
|
95
|
+
print(f"Master baseline: {os.path.basename(args.baseline)}")
|
|
96
|
+
print(f"Operational tolerance (\u03c4): {args.tau}")
|
|
97
|
+
print(f"Secondary systems found: {len(annotator_files)}")
|
|
98
|
+
print(f"-------------------------------------------------------------------------")
|
|
99
|
+
|
|
100
|
+
# Evaluate each annotator file independently
|
|
101
|
+
for target in annotator_files:
|
|
102
|
+
target_path = os.path.join(args.dir, target)
|
|
103
|
+
annotator_data = parse_csv_annotations(target_path)
|
|
104
|
+
|
|
105
|
+
cumulative_score = 0.0
|
|
106
|
+
evaluation_channels = 0
|
|
107
|
+
|
|
108
|
+
# Iterate cross-wise through every shared baseline domain image coordinate
|
|
109
|
+
for filename, labels_dict in baseline_data.items():
|
|
110
|
+
for label, base_boxes in labels_dict.items():
|
|
111
|
+
# Extract corresponding logs from target annotator
|
|
112
|
+
annotator_boxes = annotator_data.get(filename, {}).get(label, [])
|
|
113
|
+
|
|
114
|
+
score = calculate_adjusted_kappa_iou(base_boxes, annotator_boxes, tau=args.tau)
|
|
115
|
+
cumulative_score += score
|
|
116
|
+
evaluation_channels += 1
|
|
117
|
+
|
|
118
|
+
if evaluation_channels > 0:
|
|
119
|
+
final_metric = cumulative_score / evaluation_channels
|
|
120
|
+
print(f"Annotator File: {target:<30} | Adjusted Kappa-IoU: {final_metric:.4f}")
|
|
121
|
+
else:
|
|
122
|
+
print(f"Annotator File: {target:<30} | Adjusted Kappa-IoU: ERR (Zero matching image/label intersections)")
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
main()
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
Mathematical backend
|
|
4
|
+
|
|
5
|
+
- IoU
|
|
6
|
+
- Kappa-IoU
|
|
7
|
+
- adjusted Kappa-IoU (using modified BLEU for N-objects)
|
|
8
|
+
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from scipy.optimize import linear_sum_assignment
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def calculate_iou(boxA, boxB):
|
|
17
|
+
"""
|
|
18
|
+
Computes Intersection-over-Union (IoU) between two boxes in [xmin, ymin, w, h] format.
|
|
19
|
+
|
|
20
|
+
Input:
|
|
21
|
+
- boxA (list): xywh-format annotation
|
|
22
|
+
- boxB (list): xywh-format annotation
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
- Intersection-over-Union of boxA and boxB
|
|
26
|
+
"""
|
|
27
|
+
xA = max(boxA[0], boxB[0])
|
|
28
|
+
yA = max(boxA[1], boxB[1])
|
|
29
|
+
xB = min(boxA[0] + boxA[2], boxB[0] + boxB[2])
|
|
30
|
+
yB = min(boxA[1] + boxA[3], boxB[1] + boxB[3])
|
|
31
|
+
|
|
32
|
+
intersection_area = max(0, xB - xA) * max(0, yB - yA)
|
|
33
|
+
boxA_area = boxA[2] * boxA[3]
|
|
34
|
+
boxB_area = boxB[2] * boxB[3]
|
|
35
|
+
union_area = boxA_area + boxB_area - intersection_area
|
|
36
|
+
|
|
37
|
+
if union_area == 0:
|
|
38
|
+
return 0.0
|
|
39
|
+
|
|
40
|
+
return intersection_area / union_area
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def compute_image_kappa_iou(rater_a_boxes,
|
|
44
|
+
rater_b_boxes,
|
|
45
|
+
tau=0.70,
|
|
46
|
+
delta=1e-7):
|
|
47
|
+
"""
|
|
48
|
+
In case of raters annotating different numbers of objects on an image,
|
|
49
|
+
this difference is solved as a linear assignment problem via Kuhn-Munkres
|
|
50
|
+
and Kappa-IoU is calculated for a single object-set channel.
|
|
51
|
+
|
|
52
|
+
Input:
|
|
53
|
+
- rater_a_boxes (List): List of lists of boxes annotated by rater A
|
|
54
|
+
- rater_b_boxes (List): List of lists of boxes annotated by rater B
|
|
55
|
+
- tau (float): P_T, the acceptable threshold of agreement between rater A and rater B
|
|
56
|
+
- delta (float): stabilizing constant to prevent division by zero
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
- Kappa-IoU metric of rater A and rater B
|
|
60
|
+
"""
|
|
61
|
+
N_A = len(rater_a_boxes)
|
|
62
|
+
N_B = len(rater_b_boxes)
|
|
63
|
+
max_N = max(N_A, N_B)
|
|
64
|
+
|
|
65
|
+
if max_N == 0:
|
|
66
|
+
return 1.0 # Perfect consensus results on an empty image channel
|
|
67
|
+
|
|
68
|
+
# Build cost matrix (1 - IoU)
|
|
69
|
+
cost_matrix = np.zeros((N_A, N_B))
|
|
70
|
+
for j in range(N_A):
|
|
71
|
+
for k in range(N_B):
|
|
72
|
+
cost_matrix[j, k] = 1.0 - calculate_iou(rater_a_boxes[j], rater_b_boxes[k])
|
|
73
|
+
|
|
74
|
+
# Pad to square configuration with max cost penalty of 1.0
|
|
75
|
+
padded_cost = np.ones((max_N, max_N))
|
|
76
|
+
padded_cost[:N_A, :N_B] = cost_matrix
|
|
77
|
+
|
|
78
|
+
# Primal-dual optimization path routing
|
|
79
|
+
row_ind, col_ind = linear_sum_assignment(padded_cost)
|
|
80
|
+
|
|
81
|
+
optimized_ious = []
|
|
82
|
+
for r, c in zip(row_ind, col_ind):
|
|
83
|
+
if r >= N_A or c >= N_B:
|
|
84
|
+
optimized_ious.append(0.0) # Unmatched dummy penalty
|
|
85
|
+
else:
|
|
86
|
+
optimized_ious.append(1.0 - cost_matrix[r, c])
|
|
87
|
+
|
|
88
|
+
avg_iou = np.mean(optimized_ious)
|
|
89
|
+
|
|
90
|
+
P_0 = 1.0 / (1.0 - avg_iou + delta)
|
|
91
|
+
kappaiou = (P_0 - tau) / (1.0 - tau)
|
|
92
|
+
|
|
93
|
+
return kappaiou
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def calculate_adjusted_kappa_iou(rater_a_boxes,
|
|
97
|
+
rater_b_boxes,
|
|
98
|
+
tau=0.70,
|
|
99
|
+
delta=1e-7):
|
|
100
|
+
"""
|
|
101
|
+
Implements the BLEU-adapted geometric ranking summation framework.
|
|
102
|
+
Evaluates sub-sequences from 1 to N-objects and aggregates the reliability index.
|
|
103
|
+
|
|
104
|
+
Input:
|
|
105
|
+
- rater_a_boxes (List): List of lists of boxes annotated by rater A
|
|
106
|
+
- rater_b_boxes (List): List of lists of boxes annotated by rater B
|
|
107
|
+
- tau (float): P_T, the acceptable threshold of agreement between rater A and rater B
|
|
108
|
+
- delta (float): stabilizing constant to prevent division by zero
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
- Kappa-IoU metric of rater A and rater B adjusted by modified BLEU
|
|
112
|
+
"""
|
|
113
|
+
N_A = len(rater_a_boxes)
|
|
114
|
+
N_B = len(rater_b_boxes)
|
|
115
|
+
N = max(N_A, N_B)
|
|
116
|
+
|
|
117
|
+
if N == 0:
|
|
118
|
+
return 0.0
|
|
119
|
+
|
|
120
|
+
kappa_terms = []
|
|
121
|
+
|
|
122
|
+
# Evaluate sequentially across every depth layer from 1 to N
|
|
123
|
+
for n in range(1, N + 1):
|
|
124
|
+
sub_a = rater_a_boxes[:n]
|
|
125
|
+
sub_b = rater_b_boxes[:n]
|
|
126
|
+
k_n = compute_image_kappa_iou(sub_a, sub_b, tau=tau, delta=delta)
|
|
127
|
+
kappa_terms.append(k_n)
|
|
128
|
+
|
|
129
|
+
# Structural geometric log-precision summation sequence
|
|
130
|
+
# Protects against taking natural logs of zero or negative Kappa outputs
|
|
131
|
+
valid_logs = [np.log(k) if k > 0 else np.log(delta) for k in kappa_terms]
|
|
132
|
+
adjusted_score = np.exp(np.sum((1.0 / N) * np.array(valid_logs)))
|
|
133
|
+
|
|
134
|
+
return adjusted_score
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "KappaIOU"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="F. Trevor Rogers", email="rogers72@hawaii.edu" }
|
|
10
|
+
]
|
|
11
|
+
description = "A training-free, chance-corrected reliability library for continuous multi-object spatial annotations."
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.8"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"numpy>=1.20.0",
|
|
22
|
+
"scipy>=1.7.0",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
"Homepage" = "https://github.com/tr7200/KappaIOU"
|
|
27
|
+
"Bug Tracker" = "https://github.com/tr7200/KappaIOU/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
kappaiou-run = "kappaiou.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["."]
|
|
34
|
+
include = ["kappaiou*"]
|
|
35
|
+
|
|
36
|
+
[tool.pytest.ini_options]
|
|
37
|
+
minversion = "6.0"
|
|
38
|
+
testpaths = ["tests"]
|
|
39
|
+
|
|
40
|
+
[tool.ruff]
|
|
41
|
+
line-length = 88
|
kappaiou-1.0.0/setup.cfg
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
Unit tests
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from kappaiou.core import (
|
|
11
|
+
calculate_iou,
|
|
12
|
+
compute_image_kappa_iou,
|
|
13
|
+
calculate_adjusted_kappa_iou
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
# =========================================================================
|
|
17
|
+
# EMPIRICAL TRUNCATED RAW DATA CONSTANTS (FROM MANUSCRIPT SECTION 4)
|
|
18
|
+
# =========================================================================
|
|
19
|
+
|
|
20
|
+
# Tranche 1: Single-Object (Cat)
|
|
21
|
+
T1_RATER_A = [[2.7439, 53.506, 373.6281, 273.018]]
|
|
22
|
+
T1_RATER_B = [[75.457, 65.854, 418.903, 257.469]]
|
|
23
|
+
|
|
24
|
+
# Tranche 2: Multi-Object Inverted Sequence (Cows)
|
|
25
|
+
T2_RATER_A = [
|
|
26
|
+
[96.951, 105.64024, 156.403, 166.92076], # Left Cow
|
|
27
|
+
[246.494, 169.665, 143.14015, 105.18256] # Right Cow
|
|
28
|
+
]
|
|
29
|
+
T2_RATER_B = [
|
|
30
|
+
[244.207, 173.323, 148.171, 101.982], # Right Cow (Inverted)
|
|
31
|
+
[94.207, 104.726, 160.061, 167.378] # Left Cow (Inverted)
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
# Tranche 3: Multi-Object Missingness/Symmetric Execution (Dogs)
|
|
35
|
+
T3_RATER_A = [
|
|
36
|
+
[92.835, 163.72, 126.22, 134.451], # Left Running Dog
|
|
37
|
+
[252.439, 194.36, 109.756, 80.488], # Center Spotted Dog
|
|
38
|
+
[325.61, 195.732, 91.463, 89.177] # Right Spotted Dog
|
|
39
|
+
]
|
|
40
|
+
T3_RATER_B = [
|
|
41
|
+
[97.5, 168.0, 123.5, 125.0], # Left Running Dog
|
|
42
|
+
[322.5, 190.5, 93.0, 94.5], # Right Spotted Dog (Inverted)
|
|
43
|
+
[254.5, 196.5, 109.5, 83.5] # Center Spotted Dog (Inverted)
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
# =========================================================================
|
|
47
|
+
# PYTEST COMPLIANCE LAB TEST SUITE
|
|
48
|
+
# =========================================================================
|
|
49
|
+
|
|
50
|
+
def test_tranche_1_single_object_scaling():
|
|
51
|
+
"""
|
|
52
|
+
Verifies that Tranche 1 (Cat) properly exhibits the open-ended positive
|
|
53
|
+
scaling characteristics when P_0 exceeds classical limits.
|
|
54
|
+
"""
|
|
55
|
+
tau = 0.70
|
|
56
|
+
delta = 1e-7
|
|
57
|
+
|
|
58
|
+
# Extract structural baseline behavior
|
|
59
|
+
kappa_score = compute_image_kappa_iou(T1_RATER_A, T1_RATER_B, tau=tau, delta=delta)
|
|
60
|
+
|
|
61
|
+
# Assertions grounded in Section 4 empirical findings
|
|
62
|
+
assert kappa_score > 1.0, f"Expected Kappa to break upper bound, got {kappa_score}"
|
|
63
|
+
assert np.isfinite(kappa_score), "Kappa calculation returned unstable non-finite float."
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_tranche_2_sequence_invariance():
|
|
67
|
+
"""
|
|
68
|
+
Verifies that the Kuhn-Munkres engine unscrambles arbitrary sequence inversions,
|
|
69
|
+
yielding a highly stable score regardless of rater entry order.
|
|
70
|
+
"""
|
|
71
|
+
tau = 0.70
|
|
72
|
+
delta = 1e-7
|
|
73
|
+
|
|
74
|
+
# Run structural assessment on the cows (inverted indexes)
|
|
75
|
+
kappa_score = compute_image_kappa_iou(T2_RATER_A, T2_RATER_B, tau=tau, delta=delta)
|
|
76
|
+
|
|
77
|
+
# Construct a perfectly sequential reference to verify invariance
|
|
78
|
+
sequenced_rater_b = [T2_RATER_B[1], T2_RATER_B[0]]
|
|
79
|
+
reference_score = compute_image_kappa_iou(T2_RATER_A, sequenced_rater_b, tau=tau, delta=delta)
|
|
80
|
+
|
|
81
|
+
# Assert that the optimization resolves order variance completely
|
|
82
|
+
assert kappa_score == pytest.approx(reference_score, rel=1e-6)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_tranche_3_adjusted_sequential_bleu():
|
|
86
|
+
"""
|
|
87
|
+
Verifies that the multi-object BLEU aggregation accurately combines
|
|
88
|
+
sub-sequence Kappa scores across varying depths up to N objects.
|
|
89
|
+
"""
|
|
90
|
+
tau = 0.70
|
|
91
|
+
delta = 1e-7
|
|
92
|
+
|
|
93
|
+
# Run full adjusted pipeline on the dogs dataset
|
|
94
|
+
adjusted_kappa = calculate_adjusted_kappa_iou(T3_RATER_A, T3_RATER_B, tau=tau, delta=delta)
|
|
95
|
+
|
|
96
|
+
# Verify that the nested geometric logging yields a stable bounded result
|
|
97
|
+
assert isinstance(adjusted_kappa, float)
|
|
98
|
+
assert adjusted_kappa > 0.0, f"Expected valid positive adjusted matrix score, got {adjusted_kappa}"
|