iris-check 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ evidence/otsenv/
@@ -0,0 +1,10 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this package, please cite it."
3
+ title: "IRIS Research Package: checks on language-model evaluation outputs before and after a run"
4
+ authors:
5
+ - name: "Heuremen LLC"
6
+ version: 1.0.0
7
+ date-released: "2026-11-01"
8
+ url: "https://indahl.ai/iris/package"
9
+ repository-code: "https://github.com/Heuremen-LLC/indahl.ai"
10
+ keywords: [language models, evaluation, benchmarks, LLM-as-judge, reproducibility, parser check, repeat floor]
@@ -0,0 +1,15 @@
1
+ Heuremen LLC
2
+ https://indahl.ai/iris
3
+
4
+ Code
5
+ Heuremen LLC permits anyone to run, copy and change the code in this package, for any purpose, with attribution to Heuremen LLC and https://indahl.ai/iris
6
+ The code is provided as it is, without warranty of any kind.
7
+
8
+ Text and data
9
+ The text and data in this package (the README, TESTED.md and any example or test data) are free to download, read and cite with attribution (Heuremen LLC, with a link to https://indahl.ai/iris).
10
+ All rights are reserved otherwise. Text and data mining, and use of this text or data to train, fine-tune or build AI models, are not permitted.
11
+ This is an express reservation of rights, also made in machine-readable form at https://indahl.ai/robots.txt and https://indahl.ai/.well-known/tdmrep.json.
12
+ Policy: https://indahl.ai/terms#ai-training
13
+
14
+ Your results
15
+ Your files and results remain yours.
@@ -0,0 +1,110 @@
1
+ Metadata-Version: 2.5
2
+ Name: iris-check
3
+ Version: 1.0.0
4
+ Summary: Checks on language-model evaluation outputs before you trust them: reading sheets, flips between repeats and paired comparison. From Heuremen LLC. Standard library only.
5
+ Project-URL: Homepage, https://indahl.ai/iris
6
+ Project-URL: Documentation, https://indahl.ai/iris/package
7
+ Project-URL: Source, https://github.com/Heuremen-LLC/indahl.ai
8
+ Author: Heuremen LLC
9
+ License: Heuremen LLC
10
+ https://indahl.ai/iris
11
+
12
+ Code
13
+ Heuremen LLC permits anyone to run, copy and change the code in this package, for any purpose, with attribution to Heuremen LLC and https://indahl.ai/iris
14
+ The code is provided as it is, without warranty of any kind.
15
+
16
+ Text and data
17
+ The text and data in this package (the README, TESTED.md and any example or test data) are free to download, read and cite with attribution (Heuremen LLC, with a link to https://indahl.ai/iris).
18
+ All rights are reserved otherwise. Text and data mining, and use of this text or data to train, fine-tune or build AI models, are not permitted.
19
+ This is an express reservation of rights, also made in machine-readable form at https://indahl.ai/robots.txt and https://indahl.ai/.well-known/tdmrep.json.
20
+ Policy: https://indahl.ai/terms#ai-training
21
+
22
+ Your results
23
+ Your files and results remain yours.
24
+ License-File: LICENSE
25
+ Keywords: LLM evaluation,McNemar,benchmark,bootstrap,reproducibility
26
+ Classifier: Environment :: Console
27
+ Classifier: Intended Audience :: Science/Research
28
+ Classifier: License :: Other/Proprietary License
29
+ Classifier: Operating System :: OS Independent
30
+ Classifier: Programming Language :: Python :: 3
31
+ Requires-Python: >=3.9
32
+ Description-Content-Type: text/markdown
33
+
34
+ # iris-check
35
+
36
+ Heuremen LLC · https://indahl.ai/iris
37
+
38
+ Tools to check the numbers from a language-model evaluation before you trust them and after a run. This applies to benchmark scores, LLM-as-judge scores and fine-tuning studies. IRIS recounts published evaluation studies from their own released output files, and this package puts the same checks in your hands for your own outputs. Your files are opened read-only and never changed. Reports go to a separate folder, and the checker refuses a folder inside the folder that holds an input, so run it from outside your data folder. The checker sends nothing anywhere. Two parts need other software or the network when you run them: `parser-check/replycheck.py` runs a model on your machine, and installing the Faults and Guards package from PyPI downloads it. Python 3.9 or later, standard library only.
39
+
40
+ ## Install
41
+
42
+ ```
43
+ pip install iris-check
44
+ ```
45
+
46
+ The tool is the command `iris-check`. It is also importable as the module `iris_check`.
47
+
48
+ ## Before a run
49
+
50
+ 1. Read the Guards before the work. Install the package with `pip install faultsguards`, then run `python -m faultsguards guards`
51
+ 2. Decide how each answer will be extracted from a reply and write that rule down before any reply exists.
52
+ 3. If you fine-tune or compare systems, plan at least two runs of the same items with the same settings. The difference between those two runs is the repeat floor, and it is the first thing any later difference is read against.
53
+ 4. Give the plan a check block. The check block and its packs for language-model evaluations (L) and public copy (P) are in the full package at https://indahl.ai/iris/package
54
+
55
+ ## If your file holds raw replies only
56
+
57
+ `iris-check repeat` needs a correctness field for each item, so score first. A few lines of Python are enough. This one reads a JSONL file of replies and a dictionary of gold answers, and writes the `{"id", "correct"}` lines the checker reads. Replace the comparison with your own extraction rule. Write `"correct": null` for an item you did not score, and it is counted as unscored and kept out of the drop warning.
58
+
59
+ ```
60
+ import json
61
+ gold = {"q1": "B", "q2": "17"} # item id -> the right answer
62
+ with open("replies.jsonl") as f, open("scored.jsonl", "w") as out:
63
+ for line in f:
64
+ r = json.loads(line)
65
+ correct = r["reply"].strip().split()[-1] == gold[r["id"]] # your own extraction and comparison
66
+ out.write(json.dumps({"id": r["id"], "correct": correct}) + "\n")
67
+ ```
68
+
69
+ ## After a run
70
+
71
+ 1. Save the outputs of each run as a JSONL file with one reply per line. Each line needs an item id, the reply, what your parser extracted and whether it was scored correct. Keep these files in one folder and write reports to a different folder.
72
+ 2. Read a sample of replies beside what your parser extracted:
73
+ `iris-check sample run1.jsonl --reply-field reply --extracted-field pred --label-field family --id-field id -n 10 --seed 1`
74
+ This writes a reading sheet as HTML and CSV with 10 random replies per label. Read each reply. In the CSV, write x in the `read` column for every reply you read. In the `misparsed` column write x, bad, 1, wrong or misparsed where the extracted value is not the answer the reply gives. A parse that is fine takes ok, fine, 0, no, n, false or a blank. Any other entry is an error that names the row. That includes yes and y, which could mean either. Keep the `sample_....json` record beside the CSV. Then run `iris-check sample --tally SHEET.csv`, which prints the result for each label and writes its report to `./heuremen_report_tally` unless you give `--out`. The record is found by the sheet's file name, the same name ending in `.json`, so keep the two files together and do not rename one. If the record has another name or place, add `--record sample_....json`. The CSV is written as UTF-8 with a byte order mark so Excel shows accents. If you save it again from Excel, choose "CSV UTF-8". A file saved in the Windows code page is refused with that instruction.
75
+ 3. The tally rule, exactly: a label reads "cannot tell (rows missing)" if the sheet holds fewer rows for it than the sample record says. Otherwise it reads "cannot tell (not read)" if any of its rows is unread. Otherwise it reads "cannot tell (parser)" when more than 10% of its read rows are mis-parsed, so 1 of 10 is fine, 2 of 10 is not, and 1 of 9 is not. Otherwise the rate can be read. The default sample is 10 replies per label.
76
+ 4. Reply text that starts with `=`, `+`, `-` or `@` gets a single quote in front of it in the CSV, so a spreadsheet does not run it as a formula. The `--extracted-field` should hold the extracted answer itself, not a scorer's verdict such as OK or FAULT, because the reader compares it with the reply. Items your scorer excludes by design should be set to JSON null in the extracted field, or left out of the file before you sample, so the reader is not shown them as parse failures. Rows where the reply or extracted field is absent or JSON null, blank lines, and rows with a missing or null label (shown as `(none)`) are counted on the console and in the JSON report. The tally takes the quote off again.
77
+ 5. Count the flips between repeats of the same items:
78
+ `iris-check repeat run1.jsonl run2.jsonl --id-field id --correct-field correct`
79
+ Item ids are turned into strings first, so 2, 2.0 and "2" are one id. Each pair of runs is compared on the items both runs hold. For every pair the output first gives the flip count, as `flips N: right->wrong a, wrong->right b, of n matched`, and then the share of matched items that flip. The share is called "flip share between these runs". It is called the "repeat floor" only when you add `--repeats`, which says that the runs are repeats of the same setup (same model, same settings, same machine). The net change alone hides flips that cancel. With three or more runs there is no combined figure, you get one row per pair of runs.
80
+ One file should hold one model, or the model should be part of the id. If a file pools several models whose item ids repeat, give every field that makes an item unique, for example `--id-field model id`, and the fields are joined into one id. When repeated ids leave no matched items, the output says so in plain words and names this fix. The command exits with status 1 and says why whenever a comparison matches no items.
81
+ 6. The report lists, for each run, how many rows were left out and why, with up to 5 example ids. Each row is counted once, under the first reason that applies, in this order: id missing, correctness field missing, correctness field explicitly null (unscored), correctness value not accepted, duplicate id. Unscored rows (null correct) are reported on their own and are not counted as errors in the 5% warning. A duplicate id leaves out every row that holds it, and duplicate rows and duplicate ids are counted separately. The console prints rows read = usable + the reasons. The correctness value is accepted as true or false, as the numbers 1 and 0 (1.0 and 0.0 too), or as text in any case: true, t, yes, y, correct, 1 or false, f, no, n, incorrect, 0. The report also lists the items present in one run of a pair and not the other. If more than 5% of a run is left out of a comparison, a warning is printed on the console and shown at the top of the report. It always names the count of ids not in the other run, next to the in-file reasons.
82
+ 7. Compare two systems on the same items:
83
+ `iris-check repeat a_run1.jsonl a_run2.jsonl --vs b_run1.jsonl b_run2.jsonl --id-field id --correct-field correct --margin 0.02`
84
+ Run k of A is paired with run k of B, in the order given on the command line. Each pair gets an exact two-sided McNemar test on the discordant items and a bootstrap 95% interval for the accuracy difference A minus B. The number of matched items and the discordant counts are always printed. Both the interval and the McNemar p are used, in this order. ABOVE if the low end of the interval is above 0 and p is below 0.05. BELOW if the high end is below 0 and p is below 0.05. If only one of the two says different, CANNOT_TELL. Then CONSISTENT if the whole interval is within plus or minus the margin, but only with at least 20 matched items and at least 1 discordant item. Otherwise CANNOT_TELL. Because ABOVE and BELOW are checked before CONSISTENT, a small difference that is resolved reads ABOVE or BELOW even when it is inside the margin. Above 5000 matched items the interval uses count draws (two binomial draws per resample) instead of resampling items one by one. It is approximate and can differ slightly by Python version, because before Python 3.12 it uses a normal approximation. A p below 1e-300 is printed as <1e-300.
85
+ 8. Read your write-up against the Faults: `python -m faultsguards faults`
86
+ 9. Give each reading a check block before it goes in a paper or a page. The check block is in the full package.
87
+
88
+ ## Test the checker
89
+
90
+ The source distribution holds `tests/` with a synthetic dataset where the answers are planted. These are the flips between repeats, the discordant items between systems and the parser errors. Unpack the source distribution and run `python tests/make_synthetic.py` and then `python tests/test_iris_check.py`, with `iris_check.py` and `tests/replycheck.py` copied into the folder that holds `tests`. How the package was tested is in TESTED.md, installed under `share/doc/iris-check/`.
91
+
92
+ ## The rest of the IRIS package
93
+
94
+ The full package adds the Faults and Guards package, the check block, FaultsBench and the parser-check script. It is at https://indahl.ai/iris/package
95
+
96
+ ## Links
97
+
98
+ - IRIS: https://indahl.ai/iris
99
+ - List of studies: https://indahl.ai/iris/list
100
+ - Key: https://indahl.ai/iris/key
101
+ - Package page: https://indahl.ai/iris/package
102
+ - Code: https://github.com/Heuremen-LLC/indahl.ai
103
+ - Faults and Guards on PyPI: https://pypi.org/project/faultsguards/
104
+ - Data and models: https://huggingface.co/indahl
105
+
106
+ ## Terms
107
+
108
+ Heuremen LLC permits anyone to run, copy and change the code in this package, for any purpose, with attribution to Heuremen LLC and https://indahl.ai/iris
109
+
110
+ Your results remain yours. The text and data follow LICENSE, and their use to train, fine-tune or build AI models is not permitted. The Faults and Guards package and its PDFs keep their own terms. LICENSE covers only the files in this package.
@@ -0,0 +1,77 @@
1
+ # iris-check
2
+
3
+ Heuremen LLC · https://indahl.ai/iris
4
+
5
+ Tools to check the numbers from a language-model evaluation before you trust them and after a run. This applies to benchmark scores, LLM-as-judge scores and fine-tuning studies. IRIS recounts published evaluation studies from their own released output files, and this package puts the same checks in your hands for your own outputs. Your files are opened read-only and never changed. Reports go to a separate folder, and the checker refuses a folder inside the folder that holds an input, so run it from outside your data folder. The checker sends nothing anywhere. Two parts need other software or the network when you run them: `parser-check/replycheck.py` runs a model on your machine, and installing the Faults and Guards package from PyPI downloads it. Python 3.9 or later, standard library only.
6
+
7
+ ## Install
8
+
9
+ ```
10
+ pip install iris-check
11
+ ```
12
+
13
+ The tool is the command `iris-check`. It is also importable as the module `iris_check`.
14
+
15
+ ## Before a run
16
+
17
+ 1. Read the Guards before the work. Install the package with `pip install faultsguards`, then run `python -m faultsguards guards`
18
+ 2. Decide how each answer will be extracted from a reply and write that rule down before any reply exists.
19
+ 3. If you fine-tune or compare systems, plan at least two runs of the same items with the same settings. The difference between those two runs is the repeat floor, and it is the first thing any later difference is read against.
20
+ 4. Give the plan a check block. The check block and its packs for language-model evaluations (L) and public copy (P) are in the full package at https://indahl.ai/iris/package
21
+
22
+ ## If your file holds raw replies only
23
+
24
+ `iris-check repeat` needs a correctness field for each item, so score first. A few lines of Python are enough. This one reads a JSONL file of replies and a dictionary of gold answers, and writes the `{"id", "correct"}` lines the checker reads. Replace the comparison with your own extraction rule. Write `"correct": null` for an item you did not score, and it is counted as unscored and kept out of the drop warning.
25
+
26
+ ```
27
+ import json
28
+ gold = {"q1": "B", "q2": "17"} # item id -> the right answer
29
+ with open("replies.jsonl") as f, open("scored.jsonl", "w") as out:
30
+ for line in f:
31
+ r = json.loads(line)
32
+ correct = r["reply"].strip().split()[-1] == gold[r["id"]] # your own extraction and comparison
33
+ out.write(json.dumps({"id": r["id"], "correct": correct}) + "\n")
34
+ ```
35
+
36
+ ## After a run
37
+
38
+ 1. Save the outputs of each run as a JSONL file with one reply per line. Each line needs an item id, the reply, what your parser extracted and whether it was scored correct. Keep these files in one folder and write reports to a different folder.
39
+ 2. Read a sample of replies beside what your parser extracted:
40
+ `iris-check sample run1.jsonl --reply-field reply --extracted-field pred --label-field family --id-field id -n 10 --seed 1`
41
+ This writes a reading sheet as HTML and CSV with 10 random replies per label. Read each reply. In the CSV, write x in the `read` column for every reply you read. In the `misparsed` column write x, bad, 1, wrong or misparsed where the extracted value is not the answer the reply gives. A parse that is fine takes ok, fine, 0, no, n, false or a blank. Any other entry is an error that names the row. That includes yes and y, which could mean either. Keep the `sample_....json` record beside the CSV. Then run `iris-check sample --tally SHEET.csv`, which prints the result for each label and writes its report to `./heuremen_report_tally` unless you give `--out`. The record is found by the sheet's file name, the same name ending in `.json`, so keep the two files together and do not rename one. If the record has another name or place, add `--record sample_....json`. The CSV is written as UTF-8 with a byte order mark so Excel shows accents. If you save it again from Excel, choose "CSV UTF-8". A file saved in the Windows code page is refused with that instruction.
42
+ 3. The tally rule, exactly: a label reads "cannot tell (rows missing)" if the sheet holds fewer rows for it than the sample record says. Otherwise it reads "cannot tell (not read)" if any of its rows is unread. Otherwise it reads "cannot tell (parser)" when more than 10% of its read rows are mis-parsed, so 1 of 10 is fine, 2 of 10 is not, and 1 of 9 is not. Otherwise the rate can be read. The default sample is 10 replies per label.
43
+ 4. Reply text that starts with `=`, `+`, `-` or `@` gets a single quote in front of it in the CSV, so a spreadsheet does not run it as a formula. The `--extracted-field` should hold the extracted answer itself, not a scorer's verdict such as OK or FAULT, because the reader compares it with the reply. Items your scorer excludes by design should be set to JSON null in the extracted field, or left out of the file before you sample, so the reader is not shown them as parse failures. Rows where the reply or extracted field is absent or JSON null, blank lines, and rows with a missing or null label (shown as `(none)`) are counted on the console and in the JSON report. The tally takes the quote off again.
44
+ 5. Count the flips between repeats of the same items:
45
+ `iris-check repeat run1.jsonl run2.jsonl --id-field id --correct-field correct`
46
+ Item ids are turned into strings first, so 2, 2.0 and "2" are one id. Each pair of runs is compared on the items both runs hold. For every pair the output first gives the flip count, as `flips N: right->wrong a, wrong->right b, of n matched`, and then the share of matched items that flip. The share is called "flip share between these runs". It is called the "repeat floor" only when you add `--repeats`, which says that the runs are repeats of the same setup (same model, same settings, same machine). The net change alone hides flips that cancel. With three or more runs there is no combined figure, you get one row per pair of runs.
47
+ One file should hold one model, or the model should be part of the id. If a file pools several models whose item ids repeat, give every field that makes an item unique, for example `--id-field model id`, and the fields are joined into one id. When repeated ids leave no matched items, the output says so in plain words and names this fix. The command exits with status 1 and says why whenever a comparison matches no items.
48
+ 6. The report lists, for each run, how many rows were left out and why, with up to 5 example ids. Each row is counted once, under the first reason that applies, in this order: id missing, correctness field missing, correctness field explicitly null (unscored), correctness value not accepted, duplicate id. Unscored rows (null correct) are reported on their own and are not counted as errors in the 5% warning. A duplicate id leaves out every row that holds it, and duplicate rows and duplicate ids are counted separately. The console prints rows read = usable + the reasons. The correctness value is accepted as true or false, as the numbers 1 and 0 (1.0 and 0.0 too), or as text in any case: true, t, yes, y, correct, 1 or false, f, no, n, incorrect, 0. The report also lists the items present in one run of a pair and not the other. If more than 5% of a run is left out of a comparison, a warning is printed on the console and shown at the top of the report. It always names the count of ids not in the other run, next to the in-file reasons.
49
+ 7. Compare two systems on the same items:
50
+ `iris-check repeat a_run1.jsonl a_run2.jsonl --vs b_run1.jsonl b_run2.jsonl --id-field id --correct-field correct --margin 0.02`
51
+ Run k of A is paired with run k of B, in the order given on the command line. Each pair gets an exact two-sided McNemar test on the discordant items and a bootstrap 95% interval for the accuracy difference A minus B. The number of matched items and the discordant counts are always printed. Both the interval and the McNemar p are used, in this order. ABOVE if the low end of the interval is above 0 and p is below 0.05. BELOW if the high end is below 0 and p is below 0.05. If only one of the two says different, CANNOT_TELL. Then CONSISTENT if the whole interval is within plus or minus the margin, but only with at least 20 matched items and at least 1 discordant item. Otherwise CANNOT_TELL. Because ABOVE and BELOW are checked before CONSISTENT, a small difference that is resolved reads ABOVE or BELOW even when it is inside the margin. Above 5000 matched items the interval uses count draws (two binomial draws per resample) instead of resampling items one by one. It is approximate and can differ slightly by Python version, because before Python 3.12 it uses a normal approximation. A p below 1e-300 is printed as <1e-300.
52
+ 8. Read your write-up against the Faults: `python -m faultsguards faults`
53
+ 9. Give each reading a check block before it goes in a paper or a page. The check block is in the full package.
54
+
55
+ ## Test the checker
56
+
57
+ The source distribution holds `tests/` with a synthetic dataset where the answers are planted. These are the flips between repeats, the discordant items between systems and the parser errors. Unpack the source distribution and run `python tests/make_synthetic.py` and then `python tests/test_iris_check.py`, with `iris_check.py` and `tests/replycheck.py` copied into the folder that holds `tests`. How the package was tested is in TESTED.md, installed under `share/doc/iris-check/`.
58
+
59
+ ## The rest of the IRIS package
60
+
61
+ The full package adds the Faults and Guards package, the check block, FaultsBench and the parser-check script. It is at https://indahl.ai/iris/package
62
+
63
+ ## Links
64
+
65
+ - IRIS: https://indahl.ai/iris
66
+ - List of studies: https://indahl.ai/iris/list
67
+ - Key: https://indahl.ai/iris/key
68
+ - Package page: https://indahl.ai/iris/package
69
+ - Code: https://github.com/Heuremen-LLC/indahl.ai
70
+ - Faults and Guards on PyPI: https://pypi.org/project/faultsguards/
71
+ - Data and models: https://huggingface.co/indahl
72
+
73
+ ## Terms
74
+
75
+ Heuremen LLC permits anyone to run, copy and change the code in this package, for any purpose, with attribution to Heuremen LLC and https://indahl.ai/iris
76
+
77
+ Your results remain yours. The text and data follow LICENSE, and their use to train, fine-tune or build AI models is not permitted. The Faults and Guards package and its PDFs keep their own terms. LICENSE covers only the files in this package.
@@ -0,0 +1,35 @@
1
+ # How this package was tested
2
+
3
+ Heuremen LLC · https://indahl.ai/iris
4
+
5
+ This page says what was run, on what, and what came out. It also says what was not tested.
6
+
7
+ ## The unit tests
8
+
9
+ `python tests/test_iris_check.py` runs 42 tests on a synthetic dataset in `tests/data/` where the answers are planted. The planted answers are the flips between repeats, the discordant items between two systems, and the parser errors. The tests check that those answers come back, and they also cover the exact test against a Fraction-based reference, a 1,000,000-item comparison, the row accounting, the tally rules, the CSV handling and the exit codes. The tests passed on Python 3.9, 3.12 and 3.14 on macOS.
10
+
11
+ ## A real-data test
12
+
13
+ We ran the package on our own two-machine evaluation round of 29 and 30 October 2026. That round ran FaultsBench on five open models, on a Mac and on a CUDA machine, with two same-machine repeats of each (the bare run and its exact copy). We turned the saved replies into one line per item with an id and a correct field, using the scorer from that round. Items the scorer could not read have a null correct field. The ids were joined with the model name, so the five models sit in one file per run.
14
+
15
+ The repeat command gave these flip counts. The two same-machine pairs were run with `--repeats`, and the two across-machine pairs were run without it, because they are not repeats of one setup. Each pair is compared on the items both runs hold.
16
+
17
+ - Mac repeats: 1 flip, on 393 matched items
18
+ - CUDA repeats: 4 flips, on 393 matched items
19
+ - Mac against CUDA, bare runs: 11 flips, on 391 matched items
20
+ - Mac against CUDA, copy runs: 10 flips, on 391 matched items
21
+
22
+ These equal the counts we published from separate code, the script `item_flips.py` and its output `item_flips.txt`, which are in the repository at https://github.com/Heuremen-LLC/indahl.ai/tree/main/code/iris/pub2/seats_compare and give flips 1, 4, 11 and 10 on the 391 items that all four runs hold. The two same-machine pairs hold 393 items each, and the 2 extra items have no flips. Both programs were written by us. So the agreement shows that the two implementations agree. It is not a check by anyone else.
23
+
24
+ The comparison command (`--vs`) on the Mac bare run and the CUDA bare run read CANNOT_TELL at the default margin of 0.02, with an exact two-sided McNemar p of 0.2266. The published value is 0.227.
25
+
26
+ ## A reading sheet on real replies
27
+
28
+ We wrote a reading sheet from real replies of the same round and a reader read 30 of them in full beside what the parser had extracted. 29 of the 30 extractions were correct. The one that was not was an item the scorer excludes by design.
29
+
30
+ ## What was not tested
31
+
32
+ - Output formats of other harnesses, such as lm-evaluation-harness and HELM. The package reads a JSONL file with the fields you name, and nothing more was tried.
33
+ - Files of more than 100,000 items on real data. The speed test with 1,000,000 items uses synthetic data.
34
+ - Windows and Linux. The tests ran on macOS.
35
+ - `parser-check/replycheck.py` runs a model, and only its `--help` and its argument handling are tested here.
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.18"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "iris-check"
7
+ version = "1.0.0"
8
+ description = "Checks on language-model evaluation outputs before you trust them: reading sheets, flips between repeats and paired comparison. From Heuremen LLC. Standard library only."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "Heuremen LLC" }]
13
+ keywords = ["LLM evaluation", "benchmark", "reproducibility", "McNemar", "bootstrap"]
14
+ classifiers = [
15
+ "Programming Language :: Python :: 3",
16
+ "Operating System :: OS Independent",
17
+ "Environment :: Console",
18
+ "Intended Audience :: Science/Research",
19
+ "License :: Other/Proprietary License",
20
+ ]
21
+ dependencies = []
22
+
23
+ [project.urls]
24
+ Homepage = "https://indahl.ai/iris"
25
+ Documentation = "https://indahl.ai/iris/package"
26
+ Source = "https://github.com/Heuremen-LLC/indahl.ai"
27
+
28
+ [project.scripts]
29
+ iris-check = "iris_check:main"
30
+
31
+ [tool.hatch.build.targets.wheel]
32
+ packages = ["src/iris_check.py"]
33
+
34
+ [tool.hatch.build.targets.wheel.shared-data]
35
+ "TESTED.md" = "share/doc/iris-check/TESTED.md"
36
+ "CITATION.cff" = "share/doc/iris-check/CITATION.cff"
37
+
38
+ [tool.hatch.build.targets.sdist]
39
+ include = ["src/iris_check.py", "README.md", "LICENSE", "TESTED.md", "CITATION.cff", "pyproject.toml", "tests"]