omrkit 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omrkit/__init__.py +76 -0
- omrkit/__main__.py +6 -0
- omrkit/api/__init__.py +5 -0
- omrkit/api/app.py +75 -0
- omrkit/api/routes.py +490 -0
- omrkit/api/settings.py +55 -0
- omrkit/api/static/assets/index-BFfdzDls.js +224 -0
- omrkit/api/static/assets/index-DRPOjlrM.css +1 -0
- omrkit/api/static/index.html +23 -0
- omrkit/cli/__init__.py +5 -0
- omrkit/cli/app.py +440 -0
- omrkit/core/__init__.py +44 -0
- omrkit/core/auto_scanner.py +132 -0
- omrkit/core/benchmark.py +204 -0
- omrkit/core/migrator.py +149 -0
- omrkit/core/scanner.py +281 -0
- omrkit/core/validators.py +62 -0
- omrkit/generator/__init__.py +13 -0
- omrkit/generator/pdf_sheet.py +508 -0
- omrkit/generator/presets.py +109 -0
- omrkit/grading/__init__.py +6 -0
- omrkit/grading/analytics.py +177 -0
- omrkit/grading/grader.py +191 -0
- omrkit/io/__init__.py +37 -0
- omrkit/io/exporters.py +355 -0
- omrkit/io/loader.py +207 -0
- omrkit/models/__init__.py +65 -0
- omrkit/models/grading.py +222 -0
- omrkit/models/scan.py +100 -0
- omrkit/models/template.py +285 -0
- omrkit/py.typed +0 -0
- omrkit/vision/__init__.py +30 -0
- omrkit/vision/detector.py +313 -0
- omrkit/vision/grid_finder.py +280 -0
- omrkit/vision/preprocess.py +57 -0
- omrkit/vision/qr.py +83 -0
- omrkit/vision/registration.py +486 -0
- omrkit/vision/visualizer.py +99 -0
- omrkit-2.1.0.dist-info/METADATA +284 -0
- omrkit-2.1.0.dist-info/RECORD +43 -0
- omrkit-2.1.0.dist-info/WHEEL +4 -0
- omrkit-2.1.0.dist-info/entry_points.txt +2 -0
- omrkit-2.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Classical Test Theory (CTT) psychometric analytics for graded exams."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from omrkit.models.grading import (
|
|
10
|
+
AnswerKey,
|
|
11
|
+
ExamAnalytics,
|
|
12
|
+
ItemPsychometrics,
|
|
13
|
+
ScoreHistogramBin,
|
|
14
|
+
StudentGrade,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def compute_exam_analytics(
|
|
19
|
+
grades: list[StudentGrade],
|
|
20
|
+
answer_key: AnswerKey,
|
|
21
|
+
) -> ExamAnalytics:
|
|
22
|
+
"""Compute score distribution, item difficulty (p), discrimination (D), and KR-20 reliability."""
|
|
23
|
+
question_ids = answer_key.question_ids()
|
|
24
|
+
default_answers = answer_key.resolve_booklet(answer_key.default_booklet)
|
|
25
|
+
|
|
26
|
+
if not grades:
|
|
27
|
+
return ExamAnalytics(
|
|
28
|
+
student_count=0,
|
|
29
|
+
question_count=len(question_ids),
|
|
30
|
+
mean_score=0.0,
|
|
31
|
+
median_score=0.0,
|
|
32
|
+
std_score=0.0,
|
|
33
|
+
min_score=0.0,
|
|
34
|
+
max_score=0.0,
|
|
35
|
+
mean_percentage=0.0,
|
|
36
|
+
percentiles={},
|
|
37
|
+
kr20_reliability=None,
|
|
38
|
+
histogram=[],
|
|
39
|
+
items=[],
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
scores = np.asarray([g.raw_score for g in grades], dtype=np.float64)
|
|
43
|
+
pcts = np.asarray([g.percentage for g in grades], dtype=np.float64)
|
|
44
|
+
n_students = len(grades)
|
|
45
|
+
|
|
46
|
+
percentiles = {
|
|
47
|
+
"p10": round(float(np.percentile(scores, 10)), 2),
|
|
48
|
+
"p25": round(float(np.percentile(scores, 25)), 2),
|
|
49
|
+
"p50": round(float(np.percentile(scores, 50)), 2),
|
|
50
|
+
"p75": round(float(np.percentile(scores, 75)), 2),
|
|
51
|
+
"p90": round(float(np.percentile(scores, 90)), 2),
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# Build 10-bin percentage histogram (0-10, 10-20, ..., 90-100)
|
|
55
|
+
histogram: list[ScoreHistogramBin] = []
|
|
56
|
+
for idx in range(10):
|
|
57
|
+
lo = idx * 10.0
|
|
58
|
+
hi = (idx + 1) * 10.0
|
|
59
|
+
if idx == 9:
|
|
60
|
+
count = int(np.sum((pcts >= lo) & (pcts <= hi)))
|
|
61
|
+
else:
|
|
62
|
+
count = int(np.sum((pcts >= lo) & (pcts < hi)))
|
|
63
|
+
histogram.append(
|
|
64
|
+
ScoreHistogramBin(
|
|
65
|
+
label=f"{int(lo)}-{int(hi)}%",
|
|
66
|
+
low=lo,
|
|
67
|
+
high=hi,
|
|
68
|
+
count=count,
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Sort students by total score for upper/lower 27% discrimination index
|
|
73
|
+
sorted_grades = sorted(grades, key=lambda g: g.raw_score, reverse=True)
|
|
74
|
+
group_n = max(1, round(n_students * 0.27))
|
|
75
|
+
upper_group = sorted_grades[:group_n]
|
|
76
|
+
lower_group = sorted_grades[-group_n:]
|
|
77
|
+
|
|
78
|
+
verdict_lookup = [{v.question_id: v for v in g.verdicts} for g in grades]
|
|
79
|
+
upper_lookup = [{v.question_id: v for v in g.verdicts} for g in upper_group]
|
|
80
|
+
lower_lookup = [{v.question_id: v for v in g.verdicts} for g in lower_group]
|
|
81
|
+
|
|
82
|
+
items: list[ItemPsychometrics] = []
|
|
83
|
+
sum_pq = 0.0
|
|
84
|
+
binary_totals = np.zeros(n_students, dtype=np.float64)
|
|
85
|
+
|
|
86
|
+
for qid in question_ids:
|
|
87
|
+
expected_raw = default_answers.get(qid, "")
|
|
88
|
+
expected_str = (
|
|
89
|
+
"/".join(expected_raw) if isinstance(expected_raw, list) else str(expected_raw)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
correct_c = 0
|
|
93
|
+
incorrect_c = 0
|
|
94
|
+
blank_c = 0
|
|
95
|
+
option_dist: dict[str, int] = {}
|
|
96
|
+
|
|
97
|
+
for s_idx, vmap in enumerate(verdict_lookup):
|
|
98
|
+
v = vmap.get(qid)
|
|
99
|
+
if v is None:
|
|
100
|
+
blank_c += 1
|
|
101
|
+
option_dist["BLANK"] = option_dist.get("BLANK", 0) + 1
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
key_opt = v.marked if v.marked else "BLANK"
|
|
105
|
+
option_dist[key_opt] = option_dist.get(key_opt, 0) + 1
|
|
106
|
+
|
|
107
|
+
if v.verdict in {"correct", "bonus"}:
|
|
108
|
+
correct_c += 1
|
|
109
|
+
binary_totals[s_idx] += 1.0
|
|
110
|
+
elif v.verdict == "blank":
|
|
111
|
+
blank_c += 1
|
|
112
|
+
else:
|
|
113
|
+
incorrect_c += 1
|
|
114
|
+
|
|
115
|
+
p_val = correct_c / float(n_students)
|
|
116
|
+
sum_pq += p_val * (1.0 - p_val)
|
|
117
|
+
omit_rate = blank_c / float(n_students)
|
|
118
|
+
|
|
119
|
+
if n_students >= 2:
|
|
120
|
+
upper_correct = sum(
|
|
121
|
+
1 for vm in upper_lookup if qid in vm and vm[qid].verdict in {"correct", "bonus"}
|
|
122
|
+
)
|
|
123
|
+
lower_correct = sum(
|
|
124
|
+
1 for vm in lower_lookup if qid in vm and vm[qid].verdict in {"correct", "bonus"}
|
|
125
|
+
)
|
|
126
|
+
disc_d = (upper_correct - lower_correct) / float(group_n)
|
|
127
|
+
else:
|
|
128
|
+
disc_d = 0.0
|
|
129
|
+
|
|
130
|
+
quality_flag: str = "good"
|
|
131
|
+
if n_students >= 4 and disc_d < 0.15:
|
|
132
|
+
quality_flag = "poor_discrimination"
|
|
133
|
+
elif p_val > 0.90:
|
|
134
|
+
quality_flag = "too_easy"
|
|
135
|
+
elif p_val < 0.25:
|
|
136
|
+
quality_flag = "too_hard"
|
|
137
|
+
|
|
138
|
+
items.append(
|
|
139
|
+
ItemPsychometrics(
|
|
140
|
+
question_id=qid,
|
|
141
|
+
expected=expected_str,
|
|
142
|
+
attempts=n_students,
|
|
143
|
+
correct_count=correct_c,
|
|
144
|
+
incorrect_count=incorrect_c,
|
|
145
|
+
blank_count=blank_c,
|
|
146
|
+
difficulty_p=round(p_val, 4),
|
|
147
|
+
discrimination_d=round(float(np.clip(disc_d, -1.0, 1.0)), 4),
|
|
148
|
+
omit_rate=round(omit_rate, 4),
|
|
149
|
+
option_distribution=dict(sorted(option_dist.items())),
|
|
150
|
+
quality_flag=quality_flag, # type: ignore[arg-type]
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# Kuder-Richardson Formula 20 (KR-20) internal consistency reliability
|
|
155
|
+
kr20: float | None = None
|
|
156
|
+
k = len(question_ids)
|
|
157
|
+
if k >= 2 and n_students >= 2:
|
|
158
|
+
total_var = float(np.var(binary_totals, ddof=1))
|
|
159
|
+
if total_var > 1e-9:
|
|
160
|
+
raw_kr20 = (k / float(k - 1)) * (1.0 - (sum_pq / total_var))
|
|
161
|
+
if not math.isnan(raw_kr20):
|
|
162
|
+
kr20 = round(float(np.clip(raw_kr20, -1.0, 1.0)), 4)
|
|
163
|
+
|
|
164
|
+
return ExamAnalytics(
|
|
165
|
+
student_count=n_students,
|
|
166
|
+
question_count=k,
|
|
167
|
+
mean_score=round(float(np.mean(scores)), 2),
|
|
168
|
+
median_score=round(float(np.median(scores)), 2),
|
|
169
|
+
std_score=round(float(np.std(scores, ddof=0)), 2),
|
|
170
|
+
min_score=round(float(np.min(scores)), 2),
|
|
171
|
+
max_score=round(float(np.max(scores)), 2),
|
|
172
|
+
mean_percentage=round(float(np.mean(pcts)), 2),
|
|
173
|
+
percentiles=percentiles,
|
|
174
|
+
kr20_reliability=kr20,
|
|
175
|
+
histogram=histogram,
|
|
176
|
+
items=items,
|
|
177
|
+
)
|
omrkit/grading/grader.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Flexible exam grader supporting multi-booklets, negative marking, and section rules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from omrkit.grading.analytics import compute_exam_analytics
|
|
6
|
+
from omrkit.models.grading import (
|
|
7
|
+
AnswerKey,
|
|
8
|
+
GradingPolicy,
|
|
9
|
+
GradingReport,
|
|
10
|
+
QuestionVerdict,
|
|
11
|
+
SectionRule,
|
|
12
|
+
SectionScore,
|
|
13
|
+
StudentGrade,
|
|
14
|
+
VerdictType,
|
|
15
|
+
)
|
|
16
|
+
from omrkit.models.scan import SheetResult
|
|
17
|
+
from omrkit.models.template import expand_field_labels
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ExamGrader:
|
|
21
|
+
"""Evaluates scanned OMR sheets against an AnswerKey and GradingPolicy."""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
answer_key: AnswerKey,
|
|
26
|
+
policy: GradingPolicy | None = None,
|
|
27
|
+
) -> None:
|
|
28
|
+
self.answer_key = answer_key
|
|
29
|
+
self.policy = policy or GradingPolicy()
|
|
30
|
+
self._bonus_set = set(expand_field_labels(self.policy.bonus_questions))
|
|
31
|
+
self._section_map: dict[str, SectionRule] = {}
|
|
32
|
+
self._expanded_sections: list[tuple[SectionRule, list[str]]] = []
|
|
33
|
+
|
|
34
|
+
for sec in self.policy.sections:
|
|
35
|
+
q_list = expand_field_labels(sec.questions)
|
|
36
|
+
self._expanded_sections.append((sec, q_list))
|
|
37
|
+
for qid in q_list:
|
|
38
|
+
self._section_map[qid] = sec
|
|
39
|
+
|
|
40
|
+
def _rule_for_question(self, qid: str) -> tuple[float, float, float]:
|
|
41
|
+
sec = self._section_map.get(qid)
|
|
42
|
+
if sec is not None:
|
|
43
|
+
return sec.correct_points, sec.incorrect_points, sec.blank_points
|
|
44
|
+
return (
|
|
45
|
+
self.policy.correct_points,
|
|
46
|
+
self.policy.incorrect_points,
|
|
47
|
+
self.policy.blank_points,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def grade_sheet(self, sheet: SheetResult) -> StudentGrade:
|
|
51
|
+
"""Grade one scanned optical sheet and produce a full StudentGrade scorecard."""
|
|
52
|
+
flat = sheet.flat_responses()
|
|
53
|
+
booklet = self.answer_key.default_booklet
|
|
54
|
+
if self.answer_key.booklet_field:
|
|
55
|
+
candidate = flat.get(self.answer_key.booklet_field, "").strip()
|
|
56
|
+
if candidate:
|
|
57
|
+
booklet = candidate
|
|
58
|
+
|
|
59
|
+
expected_map = self.answer_key.resolve_booklet(booklet)
|
|
60
|
+
verdicts: list[QuestionVerdict] = []
|
|
61
|
+
|
|
62
|
+
correct_cnt = 0
|
|
63
|
+
incorrect_cnt = 0
|
|
64
|
+
blank_cnt = 0
|
|
65
|
+
bonus_cnt = 0
|
|
66
|
+
raw_score = 0.0
|
|
67
|
+
max_score = 0.0
|
|
68
|
+
|
|
69
|
+
for qid, expected_raw in expected_map.items():
|
|
70
|
+
c_pts, i_pts, b_pts = self._rule_for_question(qid)
|
|
71
|
+
max_score += c_pts
|
|
72
|
+
|
|
73
|
+
field_reading = sheet.fields.get(qid)
|
|
74
|
+
marked = field_reading.value.strip() if field_reading else flat.get(qid, "").strip()
|
|
75
|
+
conf = field_reading.confidence if field_reading else 1.0
|
|
76
|
+
|
|
77
|
+
acceptable = (
|
|
78
|
+
[opt.strip() for opt in expected_raw]
|
|
79
|
+
if isinstance(expected_raw, list)
|
|
80
|
+
else [str(expected_raw).strip()]
|
|
81
|
+
)
|
|
82
|
+
expected_display = "/".join(acceptable)
|
|
83
|
+
|
|
84
|
+
verdict: VerdictType
|
|
85
|
+
points: float
|
|
86
|
+
|
|
87
|
+
if qid in self._bonus_set:
|
|
88
|
+
verdict = "bonus"
|
|
89
|
+
points = c_pts
|
|
90
|
+
bonus_cnt += 1
|
|
91
|
+
elif marked == "":
|
|
92
|
+
verdict = "blank"
|
|
93
|
+
points = b_pts
|
|
94
|
+
blank_cnt += 1
|
|
95
|
+
elif marked in acceptable:
|
|
96
|
+
verdict = "correct"
|
|
97
|
+
points = c_pts
|
|
98
|
+
correct_cnt += 1
|
|
99
|
+
elif (
|
|
100
|
+
self.policy.partial_credit
|
|
101
|
+
and len(marked) > 1
|
|
102
|
+
and any(ch in acceptable for ch in marked)
|
|
103
|
+
):
|
|
104
|
+
verdict = "correct"
|
|
105
|
+
points = round(c_pts / float(len(marked)), 4)
|
|
106
|
+
correct_cnt += 1
|
|
107
|
+
else:
|
|
108
|
+
verdict = "incorrect"
|
|
109
|
+
points = i_pts
|
|
110
|
+
incorrect_cnt += 1
|
|
111
|
+
|
|
112
|
+
raw_score += points
|
|
113
|
+
verdicts.append(
|
|
114
|
+
QuestionVerdict(
|
|
115
|
+
question_id=qid,
|
|
116
|
+
marked=marked,
|
|
117
|
+
expected=expected_display,
|
|
118
|
+
verdict=verdict,
|
|
119
|
+
points=round(points, 4),
|
|
120
|
+
confidence=conf,
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# Compute per-section subtotals when section rules are configured
|
|
125
|
+
verdict_by_q = {v.question_id: v for v in verdicts}
|
|
126
|
+
sections_out: list[SectionScore] = []
|
|
127
|
+
for sec, q_list in self._expanded_sections:
|
|
128
|
+
s_correct = 0
|
|
129
|
+
s_incorrect = 0
|
|
130
|
+
s_blank = 0
|
|
131
|
+
s_score = 0.0
|
|
132
|
+
s_max = 0.0
|
|
133
|
+
for qid in q_list:
|
|
134
|
+
if qid not in verdict_by_q:
|
|
135
|
+
continue
|
|
136
|
+
v = verdict_by_q[qid]
|
|
137
|
+
s_max += sec.correct_points
|
|
138
|
+
s_score += v.points
|
|
139
|
+
if v.verdict in {"correct", "bonus"}:
|
|
140
|
+
s_correct += 1
|
|
141
|
+
elif v.verdict == "blank":
|
|
142
|
+
s_blank += 1
|
|
143
|
+
else:
|
|
144
|
+
s_incorrect += 1
|
|
145
|
+
sections_out.append(
|
|
146
|
+
SectionScore(
|
|
147
|
+
name=sec.name,
|
|
148
|
+
correct=s_correct,
|
|
149
|
+
incorrect=s_incorrect,
|
|
150
|
+
blank=s_blank,
|
|
151
|
+
score=round(s_score, 2),
|
|
152
|
+
max_score=round(s_max, 2),
|
|
153
|
+
)
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
percentage = max(0.0, (raw_score / max_score) * 100.0) if max_score > 0 else 0.0
|
|
157
|
+
|
|
158
|
+
# Preserve student identity metadata (grouped fields + booklet)
|
|
159
|
+
student_meta = dict(sheet.grouped_values)
|
|
160
|
+
if self.answer_key.booklet_field:
|
|
161
|
+
student_meta[self.answer_key.booklet_field] = booklet
|
|
162
|
+
|
|
163
|
+
return StudentGrade(
|
|
164
|
+
sheet_id=sheet.sheet_id,
|
|
165
|
+
source_file=sheet.source_file,
|
|
166
|
+
student_metadata=student_meta,
|
|
167
|
+
booklet=booklet,
|
|
168
|
+
total_questions=len(expected_map),
|
|
169
|
+
correct=correct_cnt,
|
|
170
|
+
incorrect=incorrect_cnt,
|
|
171
|
+
blank=blank_cnt,
|
|
172
|
+
bonus=bonus_cnt,
|
|
173
|
+
raw_score=round(raw_score, 2),
|
|
174
|
+
max_score=round(max_score, 2),
|
|
175
|
+
percentage=round(percentage, 2),
|
|
176
|
+
flagged_for_review=len(sheet.review_items) > 0,
|
|
177
|
+
sections=sections_out,
|
|
178
|
+
verdicts=verdicts,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def grade_batch(self, sheets: list[SheetResult]) -> GradingReport:
|
|
182
|
+
"""Grade all valid sheets and produce cohort-level psychometric analytics."""
|
|
183
|
+
valid_sheets = [s for s in sheets if s.error is None]
|
|
184
|
+
grades = [self.grade_sheet(s) for s in valid_sheets]
|
|
185
|
+
analytics = compute_exam_analytics(grades, self.answer_key)
|
|
186
|
+
return GradingReport(
|
|
187
|
+
exam_name=self.answer_key.exam_name,
|
|
188
|
+
policy_name=self.policy.name,
|
|
189
|
+
grades=grades,
|
|
190
|
+
analytics=analytics,
|
|
191
|
+
)
|
omrkit/io/__init__.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Document I/O loaders and report exporters."""
|
|
2
|
+
|
|
3
|
+
from omrkit.io.exporters import (
|
|
4
|
+
export_csv,
|
|
5
|
+
export_csv_string,
|
|
6
|
+
export_excel,
|
|
7
|
+
export_excel_bytes,
|
|
8
|
+
export_json,
|
|
9
|
+
export_pdf_scorecard,
|
|
10
|
+
export_pdf_scorecard_bytes,
|
|
11
|
+
)
|
|
12
|
+
from omrkit.io.loader import (
|
|
13
|
+
SUPPORTED_EXTENSIONS,
|
|
14
|
+
DocumentLoadError,
|
|
15
|
+
LoadedPage,
|
|
16
|
+
discover_documents,
|
|
17
|
+
load_bytes,
|
|
18
|
+
load_document,
|
|
19
|
+
strip_exif_and_save,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"SUPPORTED_EXTENSIONS",
|
|
24
|
+
"DocumentLoadError",
|
|
25
|
+
"LoadedPage",
|
|
26
|
+
"discover_documents",
|
|
27
|
+
"export_csv",
|
|
28
|
+
"export_csv_string",
|
|
29
|
+
"export_excel",
|
|
30
|
+
"export_excel_bytes",
|
|
31
|
+
"export_json",
|
|
32
|
+
"export_pdf_scorecard",
|
|
33
|
+
"export_pdf_scorecard_bytes",
|
|
34
|
+
"load_bytes",
|
|
35
|
+
"load_document",
|
|
36
|
+
"strip_exif_and_save",
|
|
37
|
+
]
|