alignsurvey 0.2.0a4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ from .data import SurveyDataset, Results, example_directory
2
+ from .analysis import evaluate, compare, distribution_distance
3
+ from .survey import Question, Survey, Respondent, Scenario
4
+ from .models import Model, APIModel, LocalModel
5
+ from .experiment import Experiment
6
+ from .training import Trainer
7
+ from .tasks import Benchmark, predict_attribute, interview, predict_stance
8
+ from .extensions import (
9
+ Retriever,
10
+ select_examples,
11
+ summarize_interview,
12
+ conduct_interview,
13
+ generate_survey,
14
+ synthetic_population,
15
+ cross_conditions,
16
+ correlation,
17
+ )
18
+ from .report import export_report
19
+
20
+ from .judge import InterviewJudge
@@ -0,0 +1,26 @@
1
+ import argparse
2
+ import json
3
+ from . import Results, SurveyDataset, evaluate, example_directory
4
+
5
+
6
+ def main():
7
+ parser = argparse.ArgumentParser(description="Evaluate one survey run")
8
+ parser.add_argument(
9
+ "--dataset",
10
+ default=example_directory(),
11
+ help="Directory with four survey CSV files",
12
+ )
13
+ parser.add_argument(
14
+ "--answers",
15
+ help="External answer CSV (default: external.csv in dataset directory)",
16
+ )
17
+ args = parser.parse_args()
18
+ from pathlib import Path
19
+
20
+ data = SurveyDataset.load(args.dataset)
21
+ answers = Results.from_csv(args.answers or Path(args.dataset) / "external.csv")
22
+ print(json.dumps(evaluate(answers, reference=data), ensure_ascii=False, indent=2))
23
+
24
+
25
+ if __name__ == "__main__":
26
+ main()
@@ -0,0 +1,100 @@
1
+ from collections import Counter, defaultdict
2
+ from math import isfinite
3
+ from .data import KEY, Results, evaluate as basic_evaluation
4
+
5
+
6
+ def distribution_distance(a, b, ordered=False):
7
+ """Total variation (nominal) or unit-step Wasserstein (ordered)."""
8
+ if set(a) != set(b):
9
+ raise ValueError("Align option codes before comparison")
10
+ if any(not isfinite(v) or v < 0 for counts in (a, b) for v in counts.values()):
11
+ raise ValueError("Expected nonnegative finite masses")
12
+ x, y = sum(a.values()), sum(b.values())
13
+ if not x or not y:
14
+ return None
15
+ delta = [a[k] / x - b[k] / y for k in a]
16
+ if not ordered:
17
+ return sum(map(abs, delta)) / 2
18
+ from itertools import accumulate
19
+
20
+ return sum(abs(d) for d in accumulate(delta[:-1]))
21
+
22
+
23
+ def evaluate(results, reference, *, group_by=None, weight=None):
24
+ runs = {(r.get("experiment_id", ""), r.get("repeat", 0)) for r in results.records}
25
+ if len(runs) > 1:
26
+ return {
27
+ "runs": [
28
+ {
29
+ "experiment_id": e,
30
+ "repeat": n,
31
+ "report": evaluate(
32
+ Results(
33
+ [
34
+ r
35
+ for r in results.records
36
+ if (r.get("experiment_id", ""), r.get("repeat", 0))
37
+ == (e, n)
38
+ ]
39
+ ),
40
+ reference,
41
+ group_by=group_by,
42
+ weight=weight,
43
+ ),
44
+ }
45
+ for e, n in sorted(runs)
46
+ ]
47
+ }
48
+ report = basic_evaluation(results, reference)
49
+ counts = Counter(tuple(r[f] for f in KEY) for r in results.records)
50
+ pred = {tuple(r[f] for f in KEY): r for r in results.records}
51
+ tables = {}
52
+ for (i, q), assignment in reference.assignments.items():
53
+ if assignment["status"] != "asked":
54
+ continue
55
+ person = reference.respondents[(i,)]
56
+ w = float(person[weight]) if weight else 1.0
57
+ if not isfinite(w) or w < 0:
58
+ raise ValueError("Invalid respondent weight")
59
+ if reference.questions[(q,)]["question_type"] not in {"single", "ordinal"}:
60
+ continue
61
+ codes = reference.questions[(q,)]["option_codes"].split("|")
62
+ t = tables.setdefault(
63
+ q, {k: dict.fromkeys(codes, 0.0) for k in ("human", "model")}
64
+ )
65
+ key = (i, q)
66
+ if key in reference.reference:
67
+ t["human"][reference.reference[key]["answer_code"]] += w
68
+ r = pred.get(key)
69
+ if (
70
+ r
71
+ and counts[key] == 1
72
+ and r["status"] == "valid"
73
+ and r["answer_code"] in codes
74
+ ):
75
+ t["model"][r["answer_code"]] += w
76
+ for q, t in tables.items():
77
+ ordered = reference.questions[(q,)]["question_type"] == "ordinal"
78
+ t["distance"] = distribution_distance(t["human"], t["model"], ordered)
79
+ t["metric"] = "wasserstein_unit_step" if ordered else "total_variation"
80
+ report.update(distributions=tables, distribution_weight=weight)
81
+ if group_by:
82
+ groups = defaultdict(set)
83
+ for (i,), p in reference.respondents.items():
84
+ groups[p[group_by]].add(i)
85
+ report["groups"] = {
86
+ g: evaluate(
87
+ Results([r for r in results.records if r["respondent_id"] in ids]),
88
+ reference.subset(ids),
89
+ weight=weight,
90
+ )
91
+ for g, ids in groups.items()
92
+ }
93
+ return report
94
+
95
+
96
+ def compare(conditions, reference, **kwargs):
97
+ return {
98
+ name: evaluate(results, reference, **kwargs)
99
+ for name, results in conditions.items()
100
+ }
alignsurvey/data.py ADDED
@@ -0,0 +1,296 @@
1
+ """Offline survey evaluation. Identifiers and answer codes are strings."""
2
+ import csv
3
+ from collections import Counter, defaultdict
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+ KEY = ("respondent_id", "question_id")
8
+ STATUSES = {"valid", "parse_error", "api_error", "refused", "unknown", "not_applicable"}
9
+
10
+
11
+ def canonical_code(question, code):
12
+ import json
13
+
14
+ kind = question["question_type"]
15
+ if kind == "text":
16
+ return code if isinstance(code, str) and code.strip() else None
17
+ allowed = question["option_codes"].split("|")
18
+ if kind == "multi":
19
+ try:
20
+ values = json.loads(code)
21
+ except (ValueError, TypeError):
22
+ return None
23
+ if (
24
+ isinstance(values, list)
25
+ and all(isinstance(v, str) and v in allowed for v in values)
26
+ and len(set(values)) == len(values)
27
+ ):
28
+ return json.dumps(sorted(values), ensure_ascii=False)
29
+ return None
30
+ return code if code in allowed else None
31
+
32
+
33
+ def _rows(path, fields):
34
+ with Path(path).open(encoding="utf-8-sig", newline="") as stream:
35
+ reader = csv.DictReader(stream)
36
+ headers = reader.fieldnames or []
37
+ if len(set(headers)) != len(headers) or set(fields) - set(headers):
38
+ raise ValueError(f"{path}: expected unique headers including {fields}")
39
+ records = []
40
+ for row in reader:
41
+ if None in row or None in row.values():
42
+ raise ValueError(f"{path}:{reader.line_num}: malformed row")
43
+ records.append({k: v.strip() for k, v in row.items()})
44
+ return records
45
+
46
+
47
+ def _index(rows, fields):
48
+ index = {}
49
+ for row in rows:
50
+ key = tuple(row[name] for name in fields)
51
+ if not all(key) or key in index:
52
+ raise ValueError(f"Empty or duplicate key: {key}")
53
+ index[key] = row
54
+ return index
55
+
56
+
57
+ @dataclass
58
+ class SurveyDataset:
59
+ respondents: dict
60
+ questions: dict
61
+ assignments: dict
62
+ reference: dict
63
+
64
+ @classmethod
65
+ def load(cls, directory):
66
+ root = Path(directory)
67
+ people = _index(_rows(root / "respondents.csv", KEY[:1]), KEY[:1])
68
+ questions = _index(
69
+ _rows(
70
+ root / "questions.csv",
71
+ [
72
+ "question_id",
73
+ "section_path",
74
+ "question_text",
75
+ "question_type",
76
+ "option_codes",
77
+ "option_labels",
78
+ ],
79
+ ),
80
+ KEY[1:],
81
+ )
82
+ for q in questions.values():
83
+ codes = q["option_codes"].split("|")
84
+ labels = q["option_labels"].split("|")
85
+ if q["question_type"] == "text" and codes == labels == [""]:
86
+ continue
87
+ if (
88
+ q["question_type"] not in {"single", "ordinal", "multi"}
89
+ or not all(codes + labels)
90
+ or len(set(codes)) != len(codes)
91
+ or len(labels) != len(codes)
92
+ ):
93
+ raise ValueError(f"Invalid question definition: {q['question_id']}")
94
+ assignments = _index(_rows(root / "assignments.csv", (*KEY, "status")), KEY)
95
+ refs = (
96
+ _index(_rows(root / "reference.csv", (*KEY, "answer_code")), KEY)
97
+ if (root / "reference.csv").exists()
98
+ else {}
99
+ )
100
+ for (person, question), row in assignments.items():
101
+ if (
102
+ (person,) not in people
103
+ or (question,) not in questions
104
+ or row["status"] not in {"asked", "skipped"}
105
+ ):
106
+ raise ValueError(f"Invalid assignment: {person}, {question}")
107
+ for key, row in refs.items():
108
+ if (
109
+ assignments.get(key, {}).get("status") != "asked"
110
+ or canonical_code(questions[(key[1],)], row["answer_code"]) is None
111
+ ):
112
+ raise ValueError(f"Invalid reference: {key}")
113
+ return cls(people, questions, assignments, refs)
114
+
115
+ @classmethod
116
+ def from_tables(
117
+ cls, respondents, questions, assignments, reference=(), *, mapping=None
118
+ ):
119
+ """Accept record iterables or DataFrames. Mapping per table: source name -> standard name."""
120
+ import tempfile
121
+
122
+ tables = dict(
123
+ respondents=respondents,
124
+ questions=questions,
125
+ assignments=assignments,
126
+ reference=reference,
127
+ )
128
+ with tempfile.TemporaryDirectory() as tmp:
129
+ for name, table in tables.items():
130
+ if hasattr(table, "to_dict"):
131
+ table = table.to_dict(orient="records")
132
+ names = (mapping or {}).get(name, {})
133
+ rows = [{names.get(k, k): v for k, v in r.items()} for r in table]
134
+ if name == "reference" and not rows:
135
+ continue
136
+ keys = list(dict.fromkeys(k for r in rows for k in r))
137
+ with (Path(tmp) / (name + ".csv")).open(
138
+ "w", encoding="utf8", newline=""
139
+ ) as stream:
140
+ writer = csv.DictWriter(stream, keys)
141
+ writer.writeheader()
142
+ writer.writerows(rows)
143
+ return cls.load(tmp)
144
+
145
+ @classmethod
146
+ def from_json(cls, path, **kwargs):
147
+ import json
148
+
149
+ return cls.from_tables(
150
+ **json.loads(Path(path).read_text(encoding="utf8")), **kwargs
151
+ )
152
+
153
+ def subset(self, people):
154
+ ids = set(people)
155
+ return SurveyDataset(
156
+ {k: v for k, v in self.respondents.items() if k[0] in ids},
157
+ self.questions,
158
+ {k: v for k, v in self.assignments.items() if k[0] in ids},
159
+ {k: v for k, v in self.reference.items() if k[0] in ids},
160
+ )
161
+
162
+ def split(self, train=0.7, validation=0.1, seed=42):
163
+ import random
164
+
165
+ if min(train, validation) < 0 or train + validation > 1:
166
+ raise ValueError("Invalid split proportions")
167
+ ids = sorted(i for (i,) in self.respondents)
168
+ random.Random(seed).shuffle(ids)
169
+ a = int(len(ids) * train)
170
+ b = a + int(len(ids) * validation)
171
+ return self.subset(ids[:a]), self.subset(ids[a:b]), self.subset(ids[b:])
172
+
173
+
174
+ @dataclass
175
+ class Results:
176
+ records: list
177
+
178
+ @classmethod
179
+ def from_csv(cls, path):
180
+ return cls(_rows(path, (*KEY, "status", "answer_code")))
181
+
182
+ def save(self, path):
183
+ import json
184
+
185
+ Path(path).write_text(
186
+ json.dumps(self.records, ensure_ascii=False), encoding="utf-8"
187
+ )
188
+
189
+ @classmethod
190
+ def load(cls, path):
191
+ import json
192
+
193
+ return cls(json.loads(Path(path).read_text(encoding="utf-8")))
194
+
195
+ def select(self, **filters):
196
+ return Results(
197
+ [r for r in self.records if all(r.get(k) == v for k, v in filters.items())]
198
+ )
199
+
200
+ def to_long(self):
201
+ return [dict(r) for r in self.records]
202
+
203
+ def to_wide(self):
204
+ table = {}
205
+ for r in self.records:
206
+ key = (r.get("experiment_id", ""), r.get("repeat", 0), r["respondent_id"])
207
+ row = table.setdefault(
208
+ key,
209
+ {"experiment_id": key[0], "repeat": key[1], "respondent_id": key[2]},
210
+ )
211
+ col = "answer." + r["question_id"]
212
+ if col in row:
213
+ raise ValueError("Duplicate answer in wide table")
214
+ row[col] = r["answer_code"] if r["status"] == "valid" else None
215
+ return list(table.values())
216
+
217
+ def to_csv(self, path, wide=False):
218
+ rows = self.to_wide() if wide else self.to_long()
219
+ columns = list(dict.fromkeys(k for r in rows for k in r))
220
+ with Path(path).open("w", newline="", encoding="utf-8") as f:
221
+ writer = csv.DictWriter(f, columns)
222
+ writer.writeheader()
223
+ writer.writerows(rows)
224
+
225
+ def to_pandas(self, wide=False):
226
+ import pandas as pd
227
+
228
+ return pd.DataFrame(self.to_wide() if wide else self.to_long())
229
+
230
+
231
+ def _metric(numerator, denominator):
232
+ return dict(
233
+ numerator=numerator,
234
+ denominator=denominator,
235
+ value=numerator / denominator if denominator else None,
236
+ )
237
+
238
+
239
+ def evaluate(results, reference):
240
+ """Evaluate one run using assigned items and optional paired human answers."""
241
+ dataset = reference
242
+ asked = {k for k, v in dataset.assignments.items() if v["status"] == "asked"}
243
+ grouped = defaultdict(list)
244
+ for row in results.records:
245
+ grouped[tuple(row[f] for f in KEY)].append(row)
246
+ issues = defaultdict(list)
247
+ valid = matched = agreed = 0
248
+ status_counts = Counter()
249
+ for key in sorted(asked | grouped.keys()):
250
+ rows = grouped.get(key, [])
251
+ if not rows:
252
+ issues["missing"].append(key)
253
+ continue
254
+ if len(rows) > 1:
255
+ issues["duplicate"].append(key)
256
+ if key not in asked:
257
+ issues[
258
+ "skipped_with_output" if key in dataset.assignments else "unassigned"
259
+ ].append(key)
260
+ continue
261
+ if len(rows) != 1:
262
+ continue
263
+ row = rows[0]
264
+ status, code = row["status"], row["answer_code"]
265
+ question = dataset.questions[(key[1],)]
266
+ if status not in STATUSES or (status != "valid" and code):
267
+ issues["invalid_status"].append(key)
268
+ continue
269
+ if status == "valid" and canonical_code(question, code) is None:
270
+ issues["invalid_code"].append(key)
271
+ continue
272
+ status_counts[status] += 1
273
+ if status != "valid":
274
+ continue
275
+ valid += 1
276
+ if key not in dataset.reference:
277
+ issues["no_reference_for_valid_answer"].append(key)
278
+ continue
279
+ matched += 1
280
+ agreed += canonical_code(question, code) == canonical_code(
281
+ question, dataset.reference[key]["answer_code"]
282
+ )
283
+ return dict(
284
+ respondents=len(dataset.respondents),
285
+ assigned=len(asked),
286
+ skipped=len(dataset.assignments) - len(asked),
287
+ imported_rows=len(results.records),
288
+ issues=dict(issues),
289
+ status_counts=dict(status_counts),
290
+ valid_response_rate=_metric(valid, len(asked)),
291
+ paired_agreement=_metric(agreed, matched),
292
+ )
293
+
294
+
295
+ def example_directory():
296
+ return Path(__file__).parent / "sample"
@@ -0,0 +1,154 @@
1
+ import hashlib
2
+ import json
3
+ import os
4
+ import sqlite3
5
+ import time
6
+ from dataclasses import asdict
7
+ from pathlib import Path
8
+ from .data import Results
9
+ from .survey import Survey, Respondent, Scenario
10
+
11
+
12
+ class Experiment:
13
+ def __init__(
14
+ self,
15
+ survey,
16
+ respondents,
17
+ model,
18
+ *,
19
+ scenarios=None,
20
+ assignment=None,
21
+ history=False,
22
+ examples=(),
23
+ repeats=1
24
+ ):
25
+ self.survey, self.people, self.model = survey, list(respondents), model
26
+ self.scenarios = scenarios or {}
27
+ self.assignment = None if assignment is None else set(assignment)
28
+ self.history, self.examples, self.repeats = history, list(examples), repeats
29
+ if len({p.id for p in self.people}) != len(self.people) or repeats < 1:
30
+ raise ValueError("Unique respondents and positive repeats required")
31
+ if set(self.scenarios) - {p.id for p in self.people}:
32
+ raise ValueError("Scenario assigned to unknown person")
33
+
34
+ @classmethod
35
+ def from_dataset(cls, data, model, fields, **kw):
36
+ people = [
37
+ Respondent(i, {f: r[f] for f in fields})
38
+ for (i,), r in data.respondents.items()
39
+ ]
40
+ return cls(
41
+ Survey.from_dataset(data),
42
+ people,
43
+ model,
44
+ assignment=[
45
+ k for k, r in data.assignments.items() if r["status"] == "asked"
46
+ ],
47
+ **kw
48
+ )
49
+
50
+ def preview(self):
51
+ return [
52
+ {
53
+ "respondent_id": p.id,
54
+ "question_id": q.id,
55
+ "repeat": n,
56
+ "conditional": bool(q.when),
57
+ "prompt": self._prompt(p, q, {}),
58
+ }
59
+ for n in range(self.repeats)
60
+ for p in self.people
61
+ for q in self.survey.questions
62
+ if self._assigned(p, q)
63
+ ]
64
+
65
+ def _assigned(self, p, q):
66
+ return self.assignment is None or (p.id, q.id) in self.assignment
67
+
68
+ def _prompt(self, p, q, hist):
69
+ material = self.scenarios.get(p.id, Scenario("", "")).material
70
+ return self.survey.prompt(
71
+ p, q, material, hist if self.history else {}, self.examples
72
+ )
73
+
74
+ def run(self, checkpoint=None, *, retries=1, retry_errors=False):
75
+ if retries < 0:
76
+ raise ValueError("retries must be nonnegative")
77
+ spec = [
78
+ asdict(self.survey),
79
+ [asdict(p) for p in self.people],
80
+ self.model.config,
81
+ sorted(self.assignment) if self.assignment is not None else None,
82
+ {k: asdict(v) for k, v in self.scenarios.items()},
83
+ self.history,
84
+ self.examples,
85
+ self.repeats,
86
+ ]
87
+ signature = hashlib.sha256(
88
+ json.dumps(spec, sort_keys=True).encode()
89
+ ).hexdigest()
90
+ db = sqlite3.connect(str(checkpoint or ":memory:"))
91
+ rows = []
92
+ try:
93
+ db.execute(
94
+ "create table if not exists responses (spec text, person text, question text, repeat integer, record text, primary key(spec,person,question,repeat))"
95
+ )
96
+ for n in range(self.repeats):
97
+ for p in self.people:
98
+ hist = {}
99
+ for q in self.survey.questions:
100
+ if not self._assigned(p, q):
101
+ continue
102
+ key = (signature, p.id, q.id, n)
103
+ prompt = self._prompt(p, q, hist)
104
+ old = db.execute(
105
+ "select record from responses where spec=? and person=? and question=? and repeat=?",
106
+ key,
107
+ ).fetchone()
108
+ row = json.loads(old[0]) if old else {}
109
+ skipped = bool(q.when and hist.get(q.when[0]) != q.when[1])
110
+ if (
111
+ not row
112
+ or row["prompt"] != prompt
113
+ or (row["status"] == "skipped") != skipped
114
+ or (retry_errors and row["status"] == "api_error")
115
+ ):
116
+ raw, code, error = "", "", ""
117
+ status = "skipped"
118
+ if not skipped:
119
+ for attempt in range(retries + 1):
120
+ try:
121
+ raw = self.model.generate(prompt)
122
+ status, code = q.parse(raw)
123
+ error = ""
124
+ break
125
+ except Exception as exc:
126
+ error, status = type(exc).__name__, "api_error"
127
+ if attempt < retries:
128
+ time.sleep(min(2**attempt, 8))
129
+ row = dict(
130
+ experiment_id=signature,
131
+ respondent_id=p.id,
132
+ question_id=q.id,
133
+ repeat=n,
134
+ model=self.model.name,
135
+ scenario_id=self.scenarios.get(
136
+ p.id, Scenario("", "")
137
+ ).id,
138
+ raw_output=raw,
139
+ status=status,
140
+ answer_code=code,
141
+ error=error,
142
+ prompt=prompt,
143
+ )
144
+ with db:
145
+ db.execute(
146
+ "insert or replace into responses values(?,?,?,?,?)",
147
+ (*key, json.dumps(row)),
148
+ )
149
+ rows.append(row)
150
+ if row["status"] == "valid":
151
+ hist[q.id] = row["answer_code"]
152
+ finally:
153
+ db.close()
154
+ return Results(rows)