alignsurvey 0.2.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alignsurvey/__init__.py +20 -0
- alignsurvey/__main__.py +26 -0
- alignsurvey/analysis.py +100 -0
- alignsurvey/data.py +296 -0
- alignsurvey/experiment.py +154 -0
- alignsurvey/extensions.py +137 -0
- alignsurvey/judge.py +99 -0
- alignsurvey/models.py +87 -0
- alignsurvey/paper.py +366 -0
- alignsurvey/report.py +54 -0
- alignsurvey/sample/assignments.csv +9 -0
- alignsurvey/sample/external.csv +8 -0
- alignsurvey/sample/questions.csv +3 -0
- alignsurvey/sample/reference.csv +8 -0
- alignsurvey/sample/respondents.csv +5 -0
- alignsurvey/survey.py +108 -0
- alignsurvey/tasks.py +168 -0
- alignsurvey/training.py +148 -0
- alignsurvey-0.2.0a4.dist-info/METADATA +14 -0
- alignsurvey-0.2.0a4.dist-info/RECORD +22 -0
- alignsurvey-0.2.0a4.dist-info/WHEEL +5 -0
- alignsurvey-0.2.0a4.dist-info/top_level.txt +1 -0
alignsurvey/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .data import SurveyDataset, Results, example_directory
|
|
2
|
+
from .analysis import evaluate, compare, distribution_distance
|
|
3
|
+
from .survey import Question, Survey, Respondent, Scenario
|
|
4
|
+
from .models import Model, APIModel, LocalModel
|
|
5
|
+
from .experiment import Experiment
|
|
6
|
+
from .training import Trainer
|
|
7
|
+
from .tasks import Benchmark, predict_attribute, interview, predict_stance
|
|
8
|
+
from .extensions import (
|
|
9
|
+
Retriever,
|
|
10
|
+
select_examples,
|
|
11
|
+
summarize_interview,
|
|
12
|
+
conduct_interview,
|
|
13
|
+
generate_survey,
|
|
14
|
+
synthetic_population,
|
|
15
|
+
cross_conditions,
|
|
16
|
+
correlation,
|
|
17
|
+
)
|
|
18
|
+
from .report import export_report
|
|
19
|
+
|
|
20
|
+
from .judge import InterviewJudge
|
alignsurvey/__main__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
from . import Results, SurveyDataset, evaluate, example_directory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
parser = argparse.ArgumentParser(description="Evaluate one survey run")
|
|
8
|
+
parser.add_argument(
|
|
9
|
+
"--dataset",
|
|
10
|
+
default=example_directory(),
|
|
11
|
+
help="Directory with four survey CSV files",
|
|
12
|
+
)
|
|
13
|
+
parser.add_argument(
|
|
14
|
+
"--answers",
|
|
15
|
+
help="External answer CSV (default: external.csv in dataset directory)",
|
|
16
|
+
)
|
|
17
|
+
args = parser.parse_args()
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
data = SurveyDataset.load(args.dataset)
|
|
21
|
+
answers = Results.from_csv(args.answers or Path(args.dataset) / "external.csv")
|
|
22
|
+
print(json.dumps(evaluate(answers, reference=data), ensure_ascii=False, indent=2))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
main()
|
alignsurvey/analysis.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from collections import Counter, defaultdict
|
|
2
|
+
from math import isfinite
|
|
3
|
+
from .data import KEY, Results, evaluate as basic_evaluation
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def distribution_distance(a, b, ordered=False):
|
|
7
|
+
"""Total variation (nominal) or unit-step Wasserstein (ordered)."""
|
|
8
|
+
if set(a) != set(b):
|
|
9
|
+
raise ValueError("Align option codes before comparison")
|
|
10
|
+
if any(not isfinite(v) or v < 0 for counts in (a, b) for v in counts.values()):
|
|
11
|
+
raise ValueError("Expected nonnegative finite masses")
|
|
12
|
+
x, y = sum(a.values()), sum(b.values())
|
|
13
|
+
if not x or not y:
|
|
14
|
+
return None
|
|
15
|
+
delta = [a[k] / x - b[k] / y for k in a]
|
|
16
|
+
if not ordered:
|
|
17
|
+
return sum(map(abs, delta)) / 2
|
|
18
|
+
from itertools import accumulate
|
|
19
|
+
|
|
20
|
+
return sum(abs(d) for d in accumulate(delta[:-1]))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def evaluate(results, reference, *, group_by=None, weight=None):
|
|
24
|
+
runs = {(r.get("experiment_id", ""), r.get("repeat", 0)) for r in results.records}
|
|
25
|
+
if len(runs) > 1:
|
|
26
|
+
return {
|
|
27
|
+
"runs": [
|
|
28
|
+
{
|
|
29
|
+
"experiment_id": e,
|
|
30
|
+
"repeat": n,
|
|
31
|
+
"report": evaluate(
|
|
32
|
+
Results(
|
|
33
|
+
[
|
|
34
|
+
r
|
|
35
|
+
for r in results.records
|
|
36
|
+
if (r.get("experiment_id", ""), r.get("repeat", 0))
|
|
37
|
+
== (e, n)
|
|
38
|
+
]
|
|
39
|
+
),
|
|
40
|
+
reference,
|
|
41
|
+
group_by=group_by,
|
|
42
|
+
weight=weight,
|
|
43
|
+
),
|
|
44
|
+
}
|
|
45
|
+
for e, n in sorted(runs)
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
report = basic_evaluation(results, reference)
|
|
49
|
+
counts = Counter(tuple(r[f] for f in KEY) for r in results.records)
|
|
50
|
+
pred = {tuple(r[f] for f in KEY): r for r in results.records}
|
|
51
|
+
tables = {}
|
|
52
|
+
for (i, q), assignment in reference.assignments.items():
|
|
53
|
+
if assignment["status"] != "asked":
|
|
54
|
+
continue
|
|
55
|
+
person = reference.respondents[(i,)]
|
|
56
|
+
w = float(person[weight]) if weight else 1.0
|
|
57
|
+
if not isfinite(w) or w < 0:
|
|
58
|
+
raise ValueError("Invalid respondent weight")
|
|
59
|
+
if reference.questions[(q,)]["question_type"] not in {"single", "ordinal"}:
|
|
60
|
+
continue
|
|
61
|
+
codes = reference.questions[(q,)]["option_codes"].split("|")
|
|
62
|
+
t = tables.setdefault(
|
|
63
|
+
q, {k: dict.fromkeys(codes, 0.0) for k in ("human", "model")}
|
|
64
|
+
)
|
|
65
|
+
key = (i, q)
|
|
66
|
+
if key in reference.reference:
|
|
67
|
+
t["human"][reference.reference[key]["answer_code"]] += w
|
|
68
|
+
r = pred.get(key)
|
|
69
|
+
if (
|
|
70
|
+
r
|
|
71
|
+
and counts[key] == 1
|
|
72
|
+
and r["status"] == "valid"
|
|
73
|
+
and r["answer_code"] in codes
|
|
74
|
+
):
|
|
75
|
+
t["model"][r["answer_code"]] += w
|
|
76
|
+
for q, t in tables.items():
|
|
77
|
+
ordered = reference.questions[(q,)]["question_type"] == "ordinal"
|
|
78
|
+
t["distance"] = distribution_distance(t["human"], t["model"], ordered)
|
|
79
|
+
t["metric"] = "wasserstein_unit_step" if ordered else "total_variation"
|
|
80
|
+
report.update(distributions=tables, distribution_weight=weight)
|
|
81
|
+
if group_by:
|
|
82
|
+
groups = defaultdict(set)
|
|
83
|
+
for (i,), p in reference.respondents.items():
|
|
84
|
+
groups[p[group_by]].add(i)
|
|
85
|
+
report["groups"] = {
|
|
86
|
+
g: evaluate(
|
|
87
|
+
Results([r for r in results.records if r["respondent_id"] in ids]),
|
|
88
|
+
reference.subset(ids),
|
|
89
|
+
weight=weight,
|
|
90
|
+
)
|
|
91
|
+
for g, ids in groups.items()
|
|
92
|
+
}
|
|
93
|
+
return report
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def compare(conditions, reference, **kwargs):
|
|
97
|
+
return {
|
|
98
|
+
name: evaluate(results, reference, **kwargs)
|
|
99
|
+
for name, results in conditions.items()
|
|
100
|
+
}
|
alignsurvey/data.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Offline survey evaluation. Identifiers and answer codes are strings."""
|
|
2
|
+
import csv
|
|
3
|
+
from collections import Counter, defaultdict
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
KEY = ("respondent_id", "question_id")
|
|
8
|
+
STATUSES = {"valid", "parse_error", "api_error", "refused", "unknown", "not_applicable"}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def canonical_code(question, code):
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
kind = question["question_type"]
|
|
15
|
+
if kind == "text":
|
|
16
|
+
return code if isinstance(code, str) and code.strip() else None
|
|
17
|
+
allowed = question["option_codes"].split("|")
|
|
18
|
+
if kind == "multi":
|
|
19
|
+
try:
|
|
20
|
+
values = json.loads(code)
|
|
21
|
+
except (ValueError, TypeError):
|
|
22
|
+
return None
|
|
23
|
+
if (
|
|
24
|
+
isinstance(values, list)
|
|
25
|
+
and all(isinstance(v, str) and v in allowed for v in values)
|
|
26
|
+
and len(set(values)) == len(values)
|
|
27
|
+
):
|
|
28
|
+
return json.dumps(sorted(values), ensure_ascii=False)
|
|
29
|
+
return None
|
|
30
|
+
return code if code in allowed else None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _rows(path, fields):
|
|
34
|
+
with Path(path).open(encoding="utf-8-sig", newline="") as stream:
|
|
35
|
+
reader = csv.DictReader(stream)
|
|
36
|
+
headers = reader.fieldnames or []
|
|
37
|
+
if len(set(headers)) != len(headers) or set(fields) - set(headers):
|
|
38
|
+
raise ValueError(f"{path}: expected unique headers including {fields}")
|
|
39
|
+
records = []
|
|
40
|
+
for row in reader:
|
|
41
|
+
if None in row or None in row.values():
|
|
42
|
+
raise ValueError(f"{path}:{reader.line_num}: malformed row")
|
|
43
|
+
records.append({k: v.strip() for k, v in row.items()})
|
|
44
|
+
return records
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _index(rows, fields):
|
|
48
|
+
index = {}
|
|
49
|
+
for row in rows:
|
|
50
|
+
key = tuple(row[name] for name in fields)
|
|
51
|
+
if not all(key) or key in index:
|
|
52
|
+
raise ValueError(f"Empty or duplicate key: {key}")
|
|
53
|
+
index[key] = row
|
|
54
|
+
return index
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class SurveyDataset:
|
|
59
|
+
respondents: dict
|
|
60
|
+
questions: dict
|
|
61
|
+
assignments: dict
|
|
62
|
+
reference: dict
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def load(cls, directory):
|
|
66
|
+
root = Path(directory)
|
|
67
|
+
people = _index(_rows(root / "respondents.csv", KEY[:1]), KEY[:1])
|
|
68
|
+
questions = _index(
|
|
69
|
+
_rows(
|
|
70
|
+
root / "questions.csv",
|
|
71
|
+
[
|
|
72
|
+
"question_id",
|
|
73
|
+
"section_path",
|
|
74
|
+
"question_text",
|
|
75
|
+
"question_type",
|
|
76
|
+
"option_codes",
|
|
77
|
+
"option_labels",
|
|
78
|
+
],
|
|
79
|
+
),
|
|
80
|
+
KEY[1:],
|
|
81
|
+
)
|
|
82
|
+
for q in questions.values():
|
|
83
|
+
codes = q["option_codes"].split("|")
|
|
84
|
+
labels = q["option_labels"].split("|")
|
|
85
|
+
if q["question_type"] == "text" and codes == labels == [""]:
|
|
86
|
+
continue
|
|
87
|
+
if (
|
|
88
|
+
q["question_type"] not in {"single", "ordinal", "multi"}
|
|
89
|
+
or not all(codes + labels)
|
|
90
|
+
or len(set(codes)) != len(codes)
|
|
91
|
+
or len(labels) != len(codes)
|
|
92
|
+
):
|
|
93
|
+
raise ValueError(f"Invalid question definition: {q['question_id']}")
|
|
94
|
+
assignments = _index(_rows(root / "assignments.csv", (*KEY, "status")), KEY)
|
|
95
|
+
refs = (
|
|
96
|
+
_index(_rows(root / "reference.csv", (*KEY, "answer_code")), KEY)
|
|
97
|
+
if (root / "reference.csv").exists()
|
|
98
|
+
else {}
|
|
99
|
+
)
|
|
100
|
+
for (person, question), row in assignments.items():
|
|
101
|
+
if (
|
|
102
|
+
(person,) not in people
|
|
103
|
+
or (question,) not in questions
|
|
104
|
+
or row["status"] not in {"asked", "skipped"}
|
|
105
|
+
):
|
|
106
|
+
raise ValueError(f"Invalid assignment: {person}, {question}")
|
|
107
|
+
for key, row in refs.items():
|
|
108
|
+
if (
|
|
109
|
+
assignments.get(key, {}).get("status") != "asked"
|
|
110
|
+
or canonical_code(questions[(key[1],)], row["answer_code"]) is None
|
|
111
|
+
):
|
|
112
|
+
raise ValueError(f"Invalid reference: {key}")
|
|
113
|
+
return cls(people, questions, assignments, refs)
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def from_tables(
|
|
117
|
+
cls, respondents, questions, assignments, reference=(), *, mapping=None
|
|
118
|
+
):
|
|
119
|
+
"""Accept record iterables or DataFrames. Mapping per table: source name -> standard name."""
|
|
120
|
+
import tempfile
|
|
121
|
+
|
|
122
|
+
tables = dict(
|
|
123
|
+
respondents=respondents,
|
|
124
|
+
questions=questions,
|
|
125
|
+
assignments=assignments,
|
|
126
|
+
reference=reference,
|
|
127
|
+
)
|
|
128
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
129
|
+
for name, table in tables.items():
|
|
130
|
+
if hasattr(table, "to_dict"):
|
|
131
|
+
table = table.to_dict(orient="records")
|
|
132
|
+
names = (mapping or {}).get(name, {})
|
|
133
|
+
rows = [{names.get(k, k): v for k, v in r.items()} for r in table]
|
|
134
|
+
if name == "reference" and not rows:
|
|
135
|
+
continue
|
|
136
|
+
keys = list(dict.fromkeys(k for r in rows for k in r))
|
|
137
|
+
with (Path(tmp) / (name + ".csv")).open(
|
|
138
|
+
"w", encoding="utf8", newline=""
|
|
139
|
+
) as stream:
|
|
140
|
+
writer = csv.DictWriter(stream, keys)
|
|
141
|
+
writer.writeheader()
|
|
142
|
+
writer.writerows(rows)
|
|
143
|
+
return cls.load(tmp)
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def from_json(cls, path, **kwargs):
|
|
147
|
+
import json
|
|
148
|
+
|
|
149
|
+
return cls.from_tables(
|
|
150
|
+
**json.loads(Path(path).read_text(encoding="utf8")), **kwargs
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
def subset(self, people):
|
|
154
|
+
ids = set(people)
|
|
155
|
+
return SurveyDataset(
|
|
156
|
+
{k: v for k, v in self.respondents.items() if k[0] in ids},
|
|
157
|
+
self.questions,
|
|
158
|
+
{k: v for k, v in self.assignments.items() if k[0] in ids},
|
|
159
|
+
{k: v for k, v in self.reference.items() if k[0] in ids},
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
def split(self, train=0.7, validation=0.1, seed=42):
|
|
163
|
+
import random
|
|
164
|
+
|
|
165
|
+
if min(train, validation) < 0 or train + validation > 1:
|
|
166
|
+
raise ValueError("Invalid split proportions")
|
|
167
|
+
ids = sorted(i for (i,) in self.respondents)
|
|
168
|
+
random.Random(seed).shuffle(ids)
|
|
169
|
+
a = int(len(ids) * train)
|
|
170
|
+
b = a + int(len(ids) * validation)
|
|
171
|
+
return self.subset(ids[:a]), self.subset(ids[a:b]), self.subset(ids[b:])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@dataclass
|
|
175
|
+
class Results:
|
|
176
|
+
records: list
|
|
177
|
+
|
|
178
|
+
@classmethod
|
|
179
|
+
def from_csv(cls, path):
|
|
180
|
+
return cls(_rows(path, (*KEY, "status", "answer_code")))
|
|
181
|
+
|
|
182
|
+
def save(self, path):
|
|
183
|
+
import json
|
|
184
|
+
|
|
185
|
+
Path(path).write_text(
|
|
186
|
+
json.dumps(self.records, ensure_ascii=False), encoding="utf-8"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
@classmethod
|
|
190
|
+
def load(cls, path):
|
|
191
|
+
import json
|
|
192
|
+
|
|
193
|
+
return cls(json.loads(Path(path).read_text(encoding="utf-8")))
|
|
194
|
+
|
|
195
|
+
def select(self, **filters):
|
|
196
|
+
return Results(
|
|
197
|
+
[r for r in self.records if all(r.get(k) == v for k, v in filters.items())]
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
def to_long(self):
|
|
201
|
+
return [dict(r) for r in self.records]
|
|
202
|
+
|
|
203
|
+
def to_wide(self):
|
|
204
|
+
table = {}
|
|
205
|
+
for r in self.records:
|
|
206
|
+
key = (r.get("experiment_id", ""), r.get("repeat", 0), r["respondent_id"])
|
|
207
|
+
row = table.setdefault(
|
|
208
|
+
key,
|
|
209
|
+
{"experiment_id": key[0], "repeat": key[1], "respondent_id": key[2]},
|
|
210
|
+
)
|
|
211
|
+
col = "answer." + r["question_id"]
|
|
212
|
+
if col in row:
|
|
213
|
+
raise ValueError("Duplicate answer in wide table")
|
|
214
|
+
row[col] = r["answer_code"] if r["status"] == "valid" else None
|
|
215
|
+
return list(table.values())
|
|
216
|
+
|
|
217
|
+
def to_csv(self, path, wide=False):
|
|
218
|
+
rows = self.to_wide() if wide else self.to_long()
|
|
219
|
+
columns = list(dict.fromkeys(k for r in rows for k in r))
|
|
220
|
+
with Path(path).open("w", newline="", encoding="utf-8") as f:
|
|
221
|
+
writer = csv.DictWriter(f, columns)
|
|
222
|
+
writer.writeheader()
|
|
223
|
+
writer.writerows(rows)
|
|
224
|
+
|
|
225
|
+
def to_pandas(self, wide=False):
|
|
226
|
+
import pandas as pd
|
|
227
|
+
|
|
228
|
+
return pd.DataFrame(self.to_wide() if wide else self.to_long())
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _metric(numerator, denominator):
|
|
232
|
+
return dict(
|
|
233
|
+
numerator=numerator,
|
|
234
|
+
denominator=denominator,
|
|
235
|
+
value=numerator / denominator if denominator else None,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def evaluate(results, reference):
|
|
240
|
+
"""Evaluate one run using assigned items and optional paired human answers."""
|
|
241
|
+
dataset = reference
|
|
242
|
+
asked = {k for k, v in dataset.assignments.items() if v["status"] == "asked"}
|
|
243
|
+
grouped = defaultdict(list)
|
|
244
|
+
for row in results.records:
|
|
245
|
+
grouped[tuple(row[f] for f in KEY)].append(row)
|
|
246
|
+
issues = defaultdict(list)
|
|
247
|
+
valid = matched = agreed = 0
|
|
248
|
+
status_counts = Counter()
|
|
249
|
+
for key in sorted(asked | grouped.keys()):
|
|
250
|
+
rows = grouped.get(key, [])
|
|
251
|
+
if not rows:
|
|
252
|
+
issues["missing"].append(key)
|
|
253
|
+
continue
|
|
254
|
+
if len(rows) > 1:
|
|
255
|
+
issues["duplicate"].append(key)
|
|
256
|
+
if key not in asked:
|
|
257
|
+
issues[
|
|
258
|
+
"skipped_with_output" if key in dataset.assignments else "unassigned"
|
|
259
|
+
].append(key)
|
|
260
|
+
continue
|
|
261
|
+
if len(rows) != 1:
|
|
262
|
+
continue
|
|
263
|
+
row = rows[0]
|
|
264
|
+
status, code = row["status"], row["answer_code"]
|
|
265
|
+
question = dataset.questions[(key[1],)]
|
|
266
|
+
if status not in STATUSES or (status != "valid" and code):
|
|
267
|
+
issues["invalid_status"].append(key)
|
|
268
|
+
continue
|
|
269
|
+
if status == "valid" and canonical_code(question, code) is None:
|
|
270
|
+
issues["invalid_code"].append(key)
|
|
271
|
+
continue
|
|
272
|
+
status_counts[status] += 1
|
|
273
|
+
if status != "valid":
|
|
274
|
+
continue
|
|
275
|
+
valid += 1
|
|
276
|
+
if key not in dataset.reference:
|
|
277
|
+
issues["no_reference_for_valid_answer"].append(key)
|
|
278
|
+
continue
|
|
279
|
+
matched += 1
|
|
280
|
+
agreed += canonical_code(question, code) == canonical_code(
|
|
281
|
+
question, dataset.reference[key]["answer_code"]
|
|
282
|
+
)
|
|
283
|
+
return dict(
|
|
284
|
+
respondents=len(dataset.respondents),
|
|
285
|
+
assigned=len(asked),
|
|
286
|
+
skipped=len(dataset.assignments) - len(asked),
|
|
287
|
+
imported_rows=len(results.records),
|
|
288
|
+
issues=dict(issues),
|
|
289
|
+
status_counts=dict(status_counts),
|
|
290
|
+
valid_response_rate=_metric(valid, len(asked)),
|
|
291
|
+
paired_agreement=_metric(agreed, matched),
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def example_directory():
|
|
296
|
+
return Path(__file__).parent / "sample"
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sqlite3
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import asdict
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from .data import Results
|
|
9
|
+
from .survey import Survey, Respondent, Scenario
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Experiment:
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
survey,
|
|
16
|
+
respondents,
|
|
17
|
+
model,
|
|
18
|
+
*,
|
|
19
|
+
scenarios=None,
|
|
20
|
+
assignment=None,
|
|
21
|
+
history=False,
|
|
22
|
+
examples=(),
|
|
23
|
+
repeats=1
|
|
24
|
+
):
|
|
25
|
+
self.survey, self.people, self.model = survey, list(respondents), model
|
|
26
|
+
self.scenarios = scenarios or {}
|
|
27
|
+
self.assignment = None if assignment is None else set(assignment)
|
|
28
|
+
self.history, self.examples, self.repeats = history, list(examples), repeats
|
|
29
|
+
if len({p.id for p in self.people}) != len(self.people) or repeats < 1:
|
|
30
|
+
raise ValueError("Unique respondents and positive repeats required")
|
|
31
|
+
if set(self.scenarios) - {p.id for p in self.people}:
|
|
32
|
+
raise ValueError("Scenario assigned to unknown person")
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def from_dataset(cls, data, model, fields, **kw):
|
|
36
|
+
people = [
|
|
37
|
+
Respondent(i, {f: r[f] for f in fields})
|
|
38
|
+
for (i,), r in data.respondents.items()
|
|
39
|
+
]
|
|
40
|
+
return cls(
|
|
41
|
+
Survey.from_dataset(data),
|
|
42
|
+
people,
|
|
43
|
+
model,
|
|
44
|
+
assignment=[
|
|
45
|
+
k for k, r in data.assignments.items() if r["status"] == "asked"
|
|
46
|
+
],
|
|
47
|
+
**kw
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def preview(self):
|
|
51
|
+
return [
|
|
52
|
+
{
|
|
53
|
+
"respondent_id": p.id,
|
|
54
|
+
"question_id": q.id,
|
|
55
|
+
"repeat": n,
|
|
56
|
+
"conditional": bool(q.when),
|
|
57
|
+
"prompt": self._prompt(p, q, {}),
|
|
58
|
+
}
|
|
59
|
+
for n in range(self.repeats)
|
|
60
|
+
for p in self.people
|
|
61
|
+
for q in self.survey.questions
|
|
62
|
+
if self._assigned(p, q)
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
def _assigned(self, p, q):
|
|
66
|
+
return self.assignment is None or (p.id, q.id) in self.assignment
|
|
67
|
+
|
|
68
|
+
def _prompt(self, p, q, hist):
|
|
69
|
+
material = self.scenarios.get(p.id, Scenario("", "")).material
|
|
70
|
+
return self.survey.prompt(
|
|
71
|
+
p, q, material, hist if self.history else {}, self.examples
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def run(self, checkpoint=None, *, retries=1, retry_errors=False):
|
|
75
|
+
if retries < 0:
|
|
76
|
+
raise ValueError("retries must be nonnegative")
|
|
77
|
+
spec = [
|
|
78
|
+
asdict(self.survey),
|
|
79
|
+
[asdict(p) for p in self.people],
|
|
80
|
+
self.model.config,
|
|
81
|
+
sorted(self.assignment) if self.assignment is not None else None,
|
|
82
|
+
{k: asdict(v) for k, v in self.scenarios.items()},
|
|
83
|
+
self.history,
|
|
84
|
+
self.examples,
|
|
85
|
+
self.repeats,
|
|
86
|
+
]
|
|
87
|
+
signature = hashlib.sha256(
|
|
88
|
+
json.dumps(spec, sort_keys=True).encode()
|
|
89
|
+
).hexdigest()
|
|
90
|
+
db = sqlite3.connect(str(checkpoint or ":memory:"))
|
|
91
|
+
rows = []
|
|
92
|
+
try:
|
|
93
|
+
db.execute(
|
|
94
|
+
"create table if not exists responses (spec text, person text, question text, repeat integer, record text, primary key(spec,person,question,repeat))"
|
|
95
|
+
)
|
|
96
|
+
for n in range(self.repeats):
|
|
97
|
+
for p in self.people:
|
|
98
|
+
hist = {}
|
|
99
|
+
for q in self.survey.questions:
|
|
100
|
+
if not self._assigned(p, q):
|
|
101
|
+
continue
|
|
102
|
+
key = (signature, p.id, q.id, n)
|
|
103
|
+
prompt = self._prompt(p, q, hist)
|
|
104
|
+
old = db.execute(
|
|
105
|
+
"select record from responses where spec=? and person=? and question=? and repeat=?",
|
|
106
|
+
key,
|
|
107
|
+
).fetchone()
|
|
108
|
+
row = json.loads(old[0]) if old else {}
|
|
109
|
+
skipped = bool(q.when and hist.get(q.when[0]) != q.when[1])
|
|
110
|
+
if (
|
|
111
|
+
not row
|
|
112
|
+
or row["prompt"] != prompt
|
|
113
|
+
or (row["status"] == "skipped") != skipped
|
|
114
|
+
or (retry_errors and row["status"] == "api_error")
|
|
115
|
+
):
|
|
116
|
+
raw, code, error = "", "", ""
|
|
117
|
+
status = "skipped"
|
|
118
|
+
if not skipped:
|
|
119
|
+
for attempt in range(retries + 1):
|
|
120
|
+
try:
|
|
121
|
+
raw = self.model.generate(prompt)
|
|
122
|
+
status, code = q.parse(raw)
|
|
123
|
+
error = ""
|
|
124
|
+
break
|
|
125
|
+
except Exception as exc:
|
|
126
|
+
error, status = type(exc).__name__, "api_error"
|
|
127
|
+
if attempt < retries:
|
|
128
|
+
time.sleep(min(2**attempt, 8))
|
|
129
|
+
row = dict(
|
|
130
|
+
experiment_id=signature,
|
|
131
|
+
respondent_id=p.id,
|
|
132
|
+
question_id=q.id,
|
|
133
|
+
repeat=n,
|
|
134
|
+
model=self.model.name,
|
|
135
|
+
scenario_id=self.scenarios.get(
|
|
136
|
+
p.id, Scenario("", "")
|
|
137
|
+
).id,
|
|
138
|
+
raw_output=raw,
|
|
139
|
+
status=status,
|
|
140
|
+
answer_code=code,
|
|
141
|
+
error=error,
|
|
142
|
+
prompt=prompt,
|
|
143
|
+
)
|
|
144
|
+
with db:
|
|
145
|
+
db.execute(
|
|
146
|
+
"insert or replace into responses values(?,?,?,?,?)",
|
|
147
|
+
(*key, json.dumps(row)),
|
|
148
|
+
)
|
|
149
|
+
rows.append(row)
|
|
150
|
+
if row["status"] == "valid":
|
|
151
|
+
hist[q.id] = row["answer_code"]
|
|
152
|
+
finally:
|
|
153
|
+
db.close()
|
|
154
|
+
return Results(rows)
|