alignsurvey 0.2.0a5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alignsurvey-0.2.0a5/MANIFEST.in +1 -0
- alignsurvey-0.2.0a5/PKG-INFO +90 -0
- alignsurvey-0.2.0a5/PYPI_README.md +78 -0
- alignsurvey-0.2.0a5/README.md +78 -0
- alignsurvey-0.2.0a5/pyproject.toml +3 -0
- alignsurvey-0.2.0a5/setup.cfg +36 -0
- alignsurvey-0.2.0a5/src/alignsurvey/__init__.py +20 -0
- alignsurvey-0.2.0a5/src/alignsurvey/__main__.py +26 -0
- alignsurvey-0.2.0a5/src/alignsurvey/analysis.py +100 -0
- alignsurvey-0.2.0a5/src/alignsurvey/data.py +296 -0
- alignsurvey-0.2.0a5/src/alignsurvey/experiment.py +154 -0
- alignsurvey-0.2.0a5/src/alignsurvey/extensions.py +137 -0
- alignsurvey-0.2.0a5/src/alignsurvey/judge.py +99 -0
- alignsurvey-0.2.0a5/src/alignsurvey/models.py +87 -0
- alignsurvey-0.2.0a5/src/alignsurvey/paper.py +366 -0
- alignsurvey-0.2.0a5/src/alignsurvey/report.py +54 -0
- alignsurvey-0.2.0a5/src/alignsurvey/sample/assignments.csv +9 -0
- alignsurvey-0.2.0a5/src/alignsurvey/sample/external.csv +8 -0
- alignsurvey-0.2.0a5/src/alignsurvey/sample/questions.csv +3 -0
- alignsurvey-0.2.0a5/src/alignsurvey/sample/reference.csv +8 -0
- alignsurvey-0.2.0a5/src/alignsurvey/sample/respondents.csv +5 -0
- alignsurvey-0.2.0a5/src/alignsurvey/survey.py +108 -0
- alignsurvey-0.2.0a5/src/alignsurvey/tasks.py +168 -0
- alignsurvey-0.2.0a5/src/alignsurvey/training.py +148 -0
- alignsurvey-0.2.0a5/src/alignsurvey.egg-info/PKG-INFO +90 -0
- alignsurvey-0.2.0a5/src/alignsurvey.egg-info/SOURCES.txt +28 -0
- alignsurvey-0.2.0a5/src/alignsurvey.egg-info/dependency_links.txt +1 -0
- alignsurvey-0.2.0a5/src/alignsurvey.egg-info/requires.txt +11 -0
- alignsurvey-0.2.0a5/src/alignsurvey.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
include PYPI_README.md
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: alignsurvey
|
|
3
|
+
Version: 0.2.0a5
|
|
4
|
+
Summary: Run, fine-tune, and evaluate language models for social surveys
|
|
5
|
+
Project-URL: Research paper, https://doi.org/10.1609/aaai.v40i45.41236
|
|
6
|
+
Project-URL: Research benchmark, https://github.com/PiLab-ZJU/AlignSurvey
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Provides-Extra: local
|
|
10
|
+
Provides-Extra: analysis
|
|
11
|
+
Provides-Extra: paper
|
|
12
|
+
|
|
13
|
+
# AlignSurvey
|
|
14
|
+
|
|
15
|
+
**AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
|
|
16
|
+
|
|
17
|
+
## Install and run your first survey evaluation
|
|
18
|
+
|
|
19
|
+
The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
python -m pip install alignsurvey==0.2.0a5
|
|
23
|
+
python -m alignsurvey
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
|
|
27
|
+
|
|
28
|
+
## Run a baseline and compare it with imported answers
|
|
29
|
+
|
|
30
|
+
This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from alignsurvey import (
|
|
34
|
+
Experiment, Model, Results, SurveyDataset, compare,
|
|
35
|
+
example_directory, export_report,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
root = example_directory()
|
|
39
|
+
survey = SurveyDataset.load(root)
|
|
40
|
+
baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
|
|
41
|
+
experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
|
|
42
|
+
print(experiment.preview()[0]["prompt"])
|
|
43
|
+
|
|
44
|
+
runs = {
|
|
45
|
+
"baseline": experiment.run(),
|
|
46
|
+
"external": Results.from_csv(root / "external.csv"),
|
|
47
|
+
}
|
|
48
|
+
report = compare(runs, survey)
|
|
49
|
+
print(report["external"]["paired_agreement"])
|
|
50
|
+
export_report(report, "survey-comparison.html")
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
|
|
54
|
+
|
|
55
|
+
## Evaluate your own answers
|
|
56
|
+
|
|
57
|
+
Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
|
|
58
|
+
|
|
59
|
+
| File | One row represents | Required columns |
|
|
60
|
+
| --- | --- | --- |
|
|
61
|
+
| respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
|
|
62
|
+
| questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
|
|
63
|
+
| assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
|
|
64
|
+
| reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
|
|
65
|
+
|
|
66
|
+
For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
|
|
67
|
+
|
|
68
|
+
The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
python -m alignsurvey --dataset my_survey --answers my_predictions.csv
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Optional features
|
|
75
|
+
|
|
76
|
+
| Install | Use it for |
|
|
77
|
+
| --- | --- |
|
|
78
|
+
| `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
|
|
79
|
+
| `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
|
|
80
|
+
| `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
|
|
81
|
+
|
|
82
|
+
For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
|
|
83
|
+
|
|
84
|
+
For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
|
|
85
|
+
|
|
86
|
+
## Workshop materials and research
|
|
87
|
+
|
|
88
|
+
The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
|
|
89
|
+
|
|
90
|
+
For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# AlignSurvey
|
|
2
|
+
|
|
3
|
+
**AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
|
|
4
|
+
|
|
5
|
+
## Install and run your first survey evaluation
|
|
6
|
+
|
|
7
|
+
The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
python -m pip install alignsurvey==0.2.0a5
|
|
11
|
+
python -m alignsurvey
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
|
|
15
|
+
|
|
16
|
+
## Run a baseline and compare it with imported answers
|
|
17
|
+
|
|
18
|
+
This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from alignsurvey import (
|
|
22
|
+
Experiment, Model, Results, SurveyDataset, compare,
|
|
23
|
+
example_directory, export_report,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
root = example_directory()
|
|
27
|
+
survey = SurveyDataset.load(root)
|
|
28
|
+
baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
|
|
29
|
+
experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
|
|
30
|
+
print(experiment.preview()[0]["prompt"])
|
|
31
|
+
|
|
32
|
+
runs = {
|
|
33
|
+
"baseline": experiment.run(),
|
|
34
|
+
"external": Results.from_csv(root / "external.csv"),
|
|
35
|
+
}
|
|
36
|
+
report = compare(runs, survey)
|
|
37
|
+
print(report["external"]["paired_agreement"])
|
|
38
|
+
export_report(report, "survey-comparison.html")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
|
|
42
|
+
|
|
43
|
+
## Evaluate your own answers
|
|
44
|
+
|
|
45
|
+
Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
|
|
46
|
+
|
|
47
|
+
| File | One row represents | Required columns |
|
|
48
|
+
| --- | --- | --- |
|
|
49
|
+
| respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
|
|
50
|
+
| questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
|
|
51
|
+
| assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
|
|
52
|
+
| reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
|
|
53
|
+
|
|
54
|
+
For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
|
|
55
|
+
|
|
56
|
+
The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
python -m alignsurvey --dataset my_survey --answers my_predictions.csv
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Optional features
|
|
63
|
+
|
|
64
|
+
| Install | Use it for |
|
|
65
|
+
| --- | --- |
|
|
66
|
+
| `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
|
|
67
|
+
| `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
|
|
68
|
+
| `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
|
|
69
|
+
|
|
70
|
+
For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
|
|
71
|
+
|
|
72
|
+
For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
|
|
73
|
+
|
|
74
|
+
## Workshop materials and research
|
|
75
|
+
|
|
76
|
+
The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
|
|
77
|
+
|
|
78
|
+
For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# AlignSurvey
|
|
2
|
+
|
|
3
|
+
**AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
|
|
4
|
+
|
|
5
|
+
## Install and run your first survey evaluation
|
|
6
|
+
|
|
7
|
+
The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
python -m pip install alignsurvey==0.2.0a5
|
|
11
|
+
python -m alignsurvey
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
|
|
15
|
+
|
|
16
|
+
## Run a baseline and compare it with imported answers
|
|
17
|
+
|
|
18
|
+
This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from alignsurvey import (
|
|
22
|
+
Experiment, Model, Results, SurveyDataset, compare,
|
|
23
|
+
example_directory, export_report,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
root = example_directory()
|
|
27
|
+
survey = SurveyDataset.load(root)
|
|
28
|
+
baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
|
|
29
|
+
experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
|
|
30
|
+
print(experiment.preview()[0]["prompt"])
|
|
31
|
+
|
|
32
|
+
runs = {
|
|
33
|
+
"baseline": experiment.run(),
|
|
34
|
+
"external": Results.from_csv(root / "external.csv"),
|
|
35
|
+
}
|
|
36
|
+
report = compare(runs, survey)
|
|
37
|
+
print(report["external"]["paired_agreement"])
|
|
38
|
+
export_report(report, "survey-comparison.html")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
|
|
42
|
+
|
|
43
|
+
## Evaluate your own answers
|
|
44
|
+
|
|
45
|
+
Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
|
|
46
|
+
|
|
47
|
+
| File | One row represents | Required columns |
|
|
48
|
+
| --- | --- | --- |
|
|
49
|
+
| respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
|
|
50
|
+
| questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
|
|
51
|
+
| assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
|
|
52
|
+
| reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
|
|
53
|
+
|
|
54
|
+
For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
|
|
55
|
+
|
|
56
|
+
The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
python -m alignsurvey --dataset my_survey --answers my_predictions.csv
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Optional features
|
|
63
|
+
|
|
64
|
+
| Install | Use it for |
|
|
65
|
+
| --- | --- |
|
|
66
|
+
| `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
|
|
67
|
+
| `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
|
|
68
|
+
| `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
|
|
69
|
+
|
|
70
|
+
For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
|
|
71
|
+
|
|
72
|
+
For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
|
|
73
|
+
|
|
74
|
+
## Workshop materials and research
|
|
75
|
+
|
|
76
|
+
The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
|
|
77
|
+
|
|
78
|
+
For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[metadata]
|
|
2
|
+
name = alignsurvey
|
|
3
|
+
version = 0.2.0a5
|
|
4
|
+
description = Run, fine-tune, and evaluate language models for social surveys
|
|
5
|
+
long_description = file: PYPI_README.md
|
|
6
|
+
long_description_content_type = text/markdown
|
|
7
|
+
project_urls =
|
|
8
|
+
Research paper = https://doi.org/10.1609/aaai.v40i45.41236
|
|
9
|
+
Research benchmark = https://github.com/PiLab-ZJU/AlignSurvey
|
|
10
|
+
|
|
11
|
+
[options]
|
|
12
|
+
package_dir =
|
|
13
|
+
= src
|
|
14
|
+
packages = find:
|
|
15
|
+
python_requires = >=3.9
|
|
16
|
+
|
|
17
|
+
[options.packages.find]
|
|
18
|
+
where = src
|
|
19
|
+
|
|
20
|
+
[options.package_data]
|
|
21
|
+
alignsurvey = sample/*.csv
|
|
22
|
+
|
|
23
|
+
[options.extras_require]
|
|
24
|
+
local =
|
|
25
|
+
torch>=2.2
|
|
26
|
+
transformers>=4.45,<6
|
|
27
|
+
analysis =
|
|
28
|
+
pandas
|
|
29
|
+
paper =
|
|
30
|
+
jieba>=0.42
|
|
31
|
+
sentence-transformers>=3
|
|
32
|
+
|
|
33
|
+
[egg_info]
|
|
34
|
+
tag_build =
|
|
35
|
+
tag_date = 0
|
|
36
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .data import SurveyDataset, Results, example_directory
|
|
2
|
+
from .analysis import evaluate, compare, distribution_distance
|
|
3
|
+
from .survey import Question, Survey, Respondent, Scenario
|
|
4
|
+
from .models import Model, APIModel, LocalModel
|
|
5
|
+
from .experiment import Experiment
|
|
6
|
+
from .training import Trainer
|
|
7
|
+
from .tasks import Benchmark, predict_attribute, interview, predict_stance
|
|
8
|
+
from .extensions import (
|
|
9
|
+
Retriever,
|
|
10
|
+
select_examples,
|
|
11
|
+
summarize_interview,
|
|
12
|
+
conduct_interview,
|
|
13
|
+
generate_survey,
|
|
14
|
+
synthetic_population,
|
|
15
|
+
cross_conditions,
|
|
16
|
+
correlation,
|
|
17
|
+
)
|
|
18
|
+
from .report import export_report
|
|
19
|
+
|
|
20
|
+
from .judge import InterviewJudge
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
from . import Results, SurveyDataset, evaluate, example_directory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
parser = argparse.ArgumentParser(description="Evaluate one survey run")
|
|
8
|
+
parser.add_argument(
|
|
9
|
+
"--dataset",
|
|
10
|
+
default=example_directory(),
|
|
11
|
+
help="Directory with four survey CSV files",
|
|
12
|
+
)
|
|
13
|
+
parser.add_argument(
|
|
14
|
+
"--answers",
|
|
15
|
+
help="External answer CSV (default: external.csv in dataset directory)",
|
|
16
|
+
)
|
|
17
|
+
args = parser.parse_args()
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
data = SurveyDataset.load(args.dataset)
|
|
21
|
+
answers = Results.from_csv(args.answers or Path(args.dataset) / "external.csv")
|
|
22
|
+
print(json.dumps(evaluate(answers, reference=data), ensure_ascii=False, indent=2))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
main()
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from collections import Counter, defaultdict
|
|
2
|
+
from math import isfinite
|
|
3
|
+
from .data import KEY, Results, evaluate as basic_evaluation
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def distribution_distance(a, b, ordered=False):
|
|
7
|
+
"""Total variation (nominal) or unit-step Wasserstein (ordered)."""
|
|
8
|
+
if set(a) != set(b):
|
|
9
|
+
raise ValueError("Align option codes before comparison")
|
|
10
|
+
if any(not isfinite(v) or v < 0 for counts in (a, b) for v in counts.values()):
|
|
11
|
+
raise ValueError("Expected nonnegative finite masses")
|
|
12
|
+
x, y = sum(a.values()), sum(b.values())
|
|
13
|
+
if not x or not y:
|
|
14
|
+
return None
|
|
15
|
+
delta = [a[k] / x - b[k] / y for k in a]
|
|
16
|
+
if not ordered:
|
|
17
|
+
return sum(map(abs, delta)) / 2
|
|
18
|
+
from itertools import accumulate
|
|
19
|
+
|
|
20
|
+
return sum(abs(d) for d in accumulate(delta[:-1]))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def evaluate(results, reference, *, group_by=None, weight=None):
|
|
24
|
+
runs = {(r.get("experiment_id", ""), r.get("repeat", 0)) for r in results.records}
|
|
25
|
+
if len(runs) > 1:
|
|
26
|
+
return {
|
|
27
|
+
"runs": [
|
|
28
|
+
{
|
|
29
|
+
"experiment_id": e,
|
|
30
|
+
"repeat": n,
|
|
31
|
+
"report": evaluate(
|
|
32
|
+
Results(
|
|
33
|
+
[
|
|
34
|
+
r
|
|
35
|
+
for r in results.records
|
|
36
|
+
if (r.get("experiment_id", ""), r.get("repeat", 0))
|
|
37
|
+
== (e, n)
|
|
38
|
+
]
|
|
39
|
+
),
|
|
40
|
+
reference,
|
|
41
|
+
group_by=group_by,
|
|
42
|
+
weight=weight,
|
|
43
|
+
),
|
|
44
|
+
}
|
|
45
|
+
for e, n in sorted(runs)
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
report = basic_evaluation(results, reference)
|
|
49
|
+
counts = Counter(tuple(r[f] for f in KEY) for r in results.records)
|
|
50
|
+
pred = {tuple(r[f] for f in KEY): r for r in results.records}
|
|
51
|
+
tables = {}
|
|
52
|
+
for (i, q), assignment in reference.assignments.items():
|
|
53
|
+
if assignment["status"] != "asked":
|
|
54
|
+
continue
|
|
55
|
+
person = reference.respondents[(i,)]
|
|
56
|
+
w = float(person[weight]) if weight else 1.0
|
|
57
|
+
if not isfinite(w) or w < 0:
|
|
58
|
+
raise ValueError("Invalid respondent weight")
|
|
59
|
+
if reference.questions[(q,)]["question_type"] not in {"single", "ordinal"}:
|
|
60
|
+
continue
|
|
61
|
+
codes = reference.questions[(q,)]["option_codes"].split("|")
|
|
62
|
+
t = tables.setdefault(
|
|
63
|
+
q, {k: dict.fromkeys(codes, 0.0) for k in ("human", "model")}
|
|
64
|
+
)
|
|
65
|
+
key = (i, q)
|
|
66
|
+
if key in reference.reference:
|
|
67
|
+
t["human"][reference.reference[key]["answer_code"]] += w
|
|
68
|
+
r = pred.get(key)
|
|
69
|
+
if (
|
|
70
|
+
r
|
|
71
|
+
and counts[key] == 1
|
|
72
|
+
and r["status"] == "valid"
|
|
73
|
+
and r["answer_code"] in codes
|
|
74
|
+
):
|
|
75
|
+
t["model"][r["answer_code"]] += w
|
|
76
|
+
for q, t in tables.items():
|
|
77
|
+
ordered = reference.questions[(q,)]["question_type"] == "ordinal"
|
|
78
|
+
t["distance"] = distribution_distance(t["human"], t["model"], ordered)
|
|
79
|
+
t["metric"] = "wasserstein_unit_step" if ordered else "total_variation"
|
|
80
|
+
report.update(distributions=tables, distribution_weight=weight)
|
|
81
|
+
if group_by:
|
|
82
|
+
groups = defaultdict(set)
|
|
83
|
+
for (i,), p in reference.respondents.items():
|
|
84
|
+
groups[p[group_by]].add(i)
|
|
85
|
+
report["groups"] = {
|
|
86
|
+
g: evaluate(
|
|
87
|
+
Results([r for r in results.records if r["respondent_id"] in ids]),
|
|
88
|
+
reference.subset(ids),
|
|
89
|
+
weight=weight,
|
|
90
|
+
)
|
|
91
|
+
for g, ids in groups.items()
|
|
92
|
+
}
|
|
93
|
+
return report
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def compare(conditions, reference, **kwargs):
|
|
97
|
+
return {
|
|
98
|
+
name: evaluate(results, reference, **kwargs)
|
|
99
|
+
for name, results in conditions.items()
|
|
100
|
+
}
|