alignsurvey 0.2.0a5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. alignsurvey-0.2.0a5/MANIFEST.in +1 -0
  2. alignsurvey-0.2.0a5/PKG-INFO +90 -0
  3. alignsurvey-0.2.0a5/PYPI_README.md +78 -0
  4. alignsurvey-0.2.0a5/README.md +78 -0
  5. alignsurvey-0.2.0a5/pyproject.toml +3 -0
  6. alignsurvey-0.2.0a5/setup.cfg +36 -0
  7. alignsurvey-0.2.0a5/src/alignsurvey/__init__.py +20 -0
  8. alignsurvey-0.2.0a5/src/alignsurvey/__main__.py +26 -0
  9. alignsurvey-0.2.0a5/src/alignsurvey/analysis.py +100 -0
  10. alignsurvey-0.2.0a5/src/alignsurvey/data.py +296 -0
  11. alignsurvey-0.2.0a5/src/alignsurvey/experiment.py +154 -0
  12. alignsurvey-0.2.0a5/src/alignsurvey/extensions.py +137 -0
  13. alignsurvey-0.2.0a5/src/alignsurvey/judge.py +99 -0
  14. alignsurvey-0.2.0a5/src/alignsurvey/models.py +87 -0
  15. alignsurvey-0.2.0a5/src/alignsurvey/paper.py +366 -0
  16. alignsurvey-0.2.0a5/src/alignsurvey/report.py +54 -0
  17. alignsurvey-0.2.0a5/src/alignsurvey/sample/assignments.csv +9 -0
  18. alignsurvey-0.2.0a5/src/alignsurvey/sample/external.csv +8 -0
  19. alignsurvey-0.2.0a5/src/alignsurvey/sample/questions.csv +3 -0
  20. alignsurvey-0.2.0a5/src/alignsurvey/sample/reference.csv +8 -0
  21. alignsurvey-0.2.0a5/src/alignsurvey/sample/respondents.csv +5 -0
  22. alignsurvey-0.2.0a5/src/alignsurvey/survey.py +108 -0
  23. alignsurvey-0.2.0a5/src/alignsurvey/tasks.py +168 -0
  24. alignsurvey-0.2.0a5/src/alignsurvey/training.py +148 -0
  25. alignsurvey-0.2.0a5/src/alignsurvey.egg-info/PKG-INFO +90 -0
  26. alignsurvey-0.2.0a5/src/alignsurvey.egg-info/SOURCES.txt +28 -0
  27. alignsurvey-0.2.0a5/src/alignsurvey.egg-info/dependency_links.txt +1 -0
  28. alignsurvey-0.2.0a5/src/alignsurvey.egg-info/requires.txt +11 -0
  29. alignsurvey-0.2.0a5/src/alignsurvey.egg-info/top_level.txt +1 -0
@@ -0,0 +1 @@
1
+ include PYPI_README.md
@@ -0,0 +1,90 @@
1
+ Metadata-Version: 2.1
2
+ Name: alignsurvey
3
+ Version: 0.2.0a5
4
+ Summary: Run, fine-tune, and evaluate language models for social surveys
5
+ Project-URL: Research paper, https://doi.org/10.1609/aaai.v40i45.41236
6
+ Project-URL: Research benchmark, https://github.com/PiLab-ZJU/AlignSurvey
7
+ Requires-Python: >=3.9
8
+ Description-Content-Type: text/markdown
9
+ Provides-Extra: local
10
+ Provides-Extra: analysis
11
+ Provides-Extra: paper
12
+
13
+ # AlignSurvey
14
+
15
+ **AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
16
+
17
+ ## Install and run your first survey evaluation
18
+
19
+ The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
20
+
21
+ ```bash
22
+ python -m pip install alignsurvey==0.2.0a5
23
+ python -m alignsurvey
24
+ ```
25
+
26
+ The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
27
+
28
+ ## Run a baseline and compare it with imported answers
29
+
30
+ This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
31
+
32
+ ```python
33
+ from alignsurvey import (
34
+ Experiment, Model, Results, SurveyDataset, compare,
35
+ example_directory, export_report,
36
+ )
37
+
38
+ root = example_directory()
39
+ survey = SurveyDataset.load(root)
40
+ baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
41
+ experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
42
+ print(experiment.preview()[0]["prompt"])
43
+
44
+ runs = {
45
+ "baseline": experiment.run(),
46
+ "external": Results.from_csv(root / "external.csv"),
47
+ }
48
+ report = compare(runs, survey)
49
+ print(report["external"]["paired_agreement"])
50
+ export_report(report, "survey-comparison.html")
51
+ ```
52
+
53
+ Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
54
+
55
+ ## Evaluate your own answers
56
+
57
+ Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
58
+
59
+ | File | One row represents | Required columns |
60
+ | --- | --- | --- |
61
+ | respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
62
+ | questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
63
+ | assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
64
+ | reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
65
+
66
+ For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
67
+
68
+ The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
69
+
70
+ ```bash
71
+ python -m alignsurvey --dataset my_survey --answers my_predictions.csv
72
+ ```
73
+
74
+ ## Optional features
75
+
76
+ | Install | Use it for |
77
+ | --- | --- |
78
+ | `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
79
+ | `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
80
+ | `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
81
+
82
+ For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
83
+
84
+ For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
85
+
86
+ ## Workshop materials and research
87
+
88
+ The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
89
+
90
+ For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
@@ -0,0 +1,78 @@
1
+ # AlignSurvey
2
+
3
+ **AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
4
+
5
+ ## Install and run your first survey evaluation
6
+
7
+ The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
8
+
9
+ ```bash
10
+ python -m pip install alignsurvey==0.2.0a5
11
+ python -m alignsurvey
12
+ ```
13
+
14
+ The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
15
+
16
+ ## Run a baseline and compare it with imported answers
17
+
18
+ This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
19
+
20
+ ```python
21
+ from alignsurvey import (
22
+ Experiment, Model, Results, SurveyDataset, compare,
23
+ example_directory, export_report,
24
+ )
25
+
26
+ root = example_directory()
27
+ survey = SurveyDataset.load(root)
28
+ baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
29
+ experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
30
+ print(experiment.preview()[0]["prompt"])
31
+
32
+ runs = {
33
+ "baseline": experiment.run(),
34
+ "external": Results.from_csv(root / "external.csv"),
35
+ }
36
+ report = compare(runs, survey)
37
+ print(report["external"]["paired_agreement"])
38
+ export_report(report, "survey-comparison.html")
39
+ ```
40
+
41
+ Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
42
+
43
+ ## Evaluate your own answers
44
+
45
+ Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
46
+
47
+ | File | One row represents | Required columns |
48
+ | --- | --- | --- |
49
+ | respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
50
+ | questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
51
+ | assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
52
+ | reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
53
+
54
+ For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
55
+
56
+ The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
57
+
58
+ ```bash
59
+ python -m alignsurvey --dataset my_survey --answers my_predictions.csv
60
+ ```
61
+
62
+ ## Optional features
63
+
64
+ | Install | Use it for |
65
+ | --- | --- |
66
+ | `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
67
+ | `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
68
+ | `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
69
+
70
+ For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
71
+
72
+ For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
73
+
74
+ ## Workshop materials and research
75
+
76
+ The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
77
+
78
+ For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
@@ -0,0 +1,78 @@
1
+ # AlignSurvey
2
+
3
+ **AlignSurvey is a Python toolkit for running and evaluating LLM-based social surveys.** Give a model a questionnaire and respondent profiles, or import answers you already collected from another system. The same evaluator can compare those answers with reference records at the respondent, question, and group levels.
4
+
5
+ ## Install and run your first survey evaluation
6
+
7
+ The core package supports Python 3.9+ and uses the Python standard library. The example runs offline with fictional records and requires no model download or API key.
8
+
9
+ ```bash
10
+ python -m pip install alignsurvey==0.2.0a5
11
+ python -m alignsurvey
12
+ ```
13
+
14
+ The command evaluates the answers bundled with the package. Four respondents receive seven applicable questions in total. Six responses have valid option codes, and five of those six match the reference. The **valid response rate is 6/7** and the **paired agreement is 5/6**. The skipped question is outside both denominators.
15
+
16
+ ## Run a baseline and compare it with imported answers
17
+
18
+ This example uses a simple model that always chooses option 1. It shows exactly which respondent field the model can see, scores the baseline alongside an external run, and writes a report you can open in a browser.
19
+
20
+ ```python
21
+ from alignsurvey import (
22
+ Experiment, Model, Results, SurveyDataset, compare,
23
+ example_directory, export_report,
24
+ )
25
+
26
+ root = example_directory()
27
+ survey = SurveyDataset.load(root)
28
+ baseline = Model("always-option-1", lambda prompt: '{"answer": "1"}')
29
+ experiment = Experiment.from_dataset(survey, baseline, fields=["employment"])
30
+ print(experiment.preview()[0]["prompt"])
31
+
32
+ runs = {
33
+ "baseline": experiment.run(),
34
+ "external": Results.from_csv(root / "external.csv"),
35
+ }
36
+ report = compare(runs, survey)
37
+ print(report["external"]["paired_agreement"])
38
+ export_report(report, "survey-comparison.html")
39
+ ```
40
+
41
+ Replace the callable with an `APIModel` for a Chat Completions-compatible endpoint or a `LocalModel` for a Hugging Face causal language model. An external API receives the profile fields and survey materials included in the prompt. Keep reference answers in the evaluation data.
42
+
43
+ ## Evaluate your own answers
44
+
45
+ Prepare a survey directory with the following UTF-8 CSV files. Keep respondent IDs, question IDs, and option codes consistent across files.
46
+
47
+ | File | One row represents | Required columns |
48
+ | --- | --- | --- |
49
+ | respondents.csv | One respondent | respondent_id, plus any profile fields you intend to use |
50
+ | questions.csv | One question | question_id, section_path, question_text, question_type, option_codes, option_labels |
51
+ | assignments.csv | One respondent–question assignment | respondent_id, question_id, status (asked or skipped) |
52
+ | reference.csv | One valid reference answer | respondent_id, question_id, answer_code |
53
+
54
+ For closed questions, separate option codes and their labels with a vertical bar in the same order, for example `1|2` and `Satisfied|Dissatisfied`. Omit reference.csv when you only want to check response coverage and validity.
55
+
56
+ The external answer file uses `respondent_id,question_id,status,answer_code`. Use `valid` with the original code for a valid answer. For a parsing failure use `parse_error` and leave the code empty. An optional `raw_output` column preserves the model's original text.
57
+
58
+ ```bash
59
+ python -m alignsurvey --dataset my_survey --answers my_predictions.csv
60
+ ```
61
+
62
+ ## Optional features
63
+
64
+ | Install | Use it for |
65
+ | --- | --- |
66
+ | `python -m pip install 'alignsurvey[local]==0.2.0a5'` | Local inference and LoRA training with PyTorch and Transformers |
67
+ | `python -m pip install 'alignsurvey[analysis]==0.2.0a5'` | Exporting results as a pandas DataFrame |
68
+ | `python -m pip install 'alignsurvey[paper]==0.2.0a5'` | Tokenization and sentence embeddings for benchmark reasoning metrics |
69
+
70
+ For a fine-tuning comparison, split the dataset by **respondent** before calling `Trainer.samples()`. Train an adapter with `Trainer.fit()`, then run the base and adapted models on the same held-out respondents and questions with identical prompts and decoding settings.
71
+
72
+ For the four AlignSurvey benchmark tasks, `Benchmark.from_published()` reads the published example format. Use `Benchmark.paper_metrics()` for social roles, attitude stances and questionnaire answers. Use `InterviewJudge` for interview naturalness, style match and consistency. The interview judge needs a configured evaluator model; the stance reasoning metrics need a sentence encoder.
73
+
74
+ ## Workshop materials and research
75
+
76
+ The wheel includes the Python library and the small CSV example used above. Workshop participants receive a separate student ZIP with executed notebooks, Qwen2.5-0.5B-Instruct LoRA adapters, recorded comparisons, and a Chinese user manual. After extracting the ZIP, start with **课前安装说明.txt**, then open **examples/workshop.ipynb**.
77
+
78
+ For the task definitions, evaluation methods, and upstream research datasets, see the [AlignSurvey paper](https://doi.org/10.1609/aaai.v40i45.41236) and the [research benchmark repository](https://github.com/PiLab-ZJU/AlignSurvey).
@@ -0,0 +1,3 @@
1
+ [build-system]
2
+ requires = ["setuptools>=40.8", "wheel"]
3
+ build-backend = "setuptools.build_meta"
@@ -0,0 +1,36 @@
1
+ [metadata]
2
+ name = alignsurvey
3
+ version = 0.2.0a5
4
+ description = Run, fine-tune, and evaluate language models for social surveys
5
+ long_description = file: PYPI_README.md
6
+ long_description_content_type = text/markdown
7
+ project_urls =
8
+ Research paper = https://doi.org/10.1609/aaai.v40i45.41236
9
+ Research benchmark = https://github.com/PiLab-ZJU/AlignSurvey
10
+
11
+ [options]
12
+ package_dir =
13
+ = src
14
+ packages = find:
15
+ python_requires = >=3.9
16
+
17
+ [options.packages.find]
18
+ where = src
19
+
20
+ [options.package_data]
21
+ alignsurvey = sample/*.csv
22
+
23
+ [options.extras_require]
24
+ local =
25
+ torch>=2.2
26
+ transformers>=4.45,<6
27
+ analysis =
28
+ pandas
29
+ paper =
30
+ jieba>=0.42
31
+ sentence-transformers>=3
32
+
33
+ [egg_info]
34
+ tag_build =
35
+ tag_date = 0
36
+
@@ -0,0 +1,20 @@
1
+ from .data import SurveyDataset, Results, example_directory
2
+ from .analysis import evaluate, compare, distribution_distance
3
+ from .survey import Question, Survey, Respondent, Scenario
4
+ from .models import Model, APIModel, LocalModel
5
+ from .experiment import Experiment
6
+ from .training import Trainer
7
+ from .tasks import Benchmark, predict_attribute, interview, predict_stance
8
+ from .extensions import (
9
+ Retriever,
10
+ select_examples,
11
+ summarize_interview,
12
+ conduct_interview,
13
+ generate_survey,
14
+ synthetic_population,
15
+ cross_conditions,
16
+ correlation,
17
+ )
18
+ from .report import export_report
19
+
20
+ from .judge import InterviewJudge
@@ -0,0 +1,26 @@
1
+ import argparse
2
+ import json
3
+ from . import Results, SurveyDataset, evaluate, example_directory
4
+
5
+
6
+ def main():
7
+ parser = argparse.ArgumentParser(description="Evaluate one survey run")
8
+ parser.add_argument(
9
+ "--dataset",
10
+ default=example_directory(),
11
+ help="Directory with four survey CSV files",
12
+ )
13
+ parser.add_argument(
14
+ "--answers",
15
+ help="External answer CSV (default: external.csv in dataset directory)",
16
+ )
17
+ args = parser.parse_args()
18
+ from pathlib import Path
19
+
20
+ data = SurveyDataset.load(args.dataset)
21
+ answers = Results.from_csv(args.answers or Path(args.dataset) / "external.csv")
22
+ print(json.dumps(evaluate(answers, reference=data), ensure_ascii=False, indent=2))
23
+
24
+
25
+ if __name__ == "__main__":
26
+ main()
@@ -0,0 +1,100 @@
1
+ from collections import Counter, defaultdict
2
+ from math import isfinite
3
+ from .data import KEY, Results, evaluate as basic_evaluation
4
+
5
+
6
+ def distribution_distance(a, b, ordered=False):
7
+ """Total variation (nominal) or unit-step Wasserstein (ordered)."""
8
+ if set(a) != set(b):
9
+ raise ValueError("Align option codes before comparison")
10
+ if any(not isfinite(v) or v < 0 for counts in (a, b) for v in counts.values()):
11
+ raise ValueError("Expected nonnegative finite masses")
12
+ x, y = sum(a.values()), sum(b.values())
13
+ if not x or not y:
14
+ return None
15
+ delta = [a[k] / x - b[k] / y for k in a]
16
+ if not ordered:
17
+ return sum(map(abs, delta)) / 2
18
+ from itertools import accumulate
19
+
20
+ return sum(abs(d) for d in accumulate(delta[:-1]))
21
+
22
+
23
+ def evaluate(results, reference, *, group_by=None, weight=None):
24
+ runs = {(r.get("experiment_id", ""), r.get("repeat", 0)) for r in results.records}
25
+ if len(runs) > 1:
26
+ return {
27
+ "runs": [
28
+ {
29
+ "experiment_id": e,
30
+ "repeat": n,
31
+ "report": evaluate(
32
+ Results(
33
+ [
34
+ r
35
+ for r in results.records
36
+ if (r.get("experiment_id", ""), r.get("repeat", 0))
37
+ == (e, n)
38
+ ]
39
+ ),
40
+ reference,
41
+ group_by=group_by,
42
+ weight=weight,
43
+ ),
44
+ }
45
+ for e, n in sorted(runs)
46
+ ]
47
+ }
48
+ report = basic_evaluation(results, reference)
49
+ counts = Counter(tuple(r[f] for f in KEY) for r in results.records)
50
+ pred = {tuple(r[f] for f in KEY): r for r in results.records}
51
+ tables = {}
52
+ for (i, q), assignment in reference.assignments.items():
53
+ if assignment["status"] != "asked":
54
+ continue
55
+ person = reference.respondents[(i,)]
56
+ w = float(person[weight]) if weight else 1.0
57
+ if not isfinite(w) or w < 0:
58
+ raise ValueError("Invalid respondent weight")
59
+ if reference.questions[(q,)]["question_type"] not in {"single", "ordinal"}:
60
+ continue
61
+ codes = reference.questions[(q,)]["option_codes"].split("|")
62
+ t = tables.setdefault(
63
+ q, {k: dict.fromkeys(codes, 0.0) for k in ("human", "model")}
64
+ )
65
+ key = (i, q)
66
+ if key in reference.reference:
67
+ t["human"][reference.reference[key]["answer_code"]] += w
68
+ r = pred.get(key)
69
+ if (
70
+ r
71
+ and counts[key] == 1
72
+ and r["status"] == "valid"
73
+ and r["answer_code"] in codes
74
+ ):
75
+ t["model"][r["answer_code"]] += w
76
+ for q, t in tables.items():
77
+ ordered = reference.questions[(q,)]["question_type"] == "ordinal"
78
+ t["distance"] = distribution_distance(t["human"], t["model"], ordered)
79
+ t["metric"] = "wasserstein_unit_step" if ordered else "total_variation"
80
+ report.update(distributions=tables, distribution_weight=weight)
81
+ if group_by:
82
+ groups = defaultdict(set)
83
+ for (i,), p in reference.respondents.items():
84
+ groups[p[group_by]].add(i)
85
+ report["groups"] = {
86
+ g: evaluate(
87
+ Results([r for r in results.records if r["respondent_id"] in ids]),
88
+ reference.subset(ids),
89
+ weight=weight,
90
+ )
91
+ for g, ids in groups.items()
92
+ }
93
+ return report
94
+
95
+
96
+ def compare(conditions, reference, **kwargs):
97
+ return {
98
+ name: evaluate(results, reference, **kwargs)
99
+ for name, results in conditions.items()
100
+ }