bead 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bead/__init__.py +11 -0
- bead/__main__.py +11 -0
- bead/active_learning/__init__.py +15 -0
- bead/active_learning/config.py +231 -0
- bead/active_learning/loop.py +566 -0
- bead/active_learning/models/__init__.py +24 -0
- bead/active_learning/models/base.py +852 -0
- bead/active_learning/models/binary.py +910 -0
- bead/active_learning/models/categorical.py +943 -0
- bead/active_learning/models/cloze.py +862 -0
- bead/active_learning/models/forced_choice.py +956 -0
- bead/active_learning/models/free_text.py +773 -0
- bead/active_learning/models/lora.py +365 -0
- bead/active_learning/models/magnitude.py +835 -0
- bead/active_learning/models/multi_select.py +795 -0
- bead/active_learning/models/ordinal_scale.py +811 -0
- bead/active_learning/models/peft_adapter.py +155 -0
- bead/active_learning/models/random_effects.py +639 -0
- bead/active_learning/selection.py +354 -0
- bead/active_learning/strategies.py +391 -0
- bead/active_learning/trainers/__init__.py +26 -0
- bead/active_learning/trainers/base.py +210 -0
- bead/active_learning/trainers/data_collator.py +172 -0
- bead/active_learning/trainers/dataset_utils.py +261 -0
- bead/active_learning/trainers/huggingface.py +304 -0
- bead/active_learning/trainers/lightning.py +324 -0
- bead/active_learning/trainers/metrics.py +424 -0
- bead/active_learning/trainers/mixed_effects.py +551 -0
- bead/active_learning/trainers/model_wrapper.py +509 -0
- bead/active_learning/trainers/registry.py +104 -0
- bead/adapters/__init__.py +11 -0
- bead/adapters/huggingface.py +61 -0
- bead/behavioral/__init__.py +116 -0
- bead/behavioral/analytics.py +646 -0
- bead/behavioral/extraction.py +343 -0
- bead/behavioral/merging.py +343 -0
- bead/cli/__init__.py +11 -0
- bead/cli/active_learning.py +513 -0
- bead/cli/active_learning_commands.py +779 -0
- bead/cli/completion.py +359 -0
- bead/cli/config.py +624 -0
- bead/cli/constraint_builders.py +286 -0
- bead/cli/deployment.py +859 -0
- bead/cli/deployment_trials.py +493 -0
- bead/cli/deployment_ui.py +332 -0
- bead/cli/display.py +378 -0
- bead/cli/items.py +960 -0
- bead/cli/items_factories.py +776 -0
- bead/cli/list_constraints.py +714 -0
- bead/cli/lists.py +490 -0
- bead/cli/main.py +430 -0
- bead/cli/models.py +877 -0
- bead/cli/resource_loaders.py +621 -0
- bead/cli/resources.py +1036 -0
- bead/cli/shell.py +356 -0
- bead/cli/simulate.py +840 -0
- bead/cli/templates.py +1158 -0
- bead/cli/training.py +1080 -0
- bead/cli/utils.py +614 -0
- bead/cli/workflow.py +1273 -0
- bead/config/__init__.py +68 -0
- bead/config/active_learning.py +1009 -0
- bead/config/config.py +192 -0
- bead/config/defaults.py +118 -0
- bead/config/deployment.py +217 -0
- bead/config/env.py +147 -0
- bead/config/item.py +45 -0
- bead/config/list.py +193 -0
- bead/config/loader.py +149 -0
- bead/config/logging.py +42 -0
- bead/config/model.py +49 -0
- bead/config/paths.py +46 -0
- bead/config/profiles.py +320 -0
- bead/config/resources.py +47 -0
- bead/config/serialization.py +210 -0
- bead/config/simulation.py +206 -0
- bead/config/template.py +238 -0
- bead/config/validation.py +267 -0
- bead/data/__init__.py +65 -0
- bead/data/base.py +87 -0
- bead/data/identifiers.py +97 -0
- bead/data/language_codes.py +61 -0
- bead/data/metadata.py +270 -0
- bead/data/range.py +123 -0
- bead/data/repository.py +358 -0
- bead/data/serialization.py +249 -0
- bead/data/timestamps.py +89 -0
- bead/data/validation.py +349 -0
- bead/data_collection/__init__.py +11 -0
- bead/data_collection/jatos.py +223 -0
- bead/data_collection/merger.py +154 -0
- bead/data_collection/prolific.py +198 -0
- bead/deployment/__init__.py +5 -0
- bead/deployment/distribution.py +402 -0
- bead/deployment/jatos/__init__.py +1 -0
- bead/deployment/jatos/api.py +200 -0
- bead/deployment/jatos/exporter.py +210 -0
- bead/deployment/jspsych/__init__.py +9 -0
- bead/deployment/jspsych/biome.json +44 -0
- bead/deployment/jspsych/config.py +411 -0
- bead/deployment/jspsych/generator.py +598 -0
- bead/deployment/jspsych/package.json +51 -0
- bead/deployment/jspsych/pnpm-lock.yaml +2141 -0
- bead/deployment/jspsych/randomizer.py +299 -0
- bead/deployment/jspsych/src/lib/list-distributor.test.ts +327 -0
- bead/deployment/jspsych/src/lib/list-distributor.ts +1282 -0
- bead/deployment/jspsych/src/lib/randomizer.test.ts +232 -0
- bead/deployment/jspsych/src/lib/randomizer.ts +367 -0
- bead/deployment/jspsych/src/plugins/cloze-dropdown.ts +252 -0
- bead/deployment/jspsych/src/plugins/forced-choice.ts +265 -0
- bead/deployment/jspsych/src/plugins/plugins.test.ts +141 -0
- bead/deployment/jspsych/src/plugins/rating.ts +248 -0
- bead/deployment/jspsych/src/slopit/index.ts +9 -0
- bead/deployment/jspsych/src/types/jatos.d.ts +256 -0
- bead/deployment/jspsych/src/types/jspsych.d.ts +228 -0
- bead/deployment/jspsych/templates/experiment.css +1 -0
- bead/deployment/jspsych/templates/experiment.js.template +289 -0
- bead/deployment/jspsych/templates/index.html +51 -0
- bead/deployment/jspsych/templates/randomizer.js +241 -0
- bead/deployment/jspsych/templates/randomizer.js.template +313 -0
- bead/deployment/jspsych/trials.py +723 -0
- bead/deployment/jspsych/tsconfig.json +23 -0
- bead/deployment/jspsych/tsup.config.ts +30 -0
- bead/deployment/jspsych/ui/__init__.py +1 -0
- bead/deployment/jspsych/ui/components.py +383 -0
- bead/deployment/jspsych/ui/styles.py +411 -0
- bead/dsl/__init__.py +80 -0
- bead/dsl/ast.py +168 -0
- bead/dsl/context.py +178 -0
- bead/dsl/errors.py +71 -0
- bead/dsl/evaluator.py +570 -0
- bead/dsl/grammar.lark +81 -0
- bead/dsl/parser.py +231 -0
- bead/dsl/stdlib.py +929 -0
- bead/evaluation/__init__.py +13 -0
- bead/evaluation/convergence.py +485 -0
- bead/evaluation/interannotator.py +398 -0
- bead/items/__init__.py +40 -0
- bead/items/adapters/__init__.py +70 -0
- bead/items/adapters/anthropic.py +224 -0
- bead/items/adapters/api_utils.py +167 -0
- bead/items/adapters/base.py +216 -0
- bead/items/adapters/google.py +259 -0
- bead/items/adapters/huggingface.py +1074 -0
- bead/items/adapters/openai.py +323 -0
- bead/items/adapters/registry.py +202 -0
- bead/items/adapters/sentence_transformers.py +224 -0
- bead/items/adapters/togetherai.py +309 -0
- bead/items/binary.py +515 -0
- bead/items/cache.py +558 -0
- bead/items/categorical.py +593 -0
- bead/items/cloze.py +757 -0
- bead/items/constructor.py +784 -0
- bead/items/forced_choice.py +413 -0
- bead/items/free_text.py +681 -0
- bead/items/generation.py +432 -0
- bead/items/item.py +396 -0
- bead/items/item_template.py +787 -0
- bead/items/magnitude.py +573 -0
- bead/items/multi_select.py +621 -0
- bead/items/ordinal_scale.py +569 -0
- bead/items/scoring.py +448 -0
- bead/items/validation.py +723 -0
- bead/lists/__init__.py +30 -0
- bead/lists/balancer.py +263 -0
- bead/lists/constraints.py +1067 -0
- bead/lists/experiment_list.py +286 -0
- bead/lists/list_collection.py +378 -0
- bead/lists/partitioner.py +1141 -0
- bead/lists/stratification.py +254 -0
- bead/participants/__init__.py +73 -0
- bead/participants/collection.py +699 -0
- bead/participants/merging.py +312 -0
- bead/participants/metadata_spec.py +491 -0
- bead/participants/models.py +276 -0
- bead/resources/__init__.py +29 -0
- bead/resources/adapters/__init__.py +19 -0
- bead/resources/adapters/base.py +104 -0
- bead/resources/adapters/cache.py +128 -0
- bead/resources/adapters/glazing.py +508 -0
- bead/resources/adapters/registry.py +117 -0
- bead/resources/adapters/unimorph.py +796 -0
- bead/resources/classification.py +856 -0
- bead/resources/constraint_builders.py +329 -0
- bead/resources/constraints.py +165 -0
- bead/resources/lexical_item.py +223 -0
- bead/resources/lexicon.py +744 -0
- bead/resources/loaders.py +209 -0
- bead/resources/template.py +441 -0
- bead/resources/template_collection.py +707 -0
- bead/resources/template_generation.py +349 -0
- bead/simulation/__init__.py +29 -0
- bead/simulation/annotators/__init__.py +15 -0
- bead/simulation/annotators/base.py +175 -0
- bead/simulation/annotators/distance_based.py +135 -0
- bead/simulation/annotators/lm_based.py +114 -0
- bead/simulation/annotators/oracle.py +182 -0
- bead/simulation/annotators/random.py +181 -0
- bead/simulation/dsl_extension/__init__.py +3 -0
- bead/simulation/noise_models/__init__.py +13 -0
- bead/simulation/noise_models/base.py +42 -0
- bead/simulation/noise_models/random_noise.py +82 -0
- bead/simulation/noise_models/systematic.py +132 -0
- bead/simulation/noise_models/temperature.py +86 -0
- bead/simulation/runner.py +144 -0
- bead/simulation/strategies/__init__.py +23 -0
- bead/simulation/strategies/base.py +123 -0
- bead/simulation/strategies/binary.py +103 -0
- bead/simulation/strategies/categorical.py +123 -0
- bead/simulation/strategies/cloze.py +224 -0
- bead/simulation/strategies/forced_choice.py +127 -0
- bead/simulation/strategies/free_text.py +105 -0
- bead/simulation/strategies/magnitude.py +116 -0
- bead/simulation/strategies/multi_select.py +129 -0
- bead/simulation/strategies/ordinal_scale.py +131 -0
- bead/templates/__init__.py +27 -0
- bead/templates/adapters/__init__.py +17 -0
- bead/templates/adapters/base.py +128 -0
- bead/templates/adapters/cache.py +178 -0
- bead/templates/adapters/huggingface.py +312 -0
- bead/templates/combinatorics.py +103 -0
- bead/templates/filler.py +605 -0
- bead/templates/renderers.py +177 -0
- bead/templates/resolver.py +178 -0
- bead/templates/strategies.py +1806 -0
- bead/templates/streaming.py +195 -0
- bead-0.1.0.dist-info/METADATA +212 -0
- bead-0.1.0.dist-info/RECORD +231 -0
- bead-0.1.0.dist-info/WHEEL +4 -0
- bead-0.1.0.dist-info/entry_points.txt +2 -0
- bead-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
"""Utilities for creating ordinal scale experimental items.
|
|
2
|
+
|
|
3
|
+
This module provides language-agnostic utilities for creating ordinal scale
|
|
4
|
+
items where participants rate a single stimulus on an ordered discrete scale
|
|
5
|
+
(e.g., 1-7 Likert scale, acceptability ratings).
|
|
6
|
+
|
|
7
|
+
Integration Points
|
|
8
|
+
------------------
|
|
9
|
+
- Active Learning: bead/active_learning/models/ordinal_scale.py
|
|
10
|
+
- Simulation: bead/simulation/strategies/ordinal_scale.py
|
|
11
|
+
- Deployment: bead/deployment/jspsych/ (slider or radio buttons)
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections import defaultdict
|
|
17
|
+
from collections.abc import Callable, Hashable
|
|
18
|
+
from itertools import product
|
|
19
|
+
from uuid import UUID, uuid4
|
|
20
|
+
|
|
21
|
+
from bead.items.item import Item, MetadataValue
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def create_ordinal_scale_item(
|
|
25
|
+
text: str,
|
|
26
|
+
scale_bounds: tuple[int, int] = (1, 7),
|
|
27
|
+
prompt: str | None = None,
|
|
28
|
+
scale_labels: dict[int, str] | None = None,
|
|
29
|
+
item_template_id: UUID | None = None,
|
|
30
|
+
metadata: dict[str, MetadataValue] | None = None,
|
|
31
|
+
) -> Item:
|
|
32
|
+
"""Create an ordinal scale rating item.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
text : str
|
|
37
|
+
The stimulus text to rate.
|
|
38
|
+
scale_bounds : tuple[int, int]
|
|
39
|
+
Tuple of (min, max) for the scale. Both must be integers with min < max.
|
|
40
|
+
Default: (1, 7) for a 7-point scale.
|
|
41
|
+
prompt : str | None
|
|
42
|
+
Optional question/prompt for the rating.
|
|
43
|
+
If None, uses "Rate this item:".
|
|
44
|
+
scale_labels : dict[int, str] | None
|
|
45
|
+
Optional labels for specific scale values (e.g., {1: "Bad", 7: "Good"}).
|
|
46
|
+
All keys must be within [scale_min, scale_max].
|
|
47
|
+
item_template_id : UUID | None
|
|
48
|
+
Template ID for the item. If None, generates new UUID.
|
|
49
|
+
metadata : dict[str, MetadataValue] | None
|
|
50
|
+
Additional metadata for item_metadata field.
|
|
51
|
+
|
|
52
|
+
Returns
|
|
53
|
+
-------
|
|
54
|
+
Item
|
|
55
|
+
Ordinal scale item with text and prompt in rendered_elements.
|
|
56
|
+
|
|
57
|
+
Raises
|
|
58
|
+
------
|
|
59
|
+
ValueError
|
|
60
|
+
If text is empty, if scale_bounds are invalid, or if scale_labels
|
|
61
|
+
contain values outside scale bounds.
|
|
62
|
+
|
|
63
|
+
Examples
|
|
64
|
+
--------
|
|
65
|
+
>>> item = create_ordinal_scale_item(
|
|
66
|
+
... text="The cat sat on the mat.",
|
|
67
|
+
... scale_bounds=(1, 7),
|
|
68
|
+
... prompt="How natural is this sentence?",
|
|
69
|
+
... metadata={"task": "acceptability"}
|
|
70
|
+
... )
|
|
71
|
+
>>> item.rendered_elements["text"]
|
|
72
|
+
'The cat sat on the mat.'
|
|
73
|
+
>>> item.item_metadata["scale_min"]
|
|
74
|
+
1
|
|
75
|
+
>>> item.item_metadata["scale_max"]
|
|
76
|
+
7
|
|
77
|
+
|
|
78
|
+
>>> # 5-point Likert with labels
|
|
79
|
+
>>> item = create_ordinal_scale_item(
|
|
80
|
+
... text="I enjoy linguistics.",
|
|
81
|
+
... scale_bounds=(1, 5),
|
|
82
|
+
... scale_labels={1: "Strongly Disagree", 5: "Strongly Agree"}
|
|
83
|
+
... )
|
|
84
|
+
>>> item.item_metadata["scale_labels"][1]
|
|
85
|
+
'Strongly Disagree'
|
|
86
|
+
"""
|
|
87
|
+
if not text or not text.strip():
|
|
88
|
+
raise ValueError("text cannot be empty")
|
|
89
|
+
|
|
90
|
+
scale_min, scale_max = scale_bounds
|
|
91
|
+
|
|
92
|
+
if scale_min >= scale_max:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"scale_min ({scale_min}) must be less than scale_max ({scale_max})"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Validate scale_labels if provided
|
|
98
|
+
if scale_labels:
|
|
99
|
+
for value in scale_labels.keys():
|
|
100
|
+
if not (scale_min <= value <= scale_max):
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"scale_labels key {value} is outside scale bounds "
|
|
103
|
+
f"[{scale_min}, {scale_max}]"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
if item_template_id is None:
|
|
107
|
+
item_template_id = uuid4()
|
|
108
|
+
|
|
109
|
+
if prompt is None:
|
|
110
|
+
prompt = "Rate this item:"
|
|
111
|
+
|
|
112
|
+
rendered_elements: dict[str, str] = {
|
|
113
|
+
"text": text,
|
|
114
|
+
"prompt": prompt,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
# Build item metadata
|
|
118
|
+
item_metadata: dict[str, MetadataValue] = {
|
|
119
|
+
"scale_min": scale_min,
|
|
120
|
+
"scale_max": scale_max,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if scale_labels:
|
|
124
|
+
item_metadata["scale_labels"] = {str(k): v for k, v in scale_labels.items()}
|
|
125
|
+
|
|
126
|
+
if metadata:
|
|
127
|
+
item_metadata.update(metadata)
|
|
128
|
+
|
|
129
|
+
return Item(
|
|
130
|
+
item_template_id=item_template_id,
|
|
131
|
+
rendered_elements=rendered_elements,
|
|
132
|
+
item_metadata=item_metadata,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def create_ordinal_scale_items_from_texts(
|
|
137
|
+
texts: list[str],
|
|
138
|
+
scale_bounds: tuple[int, int] = (1, 7),
|
|
139
|
+
prompt: str | None = None,
|
|
140
|
+
scale_labels: dict[int, str] | None = None,
|
|
141
|
+
*,
|
|
142
|
+
item_template_id: UUID | None = None,
|
|
143
|
+
metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None,
|
|
144
|
+
) -> list[Item]:
|
|
145
|
+
"""Create ordinal scale items from a list of texts.
|
|
146
|
+
|
|
147
|
+
Parameters
|
|
148
|
+
----------
|
|
149
|
+
texts : list[str]
|
|
150
|
+
List of stimulus texts.
|
|
151
|
+
scale_bounds : tuple[int, int]
|
|
152
|
+
Scale bounds (min, max) for all items.
|
|
153
|
+
prompt : str | None
|
|
154
|
+
The question/prompt for all items.
|
|
155
|
+
scale_labels : dict[int, str] | None
|
|
156
|
+
Optional scale labels for all items.
|
|
157
|
+
item_template_id : UUID | None
|
|
158
|
+
Template ID for all created items. If None, generates one per item.
|
|
159
|
+
metadata_fn : Callable[[str], dict[str, MetadataValue]] | None
|
|
160
|
+
Function to generate metadata from each text.
|
|
161
|
+
|
|
162
|
+
Returns
|
|
163
|
+
-------
|
|
164
|
+
list[Item]
|
|
165
|
+
Ordinal scale items for each text.
|
|
166
|
+
|
|
167
|
+
Examples
|
|
168
|
+
--------
|
|
169
|
+
>>> texts = ["She walks.", "She walk.", "They walk."]
|
|
170
|
+
>>> items = create_ordinal_scale_items_from_texts(
|
|
171
|
+
... texts,
|
|
172
|
+
... scale_bounds=(1, 5),
|
|
173
|
+
... prompt="How acceptable is this sentence?",
|
|
174
|
+
... metadata_fn=lambda t: {"text_length": len(t)}
|
|
175
|
+
... )
|
|
176
|
+
>>> len(items)
|
|
177
|
+
3
|
|
178
|
+
>>> items[0].item_metadata["scale_min"]
|
|
179
|
+
1
|
|
180
|
+
"""
|
|
181
|
+
ordinal_items: list[Item] = []
|
|
182
|
+
|
|
183
|
+
for text in texts:
|
|
184
|
+
item_metadata: dict[str, MetadataValue] = {}
|
|
185
|
+
if metadata_fn:
|
|
186
|
+
item_metadata = metadata_fn(text)
|
|
187
|
+
|
|
188
|
+
item = create_ordinal_scale_item(
|
|
189
|
+
text=text,
|
|
190
|
+
scale_bounds=scale_bounds,
|
|
191
|
+
prompt=prompt,
|
|
192
|
+
scale_labels=scale_labels,
|
|
193
|
+
item_template_id=item_template_id,
|
|
194
|
+
metadata=item_metadata,
|
|
195
|
+
)
|
|
196
|
+
ordinal_items.append(item)
|
|
197
|
+
|
|
198
|
+
return ordinal_items
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def create_ordinal_scale_items_from_groups(
|
|
202
|
+
items: list[Item],
|
|
203
|
+
group_by: Callable[[Item], Hashable],
|
|
204
|
+
scale_bounds: tuple[int, int] = (1, 7),
|
|
205
|
+
prompt: str | None = None,
|
|
206
|
+
scale_labels: dict[int, str] | None = None,
|
|
207
|
+
*,
|
|
208
|
+
extract_text: Callable[[Item], str] | None = None,
|
|
209
|
+
include_group_metadata: bool = True,
|
|
210
|
+
item_template_id: UUID | None = None,
|
|
211
|
+
) -> list[Item]:
|
|
212
|
+
"""Create ordinal scale items from grouped source items.
|
|
213
|
+
|
|
214
|
+
Groups items and creates one ordinal scale item per source item,
|
|
215
|
+
preserving group information in metadata.
|
|
216
|
+
|
|
217
|
+
Parameters
|
|
218
|
+
----------
|
|
219
|
+
items : list[Item]
|
|
220
|
+
Source items to process.
|
|
221
|
+
group_by : Callable[[Item], Hashable]
|
|
222
|
+
Function to extract grouping key from items.
|
|
223
|
+
scale_bounds : tuple[int, int]
|
|
224
|
+
Scale bounds (min, max) for all items.
|
|
225
|
+
prompt : str | None
|
|
226
|
+
The question/prompt for all items.
|
|
227
|
+
scale_labels : dict[int, str] | None
|
|
228
|
+
Optional scale labels for all items.
|
|
229
|
+
extract_text : Callable[[Item], str] | None
|
|
230
|
+
Function to extract text from item. If None, tries common keys.
|
|
231
|
+
include_group_metadata : bool
|
|
232
|
+
Whether to include group key in item metadata.
|
|
233
|
+
item_template_id : UUID | None
|
|
234
|
+
Template ID for all created items. If None, generates one per item.
|
|
235
|
+
|
|
236
|
+
Returns
|
|
237
|
+
-------
|
|
238
|
+
list[Item]
|
|
239
|
+
Ordinal scale items from source items.
|
|
240
|
+
|
|
241
|
+
Examples
|
|
242
|
+
--------
|
|
243
|
+
>>> source_items = [
|
|
244
|
+
... Item(
|
|
245
|
+
... uuid4(),
|
|
246
|
+
... rendered_elements={"text": "She walks."},
|
|
247
|
+
... item_metadata={"verb": "walk"}
|
|
248
|
+
... )
|
|
249
|
+
... ]
|
|
250
|
+
>>> ordinal_items = create_ordinal_scale_items_from_groups(
|
|
251
|
+
... source_items,
|
|
252
|
+
... group_by=lambda i: i.item_metadata["verb"],
|
|
253
|
+
... scale_bounds=(1, 7),
|
|
254
|
+
... prompt="Rate the acceptability:"
|
|
255
|
+
... )
|
|
256
|
+
>>> len(ordinal_items)
|
|
257
|
+
1
|
|
258
|
+
"""
|
|
259
|
+
# Group items
|
|
260
|
+
groups: dict[Hashable, list[Item]] = defaultdict(list)
|
|
261
|
+
for item in items:
|
|
262
|
+
group_key = group_by(item)
|
|
263
|
+
groups[group_key].append(item)
|
|
264
|
+
|
|
265
|
+
ordinal_items: list[Item] = []
|
|
266
|
+
|
|
267
|
+
for group_key, group_items in groups.items():
|
|
268
|
+
for item in group_items:
|
|
269
|
+
# Extract text
|
|
270
|
+
if extract_text:
|
|
271
|
+
text: str = extract_text(item)
|
|
272
|
+
else:
|
|
273
|
+
text = _extract_text_from_item(item)
|
|
274
|
+
|
|
275
|
+
# Build metadata
|
|
276
|
+
item_metadata: dict[str, MetadataValue] = {
|
|
277
|
+
"source_item_id": str(item.id),
|
|
278
|
+
}
|
|
279
|
+
if include_group_metadata:
|
|
280
|
+
item_metadata["group_key"] = str(group_key)
|
|
281
|
+
|
|
282
|
+
# Create ordinal scale item
|
|
283
|
+
ordinal_item = create_ordinal_scale_item(
|
|
284
|
+
text=text,
|
|
285
|
+
scale_bounds=scale_bounds,
|
|
286
|
+
prompt=prompt,
|
|
287
|
+
scale_labels=scale_labels,
|
|
288
|
+
item_template_id=item_template_id,
|
|
289
|
+
metadata=item_metadata,
|
|
290
|
+
)
|
|
291
|
+
ordinal_items.append(ordinal_item)
|
|
292
|
+
|
|
293
|
+
return ordinal_items
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def create_ordinal_scale_items_cross_product(
|
|
297
|
+
texts: list[str],
|
|
298
|
+
prompts: list[str],
|
|
299
|
+
scale_bounds: tuple[int, int] = (1, 7),
|
|
300
|
+
scale_labels: dict[int, str] | None = None,
|
|
301
|
+
*,
|
|
302
|
+
item_template_id: UUID | None = None,
|
|
303
|
+
metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None,
|
|
304
|
+
) -> list[Item]:
|
|
305
|
+
"""Create ordinal scale items from cross-product of texts and prompts.
|
|
306
|
+
|
|
307
|
+
Useful when you want to apply multiple prompts to each text.
|
|
308
|
+
|
|
309
|
+
Parameters
|
|
310
|
+
----------
|
|
311
|
+
texts : list[str]
|
|
312
|
+
List of stimulus texts.
|
|
313
|
+
prompts : list[str]
|
|
314
|
+
List of prompts to apply.
|
|
315
|
+
scale_bounds : tuple[int, int]
|
|
316
|
+
Scale bounds (min, max) for all items.
|
|
317
|
+
scale_labels : dict[int, str] | None
|
|
318
|
+
Optional scale labels for all items.
|
|
319
|
+
item_template_id : UUID | None
|
|
320
|
+
Template ID for all created items.
|
|
321
|
+
metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None
|
|
322
|
+
Function to generate metadata from (text, prompt).
|
|
323
|
+
|
|
324
|
+
Returns
|
|
325
|
+
-------
|
|
326
|
+
list[Item]
|
|
327
|
+
Ordinal scale items from cross-product.
|
|
328
|
+
|
|
329
|
+
Examples
|
|
330
|
+
--------
|
|
331
|
+
>>> texts = ["The cat sat.", "The dog ran."]
|
|
332
|
+
>>> prompts = ["How natural is this?", "How acceptable is this?"]
|
|
333
|
+
>>> items = create_ordinal_scale_items_cross_product(
|
|
334
|
+
... texts, prompts, scale_bounds=(1, 5)
|
|
335
|
+
... )
|
|
336
|
+
>>> len(items)
|
|
337
|
+
4
|
|
338
|
+
"""
|
|
339
|
+
ordinal_items: list[Item] = []
|
|
340
|
+
|
|
341
|
+
for text, prompt in product(texts, prompts):
|
|
342
|
+
item_metadata: dict[str, MetadataValue] = {}
|
|
343
|
+
if metadata_fn:
|
|
344
|
+
item_metadata = metadata_fn(text, prompt)
|
|
345
|
+
|
|
346
|
+
item = create_ordinal_scale_item(
|
|
347
|
+
text=text,
|
|
348
|
+
scale_bounds=scale_bounds,
|
|
349
|
+
prompt=prompt,
|
|
350
|
+
scale_labels=scale_labels,
|
|
351
|
+
item_template_id=item_template_id,
|
|
352
|
+
metadata=item_metadata,
|
|
353
|
+
)
|
|
354
|
+
ordinal_items.append(item)
|
|
355
|
+
|
|
356
|
+
return ordinal_items
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def create_filtered_ordinal_scale_items(
|
|
360
|
+
items: list[Item],
|
|
361
|
+
scale_bounds: tuple[int, int] = (1, 7),
|
|
362
|
+
prompt: str | None = None,
|
|
363
|
+
scale_labels: dict[int, str] | None = None,
|
|
364
|
+
*,
|
|
365
|
+
item_filter: Callable[[Item], bool] | None = None,
|
|
366
|
+
extract_text: Callable[[Item], str] | None = None,
|
|
367
|
+
item_template_id: UUID | None = None,
|
|
368
|
+
) -> list[Item]:
|
|
369
|
+
"""Create ordinal scale items with filtering.
|
|
370
|
+
|
|
371
|
+
Parameters
|
|
372
|
+
----------
|
|
373
|
+
items : list[Item]
|
|
374
|
+
Source items.
|
|
375
|
+
scale_bounds : tuple[int, int]
|
|
376
|
+
Scale bounds (min, max) for all items.
|
|
377
|
+
prompt : str | None
|
|
378
|
+
The question/prompt for all items.
|
|
379
|
+
scale_labels : dict[int, str] | None
|
|
380
|
+
Optional scale labels for all items.
|
|
381
|
+
item_filter : Callable[[Item], bool] | None
|
|
382
|
+
Filter individual items.
|
|
383
|
+
extract_text : Callable[[Item], str] | None
|
|
384
|
+
Text extraction function.
|
|
385
|
+
item_template_id : UUID | None
|
|
386
|
+
Template ID for created items.
|
|
387
|
+
|
|
388
|
+
Returns
|
|
389
|
+
-------
|
|
390
|
+
list[Item]
|
|
391
|
+
Filtered ordinal scale items.
|
|
392
|
+
|
|
393
|
+
Examples
|
|
394
|
+
--------
|
|
395
|
+
>>> ordinal_items = create_filtered_ordinal_scale_items(
|
|
396
|
+
... items,
|
|
397
|
+
... scale_bounds=(1, 7),
|
|
398
|
+
... prompt="Rate the acceptability:",
|
|
399
|
+
... item_filter=lambda i: i.item_metadata.get("valid", True)
|
|
400
|
+
... ) # doctest: +SKIP
|
|
401
|
+
"""
|
|
402
|
+
# Filter items
|
|
403
|
+
filtered_items = items
|
|
404
|
+
if item_filter:
|
|
405
|
+
filtered_items = [item for item in items if item_filter(item)]
|
|
406
|
+
|
|
407
|
+
ordinal_items: list[Item] = []
|
|
408
|
+
|
|
409
|
+
for item in filtered_items:
|
|
410
|
+
# Extract text
|
|
411
|
+
if extract_text:
|
|
412
|
+
text: str = extract_text(item)
|
|
413
|
+
else:
|
|
414
|
+
text = _extract_text_from_item(item)
|
|
415
|
+
|
|
416
|
+
# Create ordinal scale item
|
|
417
|
+
item_metadata: dict[str, MetadataValue] = {
|
|
418
|
+
"source_item_id": str(item.id),
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
ordinal_item = create_ordinal_scale_item(
|
|
422
|
+
text=text,
|
|
423
|
+
scale_bounds=scale_bounds,
|
|
424
|
+
prompt=prompt,
|
|
425
|
+
scale_labels=scale_labels,
|
|
426
|
+
item_template_id=item_template_id,
|
|
427
|
+
metadata=item_metadata,
|
|
428
|
+
)
|
|
429
|
+
ordinal_items.append(ordinal_item)
|
|
430
|
+
|
|
431
|
+
return ordinal_items
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def create_likert_5_item(
|
|
435
|
+
text: str,
|
|
436
|
+
prompt: str | None = None,
|
|
437
|
+
item_template_id: UUID | None = None,
|
|
438
|
+
metadata: dict[str, MetadataValue] | None = None,
|
|
439
|
+
) -> Item:
|
|
440
|
+
"""Create a 5-point Likert scale item.
|
|
441
|
+
|
|
442
|
+
Convenience function for standard 5-point Likert scale with
|
|
443
|
+
"Strongly Disagree" to "Strongly Agree" labels.
|
|
444
|
+
|
|
445
|
+
Parameters
|
|
446
|
+
----------
|
|
447
|
+
text : str
|
|
448
|
+
The stimulus text (statement) to rate.
|
|
449
|
+
prompt : str | None
|
|
450
|
+
Optional prompt. If None, uses "Rate your agreement:".
|
|
451
|
+
item_template_id : UUID | None
|
|
452
|
+
Template ID for the item. If None, generates new UUID.
|
|
453
|
+
metadata : dict[str, MetadataValue] | None
|
|
454
|
+
Additional metadata for item_metadata field.
|
|
455
|
+
|
|
456
|
+
Returns
|
|
457
|
+
-------
|
|
458
|
+
Item
|
|
459
|
+
5-point Likert scale item.
|
|
460
|
+
|
|
461
|
+
Examples
|
|
462
|
+
--------
|
|
463
|
+
>>> item = create_likert_5_item("I enjoy studying linguistics.")
|
|
464
|
+
>>> item.item_metadata["scale_min"]
|
|
465
|
+
1
|
|
466
|
+
>>> item.item_metadata["scale_max"]
|
|
467
|
+
5
|
|
468
|
+
"""
|
|
469
|
+
if prompt is None:
|
|
470
|
+
prompt = "Rate your agreement:"
|
|
471
|
+
|
|
472
|
+
return create_ordinal_scale_item(
|
|
473
|
+
text,
|
|
474
|
+
scale_bounds=(1, 5),
|
|
475
|
+
prompt=prompt,
|
|
476
|
+
scale_labels={
|
|
477
|
+
1: "Strongly Disagree",
|
|
478
|
+
2: "Disagree",
|
|
479
|
+
3: "Neutral",
|
|
480
|
+
4: "Agree",
|
|
481
|
+
5: "Strongly Agree",
|
|
482
|
+
},
|
|
483
|
+
item_template_id=item_template_id,
|
|
484
|
+
metadata=metadata,
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def create_likert_7_item(
|
|
489
|
+
text: str,
|
|
490
|
+
prompt: str | None = None,
|
|
491
|
+
item_template_id: UUID | None = None,
|
|
492
|
+
metadata: dict[str, MetadataValue] | None = None,
|
|
493
|
+
) -> Item:
|
|
494
|
+
"""Create a 7-point Likert scale item.
|
|
495
|
+
|
|
496
|
+
Convenience function for standard 7-point Likert scale with
|
|
497
|
+
"Strongly Disagree" to "Strongly Agree" labels.
|
|
498
|
+
|
|
499
|
+
Parameters
|
|
500
|
+
----------
|
|
501
|
+
text : str
|
|
502
|
+
The stimulus text (statement) to rate.
|
|
503
|
+
prompt : str | None
|
|
504
|
+
Optional prompt. If None, uses "Rate your agreement:".
|
|
505
|
+
item_template_id : UUID | None
|
|
506
|
+
Template ID for the item. If None, generates new UUID.
|
|
507
|
+
metadata : dict[str, MetadataValue] | None
|
|
508
|
+
Additional metadata for item_metadata field.
|
|
509
|
+
|
|
510
|
+
Returns
|
|
511
|
+
-------
|
|
512
|
+
Item
|
|
513
|
+
7-point Likert scale item.
|
|
514
|
+
|
|
515
|
+
Examples
|
|
516
|
+
--------
|
|
517
|
+
>>> item = create_likert_7_item("I enjoy studying linguistics.")
|
|
518
|
+
>>> item.item_metadata["scale_min"]
|
|
519
|
+
1
|
|
520
|
+
>>> item.item_metadata["scale_max"]
|
|
521
|
+
7
|
|
522
|
+
"""
|
|
523
|
+
if prompt is None:
|
|
524
|
+
prompt = "Rate your agreement:"
|
|
525
|
+
|
|
526
|
+
return create_ordinal_scale_item(
|
|
527
|
+
text,
|
|
528
|
+
scale_bounds=(1, 7),
|
|
529
|
+
prompt=prompt,
|
|
530
|
+
scale_labels={
|
|
531
|
+
1: "Strongly Disagree",
|
|
532
|
+
7: "Strongly Agree",
|
|
533
|
+
},
|
|
534
|
+
item_template_id=item_template_id,
|
|
535
|
+
metadata=metadata,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _extract_text_from_item(item: Item) -> str:
|
|
540
|
+
"""Extract text from item's rendered_elements.
|
|
541
|
+
|
|
542
|
+
Tries common keys: "text", "sentence", "content".
|
|
543
|
+
Raises error if no suitable text found.
|
|
544
|
+
|
|
545
|
+
Parameters
|
|
546
|
+
----------
|
|
547
|
+
item : Item
|
|
548
|
+
Item to extract text from.
|
|
549
|
+
|
|
550
|
+
Returns
|
|
551
|
+
-------
|
|
552
|
+
str
|
|
553
|
+
Extracted text.
|
|
554
|
+
|
|
555
|
+
Raises
|
|
556
|
+
------
|
|
557
|
+
ValueError
|
|
558
|
+
If no suitable text key found in rendered_elements.
|
|
559
|
+
"""
|
|
560
|
+
for key in ["text", "sentence", "content"]:
|
|
561
|
+
if key in item.rendered_elements:
|
|
562
|
+
return item.rendered_elements[key]
|
|
563
|
+
|
|
564
|
+
raise ValueError(
|
|
565
|
+
f"Cannot extract text from item {item.id}. "
|
|
566
|
+
f"Expected one of ['text', 'sentence', 'content'] in rendered_elements, "
|
|
567
|
+
f"but found keys: {list(item.rendered_elements.keys())}. "
|
|
568
|
+
f"Use the extract_text parameter to provide a custom extraction function."
|
|
569
|
+
)
|