radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
radlab_data/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
__version__ = version("radlab-data")
|
|
6
|
+
except PackageNotFoundError: # running from a source checkout
|
|
7
|
+
__version__ = (Path(__file__).resolve().parent.parent / ".version").read_text(
|
|
8
|
+
encoding="utf-8"
|
|
9
|
+
).strip()
|
|
File without changes
|
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from datasets import Dataset
|
|
8
|
+
from tqdm import tqdm
|
|
9
|
+
from transformers.tokenization_utils import PreTrainedTokenizer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DatasetBaseClass(ABC, Dataset):
|
|
13
|
+
BATCHED_TOKENIZATION = True
|
|
14
|
+
MAPPING_ENABLED = True
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
data,
|
|
19
|
+
metadata,
|
|
20
|
+
cached_dataset_path,
|
|
21
|
+
tokenizer,
|
|
22
|
+
tokenizer_kwargs=None,
|
|
23
|
+
class_label_field="label",
|
|
24
|
+
split_train_valid_test=None,
|
|
25
|
+
pre_shuffle=False,
|
|
26
|
+
) -> None:
|
|
27
|
+
self._random_seed = 1337
|
|
28
|
+
if tokenizer_kwargs is None:
|
|
29
|
+
tokenizer_kwargs = {}
|
|
30
|
+
self.tokenizer_kwargs = tokenizer_kwargs
|
|
31
|
+
|
|
32
|
+
self._raw_data = data
|
|
33
|
+
self._metadata = metadata
|
|
34
|
+
self.class_label_field = class_label_field
|
|
35
|
+
self._label_data = self._metadata.get("labels", None)
|
|
36
|
+
self._cached_dataset_path = cached_dataset_path
|
|
37
|
+
self._tokenizer: PreTrainedTokenizer = tokenizer
|
|
38
|
+
if (
|
|
39
|
+
"special_token_info" in self._metadata
|
|
40
|
+
and self._metadata["special_token_info"] is not None
|
|
41
|
+
):
|
|
42
|
+
self._tokenizer.add_special_tokens(
|
|
43
|
+
{
|
|
44
|
+
"additional_special_tokens": self._metadata["special_token_info"][
|
|
45
|
+
"special_tokens"
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
if self._label_data is not None:
|
|
51
|
+
self.labels = self._label_data["names"]
|
|
52
|
+
self.id2label = {int(k): v for k, v in self._label_data["id2label"].items()}
|
|
53
|
+
self.label2id = self._label_data["label2id"]
|
|
54
|
+
self.num_labels = self._label_data["num_labels"]
|
|
55
|
+
|
|
56
|
+
self._train_dataset = None
|
|
57
|
+
self._valid_dataset = None
|
|
58
|
+
self._test_dataset = None
|
|
59
|
+
self._split_train_valid_test = split_train_valid_test
|
|
60
|
+
self._pre_shuffle = pre_shuffle
|
|
61
|
+
|
|
62
|
+
self._load()
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def train_dataset(self):
|
|
66
|
+
return self._train_dataset
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def valid_dataset(self):
|
|
70
|
+
return self._valid_dataset
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def test_dataset(self):
|
|
74
|
+
return self._test_dataset
|
|
75
|
+
|
|
76
|
+
def _load(self):
|
|
77
|
+
if os.path.exists(self._cached_dataset_path):
|
|
78
|
+
hf_dataset = Dataset.from_file(self._cached_dataset_path)
|
|
79
|
+
else:
|
|
80
|
+
for idx, elem in tqdm(
|
|
81
|
+
enumerate(self._raw_data), desc="Preprocessing raw data"
|
|
82
|
+
):
|
|
83
|
+
self._preprocess_raw_data(idx, elem)
|
|
84
|
+
|
|
85
|
+
self._raw_data = self._apply_custom_preprocess_function(self._raw_data)
|
|
86
|
+
|
|
87
|
+
hf_dataset = self._transform_to_hf_dataset(self._raw_data)
|
|
88
|
+
|
|
89
|
+
if self.MAPPING_ENABLED:
|
|
90
|
+
hf_dataset = hf_dataset.map(
|
|
91
|
+
self._mapping_function,
|
|
92
|
+
batched=self.BATCHED_TOKENIZATION,
|
|
93
|
+
cache_file_name=self._cached_dataset_path,
|
|
94
|
+
remove_columns=hf_dataset.column_names,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
if self._pre_shuffle:
|
|
98
|
+
hf_dataset = hf_dataset.shuffle(seed=self._random_seed)
|
|
99
|
+
|
|
100
|
+
self._train_dataset = hf_dataset
|
|
101
|
+
self._valid_dataset = None
|
|
102
|
+
self._test_dataset = None
|
|
103
|
+
|
|
104
|
+
if self._split_train_valid_test is not None:
|
|
105
|
+
train_split = self._split_train_valid_test[0]
|
|
106
|
+
valid_split = self._split_train_valid_test[1]
|
|
107
|
+
train_split_idx = int(len(hf_dataset) * train_split)
|
|
108
|
+
valid_split_idx = int(len(hf_dataset) * (train_split + valid_split))
|
|
109
|
+
self._train_dataset = Dataset.from_dict(hf_dataset[:train_split_idx])
|
|
110
|
+
self._valid_dataset = Dataset.from_dict(
|
|
111
|
+
hf_dataset[train_split_idx:valid_split_idx]
|
|
112
|
+
)
|
|
113
|
+
if len(self._split_train_valid_test) == 3:
|
|
114
|
+
self._test_dataset = Dataset.from_dict(hf_dataset[valid_split_idx:])
|
|
115
|
+
|
|
116
|
+
def _transform_to_hf_dataset(self, dataset) -> Dataset:
|
|
117
|
+
texts = []
|
|
118
|
+
labels = []
|
|
119
|
+
for item in dataset:
|
|
120
|
+
texts.append(item["text"])
|
|
121
|
+
cl_label = item["label"]
|
|
122
|
+
labels.append(cl_label)
|
|
123
|
+
return Dataset.from_dict({"text": texts, "label": labels})
|
|
124
|
+
|
|
125
|
+
@abstractmethod
|
|
126
|
+
def _mapping_function(self, batch):
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
@abstractmethod
|
|
130
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
131
|
+
"""
|
|
132
|
+
Raw data preprocessing, element by element.
|
|
133
|
+
Prepares data for further usage and conversion to datasets.Dataset object.
|
|
134
|
+
"""
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
def _apply_custom_preprocess_function(self, raw_data):
|
|
138
|
+
"""
|
|
139
|
+
It is highly recommended to use _preprocess_raw_data method instead (if possible)!
|
|
140
|
+
|
|
141
|
+
This method allows to apply a custom function to a dataset after element-wise preprocessing,
|
|
142
|
+
but prior to transforming the dataset into a huggingface dataset.
|
|
143
|
+
"""
|
|
144
|
+
return raw_data
|
|
145
|
+
|
|
146
|
+
NOT_IMPLEMENTED_MESSAGE = "Use dataset.train_dataset or dataset.test_dataset."
|
|
147
|
+
|
|
148
|
+
def __len__(self):
|
|
149
|
+
raise NotImplementedError(self.NOT_IMPLEMENTED_MESSAGE)
|
|
150
|
+
|
|
151
|
+
def __getitem__(self, index):
|
|
152
|
+
raise NotImplementedError(self.NOT_IMPLEMENTED_MESSAGE)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class SequenceLabellingDataset(DatasetBaseClass):
|
|
156
|
+
def _preprocess_raw_data(self, idx, elem) -> None:
|
|
157
|
+
self._raw_data[idx] = {
|
|
158
|
+
"text": elem["text"],
|
|
159
|
+
self.class_label_field: elem["metadata"][self.class_label_field],
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
def _mapping_function(self, batch):
|
|
163
|
+
tokenized_texts = self._tokenizer(batch["text"], **self.tokenizer_kwargs)
|
|
164
|
+
labels = batch[self.class_label_field]
|
|
165
|
+
|
|
166
|
+
# generate ner_tags lists
|
|
167
|
+
all_ner_tags = []
|
|
168
|
+
outside_label = self.label2id["O"]
|
|
169
|
+
for text_idx, (t_text, label_list) in enumerate(
|
|
170
|
+
zip(tokenized_texts["input_ids"], labels)
|
|
171
|
+
):
|
|
172
|
+
ner_tags = np.full(len(t_text), outside_label)
|
|
173
|
+
# # print(100 * "-")
|
|
174
|
+
# # print("text_id", text_idx)
|
|
175
|
+
# # print("label_list", label_list)
|
|
176
|
+
# # print("t_text", t_text)
|
|
177
|
+
# # print(batch["text"][text_idx])
|
|
178
|
+
# for begin, end, b_label, i_label in label_list:
|
|
179
|
+
# # subtract 1 because end is the index of a character succeeding the labelled sequence
|
|
180
|
+
# end -= 1
|
|
181
|
+
#
|
|
182
|
+
# begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
|
|
183
|
+
# end_token_idx = tokenized_texts.char_to_token(text_idx, end)
|
|
184
|
+
# # print("begin=", begin, "end=", end, "b_label=", b_label, "i_label=", i_label)
|
|
185
|
+
# # print("begin_token_idx=", begin_token_idx, "end_token_idx=", end_token_idx)
|
|
186
|
+
# # in case the tokenizer truncates text
|
|
187
|
+
# # if begin_token_idx is None or end_token_idx is None:
|
|
188
|
+
# # continue
|
|
189
|
+
# if begin_token_idx is None:
|
|
190
|
+
# continue
|
|
191
|
+
# if end_token_idx is None:
|
|
192
|
+
# end_token_idx = len(batch["text"][text_idx])
|
|
193
|
+
# if begin_token_idx < end_token_idx:
|
|
194
|
+
# ner_tags[begin_token_idx + 1 : end_token_idx + 1] = i_label
|
|
195
|
+
#
|
|
196
|
+
# # print("begin=", begin, "end=", end, "b_label=", b_label, "i_label=", i_label)
|
|
197
|
+
# # print(
|
|
198
|
+
# # "begin_token_idx=", begin_token_idx, type(begin_token_idx),
|
|
199
|
+
# # "end_token_idx=", end_token_idx
|
|
200
|
+
# # )
|
|
201
|
+
# ner_tags[begin_token_idx] = b_label
|
|
202
|
+
# all_ner_tags.append(list(ner_tags))
|
|
203
|
+
# # print(ner_tags)
|
|
204
|
+
# tokenized_texts["labels"] = all_ner_tags
|
|
205
|
+
# # print("all_ner_tags", all_ner_tags)
|
|
206
|
+
# # print(100 * "#")
|
|
207
|
+
for begin, end, b_label, i_label in label_list:
|
|
208
|
+
end -= 1
|
|
209
|
+
begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
|
|
210
|
+
end_token_idx = tokenized_texts.char_to_token(text_idx, end)
|
|
211
|
+
|
|
212
|
+
# in case a whitespace character is selected at begin or end idx
|
|
213
|
+
while begin_token_idx is None and begin <= end:
|
|
214
|
+
begin += 1
|
|
215
|
+
begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
|
|
216
|
+
|
|
217
|
+
while end_token_idx is None and begin <= end:
|
|
218
|
+
end -= 1
|
|
219
|
+
end_token_idx = tokenized_texts.char_to_token(text_idx, end)
|
|
220
|
+
|
|
221
|
+
if begin_token_idx is None or end_token_idx is None or begin > end:
|
|
222
|
+
continue
|
|
223
|
+
|
|
224
|
+
if begin_token_idx < end_token_idx:
|
|
225
|
+
ner_tags[begin_token_idx + 1 : end_token_idx + 1] = i_label
|
|
226
|
+
ner_tags[begin_token_idx] = b_label
|
|
227
|
+
|
|
228
|
+
all_ner_tags.append(list(ner_tags))
|
|
229
|
+
|
|
230
|
+
tokenized_texts["labels"] = all_ner_tags
|
|
231
|
+
return tokenized_texts
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class TextClassificationDataset(DatasetBaseClass):
|
|
235
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
236
|
+
self._raw_data[idx] = {
|
|
237
|
+
"text": elem["text"],
|
|
238
|
+
self.class_label_field: elem["metadata"][self.class_label_field][0],
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
def _mapping_function(self, batch):
|
|
242
|
+
tokenized_batch = self._tokenizer(batch["text"], **self.tokenizer_kwargs)
|
|
243
|
+
tokenized_batch["labels"] = batch[self.class_label_field]
|
|
244
|
+
return tokenized_batch
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class SequenceToSequenceDataset(DatasetBaseClass):
|
|
248
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
249
|
+
left_ctx = None
|
|
250
|
+
right_ctx = None
|
|
251
|
+
if "left_ctx" in elem["metadata"]:
|
|
252
|
+
left_ctx = elem["metadata"]["left_ctx"]
|
|
253
|
+
if "right_ctx" in elem["metadata"]:
|
|
254
|
+
right_ctx = elem["metadata"]["right_ctx"]
|
|
255
|
+
|
|
256
|
+
text = f"{left_ctx + ' ' if left_ctx is not None else ''}{elem['text']}{' ' + right_ctx if right_ctx is not None else ''}"
|
|
257
|
+
self._raw_data[idx] = {
|
|
258
|
+
"text": text,
|
|
259
|
+
self.class_label_field: elem[self.class_label_field][0],
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
def _mapping_function(self, batch):
|
|
263
|
+
tokenized_batch = self._tokenizer(
|
|
264
|
+
text=batch["text"],
|
|
265
|
+
text_target=batch[self.class_label_field],
|
|
266
|
+
is_split_into_words=False,
|
|
267
|
+
**self.tokenizer_kwargs,
|
|
268
|
+
)
|
|
269
|
+
return tokenized_batch
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class SemanticSimilarityDataset(DatasetBaseClass):
|
|
273
|
+
# MAPPING_ENABLED = False
|
|
274
|
+
|
|
275
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
276
|
+
self._raw_data[idx] = {
|
|
277
|
+
"text": elem["text"],
|
|
278
|
+
"label": elem["labels"],
|
|
279
|
+
"score": elem["metadata"]["score"],
|
|
280
|
+
"relation": elem["metadata"]["relation"],
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
RANDOM_SEED = 1337
|
|
284
|
+
|
|
285
|
+
def _apply_custom_preprocess_function(self, raw_data):
|
|
286
|
+
rng = np.random.default_rng(self.RANDOM_SEED)
|
|
287
|
+
positive_raw_data = [x for x in raw_data if x["relation"] == "positive"]
|
|
288
|
+
negative_unlabelled_raw_data = [
|
|
289
|
+
x for x in raw_data if x["relation"] == "negative" and not x["label"]
|
|
290
|
+
]
|
|
291
|
+
# TODO move these variables outside
|
|
292
|
+
negative_to_positive_proportion = 2
|
|
293
|
+
first_method_weight = 0.8
|
|
294
|
+
second_method_weight = 1 - first_method_weight
|
|
295
|
+
score_noise_std = 0.001
|
|
296
|
+
|
|
297
|
+
# adjust if there's not enough data in the unlabelled negative subset
|
|
298
|
+
positive_raw_data_len = len(positive_raw_data)
|
|
299
|
+
max_possible_second_method_weight = min(
|
|
300
|
+
(len(negative_unlabelled_raw_data) / positive_raw_data_len), 1
|
|
301
|
+
)
|
|
302
|
+
second_method_weight = min(
|
|
303
|
+
max_possible_second_method_weight, second_method_weight
|
|
304
|
+
)
|
|
305
|
+
first_method_weight = 1 - second_method_weight
|
|
306
|
+
|
|
307
|
+
# 1. Generation of negative examples by mixing positives.
|
|
308
|
+
negative_count = int(
|
|
309
|
+
positive_raw_data_len
|
|
310
|
+
* negative_to_positive_proportion
|
|
311
|
+
* first_method_weight
|
|
312
|
+
)
|
|
313
|
+
pair_indexes = rng.choice(
|
|
314
|
+
negative_count, negative_count, replace=False
|
|
315
|
+
) # we're doing it this way to maximize randomness, overflowing indices will sadly be repeats, but in case of no overflow, there will be no repetitions
|
|
316
|
+
pair_indexes = pair_indexes % len(
|
|
317
|
+
positive_raw_data
|
|
318
|
+
) # ensure indices are in range of len(positive_raw_data)
|
|
319
|
+
|
|
320
|
+
for i in tqdm(
|
|
321
|
+
range(negative_count),
|
|
322
|
+
desc="Generating negative examples by mixing positives.",
|
|
323
|
+
):
|
|
324
|
+
idx = i % positive_raw_data_len
|
|
325
|
+
pair_idx = pair_indexes[idx]
|
|
326
|
+
elem = positive_raw_data[idx]
|
|
327
|
+
pair_elem = positive_raw_data[pair_idx]
|
|
328
|
+
new_elem = {
|
|
329
|
+
"text": elem["text"],
|
|
330
|
+
"label": pair_elem["label"],
|
|
331
|
+
"score": 0,
|
|
332
|
+
"relation": "negative",
|
|
333
|
+
}
|
|
334
|
+
raw_data.append(new_elem)
|
|
335
|
+
|
|
336
|
+
# 2. Generating negative examples by attaching positive labels to unlabelled negative texts.
|
|
337
|
+
negative_count = int(
|
|
338
|
+
negative_count / first_method_weight * second_method_weight
|
|
339
|
+
) # inversion of the count
|
|
340
|
+
pair_indexes = rng.choice(len(positive_raw_data), negative_count, replace=True)
|
|
341
|
+
negative_indexes = rng.choice(
|
|
342
|
+
len(negative_unlabelled_raw_data),
|
|
343
|
+
negative_count,
|
|
344
|
+
replace=True,
|
|
345
|
+
) # subset of negatives
|
|
346
|
+
for i in tqdm(
|
|
347
|
+
range(negative_count),
|
|
348
|
+
desc="Generating negative examples by attaching positive labels to unlabelled negative texts.",
|
|
349
|
+
):
|
|
350
|
+
negative_elem = negative_unlabelled_raw_data[negative_indexes[i]]
|
|
351
|
+
negative_elem["label"] = positive_raw_data[pair_indexes[i]]["label"]
|
|
352
|
+
# no need to append to raw_data, because we are overwriting 'label' of negative_elem by reference
|
|
353
|
+
|
|
354
|
+
# remove unlabelled data
|
|
355
|
+
raw_data = [x for x in raw_data if x["label"]]
|
|
356
|
+
|
|
357
|
+
# expand along the label dimension
|
|
358
|
+
tmp_data = []
|
|
359
|
+
for elem in raw_data:
|
|
360
|
+
for label in elem["label"]:
|
|
361
|
+
tmp_elem = elem.copy()
|
|
362
|
+
tmp_elem["label"] = label
|
|
363
|
+
tmp_data.append(tmp_elem)
|
|
364
|
+
raw_data = tmp_data
|
|
365
|
+
|
|
366
|
+
# add noise to score to mimic a real dataset
|
|
367
|
+
for elem in raw_data:
|
|
368
|
+
if elem["relation"] == "positive":
|
|
369
|
+
elem["score"] -= abs(rng.normal(0, score_noise_std))
|
|
370
|
+
else:
|
|
371
|
+
elem["score"] += abs(rng.normal(0, score_noise_std))
|
|
372
|
+
|
|
373
|
+
return raw_data
|
|
374
|
+
|
|
375
|
+
def _mapping_function(self, batch):
|
|
376
|
+
# tokenized_text = self._tokenizer(batch["text"], **self.tokenizer_kwargs)[
|
|
377
|
+
# "input_ids"
|
|
378
|
+
# ]
|
|
379
|
+
# tokenized_label = self._tokenizer(batch["label"], **self.tokenizer_kwargs)[
|
|
380
|
+
# "input_ids"
|
|
381
|
+
# ]
|
|
382
|
+
# return {
|
|
383
|
+
# "text": tokenized_text,
|
|
384
|
+
# "label": tokenized_label,
|
|
385
|
+
# "score": batch["score"],
|
|
386
|
+
# "relation": batch["relation"],
|
|
387
|
+
# }
|
|
388
|
+
return {
|
|
389
|
+
"sentence1": batch["text"],
|
|
390
|
+
"sentence2": batch["label"],
|
|
391
|
+
"score": batch["score"],
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
def _transform_to_hf_dataset(self, dataset) -> Dataset:
|
|
395
|
+
texts = []
|
|
396
|
+
labels = []
|
|
397
|
+
scores = []
|
|
398
|
+
relations = []
|
|
399
|
+
for item in dataset:
|
|
400
|
+
texts.append(item["text"])
|
|
401
|
+
cl_label = item["label"]
|
|
402
|
+
labels.append(cl_label)
|
|
403
|
+
scores.append(item["score"])
|
|
404
|
+
relations.append(item["relation"])
|
|
405
|
+
|
|
406
|
+
return Dataset.from_dict(
|
|
407
|
+
{"text": texts, "label": labels, "score": scores, "relation": relations}
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
class DPODataset(DatasetBaseClass):
|
|
412
|
+
MAPPING_ENABLED = False # this type of dataset is not preprocessed (does not utilize preprocessing.dataset.Dataset), not tokenized therefore does not require caching
|
|
413
|
+
|
|
414
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
415
|
+
data_dict = {
|
|
416
|
+
"prompt": elem["prompt"],
|
|
417
|
+
"chosen": elem["chosen"],
|
|
418
|
+
"rejected": elem["rejected"],
|
|
419
|
+
}
|
|
420
|
+
self._raw_data[idx] = data_dict
|
|
421
|
+
|
|
422
|
+
def _mapping_function(self, element):
|
|
423
|
+
return element
|
|
424
|
+
|
|
425
|
+
def _transform_to_hf_dataset(self, dataset) -> Dataset:
|
|
426
|
+
prompt = []
|
|
427
|
+
chosen = []
|
|
428
|
+
rejected = []
|
|
429
|
+
for item in dataset:
|
|
430
|
+
prompt.append(item["prompt"])
|
|
431
|
+
chosen.append(item["chosen"])
|
|
432
|
+
rejected.append(item["rejected"])
|
|
433
|
+
return Dataset.from_dict(
|
|
434
|
+
{"prompt": prompt, "chosen": chosen, "rejected": rejected}
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
class DialogueAlternativesDataset(DatasetBaseClass):
|
|
439
|
+
BATCHED_TOKENIZATION = False
|
|
440
|
+
|
|
441
|
+
def _preprocess_raw_data(self, idx, elem):
|
|
442
|
+
data_dict = {
|
|
443
|
+
"shared_part": elem["dialogue"],
|
|
444
|
+
"endings": elem["endings"],
|
|
445
|
+
"scores": elem["metadata"]["ending_scores"],
|
|
446
|
+
}
|
|
447
|
+
self._raw_data[idx] = data_dict
|
|
448
|
+
|
|
449
|
+
def _mapping_function(self, element):
|
|
450
|
+
tokenized_endings = {"endings": []}
|
|
451
|
+
shared_part = element["shared_part"]
|
|
452
|
+
for ending in element["endings"]:
|
|
453
|
+
# TODO optimize (batched processing) like here https://github.com/imoneoi/openchat/blob/master/ochat/data/generate_dataset.py
|
|
454
|
+
# TODO invent a way to tokenize the context only once and share it via reference (which is the dialogue before an ending)
|
|
455
|
+
dialogue = shared_part + [ending]
|
|
456
|
+
templated_dialogue: str = self._tokenizer.apply_chat_template(
|
|
457
|
+
dialogue, tokenize=False
|
|
458
|
+
)
|
|
459
|
+
tokenized_dialogue = self._tokenizer(
|
|
460
|
+
templated_dialogue, **self.tokenizer_kwargs
|
|
461
|
+
)
|
|
462
|
+
del tokenized_dialogue["attention_mask"]
|
|
463
|
+
tokenized_endings["endings"].append(tokenized_dialogue)
|
|
464
|
+
|
|
465
|
+
shared_part_end_idx = 0
|
|
466
|
+
dialogue_input_ids = [
|
|
467
|
+
dialogue["input_ids"] for dialogue in tokenized_endings["endings"]
|
|
468
|
+
]
|
|
469
|
+
for matching_tokens in zip(*dialogue_input_ids):
|
|
470
|
+
if len(set(matching_tokens)) == 1:
|
|
471
|
+
shared_part_end_idx += 1
|
|
472
|
+
else:
|
|
473
|
+
break
|
|
474
|
+
|
|
475
|
+
shared_part = tokenized_endings["endings"][0]["input_ids"][:shared_part_end_idx]
|
|
476
|
+
for dialogue in tokenized_endings["endings"]:
|
|
477
|
+
dialogue["input_ids"] = dialogue["input_ids"][shared_part_end_idx:]
|
|
478
|
+
tokenized_endings["shared_part"] = shared_part
|
|
479
|
+
tokenized_endings["scores"] = element["scores"]
|
|
480
|
+
return tokenized_endings
|
|
481
|
+
|
|
482
|
+
def _transform_to_hf_dataset(self, dataset) -> Dataset:
|
|
483
|
+
shared_part = []
|
|
484
|
+
scores = []
|
|
485
|
+
endings = []
|
|
486
|
+
for item in dataset:
|
|
487
|
+
shared_part.append(item["shared_part"])
|
|
488
|
+
scores.append(item["scores"])
|
|
489
|
+
endings.append(item["endings"])
|
|
490
|
+
return Dataset.from_dict(
|
|
491
|
+
{"shared_part": shared_part, "scores": scores, "endings": endings}
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
cached_classes_and_names = None
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def get_class_from_type_name(type_name: str) -> DatasetBaseClass:
|
|
499
|
+
global cached_classes_and_names
|
|
500
|
+
if cached_classes_and_names is None:
|
|
501
|
+
cached_classes_and_names = {
|
|
502
|
+
k: v
|
|
503
|
+
for k, v in inspect.getmembers(sys.modules[__name__])
|
|
504
|
+
if inspect.isclass(v) and issubclass(v, DatasetBaseClass)
|
|
505
|
+
}
|
|
506
|
+
if type_name in cached_classes_and_names:
|
|
507
|
+
return cached_classes_and_names[type_name]
|
|
508
|
+
else:
|
|
509
|
+
raise KeyError(f"Dataset class of type: {type_name} does not exist.")
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from radlab_data.datasets.dataset import DatasetBaseClass, get_class_from_type_name
|
|
4
|
+
from radlab_data.preprocessing.dataset import SequenceToSequenceDataset, json_loader
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DatasetLoader:
|
|
8
|
+
"""
|
|
9
|
+
An interface for dataset loading and tokenization.
|
|
10
|
+
|
|
11
|
+
Usage:
|
|
12
|
+
1. Initialize the class.
|
|
13
|
+
2. Use the load method.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
dataset_path,
|
|
19
|
+
tokenizer,
|
|
20
|
+
tokenizer_kwargs=None,
|
|
21
|
+
cache_path="./cache/",
|
|
22
|
+
class_label_field_name=None,
|
|
23
|
+
split_train_valid_test: str = None,
|
|
24
|
+
pre_shuffle=False,
|
|
25
|
+
) -> None:
|
|
26
|
+
"""
|
|
27
|
+
:param dataset_path: Path to dataset
|
|
28
|
+
:param tokenizer: Tokenizer
|
|
29
|
+
:param cache_path: Path to cached data
|
|
30
|
+
:param class_label_field_name: Class label field name
|
|
31
|
+
:param split_train_test: Example format (train:valid:test): "0.4" (train 0.4, valid 0.6) or "0.4:0.6" (train 0.4, valid 0.6) or "0.4:0.1" (train 0.4, valid 0.1, test 0.5) or "0.2:0.3:0.5" (train 0.2, valid 0.3, test 0.5) or None. Create train and test datasets by a given ratio,
|
|
32
|
+
create only train dataset if this variable is None
|
|
33
|
+
:param pre_shuffle: Shuffle the dataset at initialization.
|
|
34
|
+
"""
|
|
35
|
+
if not os.path.exists(cache_path):
|
|
36
|
+
os.makedirs(cache_path)
|
|
37
|
+
|
|
38
|
+
self._dataset_path = dataset_path
|
|
39
|
+
self._metadata = self._read_json("metadata.json")
|
|
40
|
+
self._dataset_type = get_class_from_type_name(self._metadata["dataset_type"])
|
|
41
|
+
|
|
42
|
+
if class_label_field_name is None:
|
|
43
|
+
if issubclass(self._dataset_type, SequenceToSequenceDataset):
|
|
44
|
+
self._class_label_field_name = "labels"
|
|
45
|
+
else:
|
|
46
|
+
self._class_label_field_name = "label"
|
|
47
|
+
else:
|
|
48
|
+
self._class_label_field_name = class_label_field_name
|
|
49
|
+
|
|
50
|
+
self._tokenizer = tokenizer
|
|
51
|
+
self._tokenizer_kwargs = tokenizer_kwargs
|
|
52
|
+
self._cache_path = cache_path
|
|
53
|
+
self._cached_dataset_path = self._get_cached_dataset_path()
|
|
54
|
+
|
|
55
|
+
if split_train_valid_test is not None:
|
|
56
|
+
split_train_valid_test = split_train_valid_test.split(":")
|
|
57
|
+
assert len(split_train_valid_test) in [
|
|
58
|
+
1,
|
|
59
|
+
2,
|
|
60
|
+
3,
|
|
61
|
+
], "Incorrect amount of splits."
|
|
62
|
+
split_train_valid_test = tuple(float(x) for x in split_train_valid_test)
|
|
63
|
+
if len(split_train_valid_test) == 1:
|
|
64
|
+
split_train_valid_test = (
|
|
65
|
+
split_train_valid_test[0],
|
|
66
|
+
1 - split_train_valid_test[0],
|
|
67
|
+
)
|
|
68
|
+
elif len(split_train_valid_test) == 2 and sum(split_train_valid_test) != 1:
|
|
69
|
+
test_num = 1 - sum(split_train_valid_test)
|
|
70
|
+
split_train_valid_test = (*split_train_valid_test, test_num)
|
|
71
|
+
|
|
72
|
+
for split in split_train_valid_test:
|
|
73
|
+
assert (
|
|
74
|
+
0 < split < 1
|
|
75
|
+
), "Split values should be greater than 0 and less than 1"
|
|
76
|
+
assert (
|
|
77
|
+
sum(split_train_valid_test) == 1
|
|
78
|
+
), "Train, Valid and Test rations must add up to 1."
|
|
79
|
+
|
|
80
|
+
self._split_train_valid_test = split_train_valid_test
|
|
81
|
+
self._pre_shuffle = pre_shuffle
|
|
82
|
+
|
|
83
|
+
def load(self) -> DatasetBaseClass:
|
|
84
|
+
"""
|
|
85
|
+
Loads the dataset.
|
|
86
|
+
"""
|
|
87
|
+
raw_data = self._read_json("data.json")
|
|
88
|
+
|
|
89
|
+
dataset = self._dataset_type(
|
|
90
|
+
raw_data,
|
|
91
|
+
self._metadata,
|
|
92
|
+
self._cached_dataset_path,
|
|
93
|
+
self._tokenizer,
|
|
94
|
+
self._tokenizer_kwargs,
|
|
95
|
+
self._class_label_field_name,
|
|
96
|
+
self._split_train_valid_test,
|
|
97
|
+
self._pre_shuffle,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
return dataset
|
|
101
|
+
|
|
102
|
+
def _get_cached_dataset_path(self) -> str:
|
|
103
|
+
import hashlib
|
|
104
|
+
import json as _json
|
|
105
|
+
|
|
106
|
+
date_created = self._metadata["date_created"]
|
|
107
|
+
tokenizer_class_name = self._tokenizer.__class__.__name__
|
|
108
|
+
dataset_name = os.path.basename(self._dataset_path)
|
|
109
|
+
dataset_class_name = self._dataset_type.__name__
|
|
110
|
+
tokenizer_kwargs = _json.dumps(
|
|
111
|
+
self._tokenizer_kwargs, sort_keys=True, default=str
|
|
112
|
+
)
|
|
113
|
+
tokenizer_kwargs_hash = hashlib.sha256(tokenizer_kwargs.encode()).hexdigest()[
|
|
114
|
+
:12
|
|
115
|
+
]
|
|
116
|
+
key_string = (
|
|
117
|
+
dataset_name
|
|
118
|
+
+ "__"
|
|
119
|
+
+ dataset_class_name
|
|
120
|
+
+ "__"
|
|
121
|
+
+ tokenizer_class_name
|
|
122
|
+
+ "__"
|
|
123
|
+
+ tokenizer_kwargs_hash
|
|
124
|
+
+ "__"
|
|
125
|
+
+ date_created
|
|
126
|
+
)
|
|
127
|
+
cache_name = key_string + ".cache"
|
|
128
|
+
cached_dataset_path = os.path.join(self._cache_path, cache_name)
|
|
129
|
+
return cached_dataset_path
|
|
130
|
+
|
|
131
|
+
def _read_json(self, file_name) -> dict:
|
|
132
|
+
"""
|
|
133
|
+
Loads JSON formatted file and tokenizes allowed fields.
|
|
134
|
+
"""
|
|
135
|
+
path_to_load = os.path.join(self._dataset_path, file_name)
|
|
136
|
+
assert os.path.exists(path_to_load), f"{path_to_load} does not exist."
|
|
137
|
+
assert os.path.isfile(path_to_load), f"{path_to_load} is not a file."
|
|
138
|
+
_, extension = os.path.splitext(path_to_load)
|
|
139
|
+
assert extension == ".json", "Only JSON formatted dataset files are supported."
|
|
140
|
+
return json_loader(path_to_load)
|