radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,9 @@
1
+ from importlib.metadata import PackageNotFoundError, version
2
+ from pathlib import Path
3
+
4
+ try:
5
+ __version__ = version("radlab-data")
6
+ except PackageNotFoundError: # running from a source checkout
7
+ __version__ = (Path(__file__).resolve().parent.parent / ".version").read_text(
8
+ encoding="utf-8"
9
+ ).strip()
File without changes
@@ -0,0 +1,509 @@
1
+ import inspect
2
+ import os
3
+ import sys
4
+ from abc import ABC, abstractmethod
5
+
6
+ import numpy as np
7
+ from datasets import Dataset
8
+ from tqdm import tqdm
9
+ from transformers.tokenization_utils import PreTrainedTokenizer
10
+
11
+
12
+ class DatasetBaseClass(ABC, Dataset):
13
+ BATCHED_TOKENIZATION = True
14
+ MAPPING_ENABLED = True
15
+
16
+ def __init__(
17
+ self,
18
+ data,
19
+ metadata,
20
+ cached_dataset_path,
21
+ tokenizer,
22
+ tokenizer_kwargs=None,
23
+ class_label_field="label",
24
+ split_train_valid_test=None,
25
+ pre_shuffle=False,
26
+ ) -> None:
27
+ self._random_seed = 1337
28
+ if tokenizer_kwargs is None:
29
+ tokenizer_kwargs = {}
30
+ self.tokenizer_kwargs = tokenizer_kwargs
31
+
32
+ self._raw_data = data
33
+ self._metadata = metadata
34
+ self.class_label_field = class_label_field
35
+ self._label_data = self._metadata.get("labels", None)
36
+ self._cached_dataset_path = cached_dataset_path
37
+ self._tokenizer: PreTrainedTokenizer = tokenizer
38
+ if (
39
+ "special_token_info" in self._metadata
40
+ and self._metadata["special_token_info"] is not None
41
+ ):
42
+ self._tokenizer.add_special_tokens(
43
+ {
44
+ "additional_special_tokens": self._metadata["special_token_info"][
45
+ "special_tokens"
46
+ ]
47
+ }
48
+ )
49
+
50
+ if self._label_data is not None:
51
+ self.labels = self._label_data["names"]
52
+ self.id2label = {int(k): v for k, v in self._label_data["id2label"].items()}
53
+ self.label2id = self._label_data["label2id"]
54
+ self.num_labels = self._label_data["num_labels"]
55
+
56
+ self._train_dataset = None
57
+ self._valid_dataset = None
58
+ self._test_dataset = None
59
+ self._split_train_valid_test = split_train_valid_test
60
+ self._pre_shuffle = pre_shuffle
61
+
62
+ self._load()
63
+
64
+ @property
65
+ def train_dataset(self):
66
+ return self._train_dataset
67
+
68
+ @property
69
+ def valid_dataset(self):
70
+ return self._valid_dataset
71
+
72
+ @property
73
+ def test_dataset(self):
74
+ return self._test_dataset
75
+
76
+ def _load(self):
77
+ if os.path.exists(self._cached_dataset_path):
78
+ hf_dataset = Dataset.from_file(self._cached_dataset_path)
79
+ else:
80
+ for idx, elem in tqdm(
81
+ enumerate(self._raw_data), desc="Preprocessing raw data"
82
+ ):
83
+ self._preprocess_raw_data(idx, elem)
84
+
85
+ self._raw_data = self._apply_custom_preprocess_function(self._raw_data)
86
+
87
+ hf_dataset = self._transform_to_hf_dataset(self._raw_data)
88
+
89
+ if self.MAPPING_ENABLED:
90
+ hf_dataset = hf_dataset.map(
91
+ self._mapping_function,
92
+ batched=self.BATCHED_TOKENIZATION,
93
+ cache_file_name=self._cached_dataset_path,
94
+ remove_columns=hf_dataset.column_names,
95
+ )
96
+
97
+ if self._pre_shuffle:
98
+ hf_dataset = hf_dataset.shuffle(seed=self._random_seed)
99
+
100
+ self._train_dataset = hf_dataset
101
+ self._valid_dataset = None
102
+ self._test_dataset = None
103
+
104
+ if self._split_train_valid_test is not None:
105
+ train_split = self._split_train_valid_test[0]
106
+ valid_split = self._split_train_valid_test[1]
107
+ train_split_idx = int(len(hf_dataset) * train_split)
108
+ valid_split_idx = int(len(hf_dataset) * (train_split + valid_split))
109
+ self._train_dataset = Dataset.from_dict(hf_dataset[:train_split_idx])
110
+ self._valid_dataset = Dataset.from_dict(
111
+ hf_dataset[train_split_idx:valid_split_idx]
112
+ )
113
+ if len(self._split_train_valid_test) == 3:
114
+ self._test_dataset = Dataset.from_dict(hf_dataset[valid_split_idx:])
115
+
116
+ def _transform_to_hf_dataset(self, dataset) -> Dataset:
117
+ texts = []
118
+ labels = []
119
+ for item in dataset:
120
+ texts.append(item["text"])
121
+ cl_label = item["label"]
122
+ labels.append(cl_label)
123
+ return Dataset.from_dict({"text": texts, "label": labels})
124
+
125
+ @abstractmethod
126
+ def _mapping_function(self, batch):
127
+ pass
128
+
129
+ @abstractmethod
130
+ def _preprocess_raw_data(self, idx, elem):
131
+ """
132
+ Raw data preprocessing, element by element.
133
+ Prepares data for further usage and conversion to datasets.Dataset object.
134
+ """
135
+ pass
136
+
137
+ def _apply_custom_preprocess_function(self, raw_data):
138
+ """
139
+ It is highly recommended to use _preprocess_raw_data method instead (if possible)!
140
+
141
+ This method allows to apply a custom function to a dataset after element-wise preprocessing,
142
+ but prior to transforming the dataset into a huggingface dataset.
143
+ """
144
+ return raw_data
145
+
146
+ NOT_IMPLEMENTED_MESSAGE = "Use dataset.train_dataset or dataset.test_dataset."
147
+
148
+ def __len__(self):
149
+ raise NotImplementedError(self.NOT_IMPLEMENTED_MESSAGE)
150
+
151
+ def __getitem__(self, index):
152
+ raise NotImplementedError(self.NOT_IMPLEMENTED_MESSAGE)
153
+
154
+
155
+ class SequenceLabellingDataset(DatasetBaseClass):
156
+ def _preprocess_raw_data(self, idx, elem) -> None:
157
+ self._raw_data[idx] = {
158
+ "text": elem["text"],
159
+ self.class_label_field: elem["metadata"][self.class_label_field],
160
+ }
161
+
162
+ def _mapping_function(self, batch):
163
+ tokenized_texts = self._tokenizer(batch["text"], **self.tokenizer_kwargs)
164
+ labels = batch[self.class_label_field]
165
+
166
+ # generate ner_tags lists
167
+ all_ner_tags = []
168
+ outside_label = self.label2id["O"]
169
+ for text_idx, (t_text, label_list) in enumerate(
170
+ zip(tokenized_texts["input_ids"], labels)
171
+ ):
172
+ ner_tags = np.full(len(t_text), outside_label)
173
+ # # print(100 * "-")
174
+ # # print("text_id", text_idx)
175
+ # # print("label_list", label_list)
176
+ # # print("t_text", t_text)
177
+ # # print(batch["text"][text_idx])
178
+ # for begin, end, b_label, i_label in label_list:
179
+ # # subtract 1 because end is the index of a character succeeding the labelled sequence
180
+ # end -= 1
181
+ #
182
+ # begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
183
+ # end_token_idx = tokenized_texts.char_to_token(text_idx, end)
184
+ # # print("begin=", begin, "end=", end, "b_label=", b_label, "i_label=", i_label)
185
+ # # print("begin_token_idx=", begin_token_idx, "end_token_idx=", end_token_idx)
186
+ # # in case the tokenizer truncates text
187
+ # # if begin_token_idx is None or end_token_idx is None:
188
+ # # continue
189
+ # if begin_token_idx is None:
190
+ # continue
191
+ # if end_token_idx is None:
192
+ # end_token_idx = len(batch["text"][text_idx])
193
+ # if begin_token_idx < end_token_idx:
194
+ # ner_tags[begin_token_idx + 1 : end_token_idx + 1] = i_label
195
+ #
196
+ # # print("begin=", begin, "end=", end, "b_label=", b_label, "i_label=", i_label)
197
+ # # print(
198
+ # # "begin_token_idx=", begin_token_idx, type(begin_token_idx),
199
+ # # "end_token_idx=", end_token_idx
200
+ # # )
201
+ # ner_tags[begin_token_idx] = b_label
202
+ # all_ner_tags.append(list(ner_tags))
203
+ # # print(ner_tags)
204
+ # tokenized_texts["labels"] = all_ner_tags
205
+ # # print("all_ner_tags", all_ner_tags)
206
+ # # print(100 * "#")
207
+ for begin, end, b_label, i_label in label_list:
208
+ end -= 1
209
+ begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
210
+ end_token_idx = tokenized_texts.char_to_token(text_idx, end)
211
+
212
+ # in case a whitespace character is selected at begin or end idx
213
+ while begin_token_idx is None and begin <= end:
214
+ begin += 1
215
+ begin_token_idx = tokenized_texts.char_to_token(text_idx, begin)
216
+
217
+ while end_token_idx is None and begin <= end:
218
+ end -= 1
219
+ end_token_idx = tokenized_texts.char_to_token(text_idx, end)
220
+
221
+ if begin_token_idx is None or end_token_idx is None or begin > end:
222
+ continue
223
+
224
+ if begin_token_idx < end_token_idx:
225
+ ner_tags[begin_token_idx + 1 : end_token_idx + 1] = i_label
226
+ ner_tags[begin_token_idx] = b_label
227
+
228
+ all_ner_tags.append(list(ner_tags))
229
+
230
+ tokenized_texts["labels"] = all_ner_tags
231
+ return tokenized_texts
232
+
233
+
234
+ class TextClassificationDataset(DatasetBaseClass):
235
+ def _preprocess_raw_data(self, idx, elem):
236
+ self._raw_data[idx] = {
237
+ "text": elem["text"],
238
+ self.class_label_field: elem["metadata"][self.class_label_field][0],
239
+ }
240
+
241
+ def _mapping_function(self, batch):
242
+ tokenized_batch = self._tokenizer(batch["text"], **self.tokenizer_kwargs)
243
+ tokenized_batch["labels"] = batch[self.class_label_field]
244
+ return tokenized_batch
245
+
246
+
247
+ class SequenceToSequenceDataset(DatasetBaseClass):
248
+ def _preprocess_raw_data(self, idx, elem):
249
+ left_ctx = None
250
+ right_ctx = None
251
+ if "left_ctx" in elem["metadata"]:
252
+ left_ctx = elem["metadata"]["left_ctx"]
253
+ if "right_ctx" in elem["metadata"]:
254
+ right_ctx = elem["metadata"]["right_ctx"]
255
+
256
+ text = f"{left_ctx + ' ' if left_ctx is not None else ''}{elem['text']}{' ' + right_ctx if right_ctx is not None else ''}"
257
+ self._raw_data[idx] = {
258
+ "text": text,
259
+ self.class_label_field: elem[self.class_label_field][0],
260
+ }
261
+
262
+ def _mapping_function(self, batch):
263
+ tokenized_batch = self._tokenizer(
264
+ text=batch["text"],
265
+ text_target=batch[self.class_label_field],
266
+ is_split_into_words=False,
267
+ **self.tokenizer_kwargs,
268
+ )
269
+ return tokenized_batch
270
+
271
+
272
+ class SemanticSimilarityDataset(DatasetBaseClass):
273
+ # MAPPING_ENABLED = False
274
+
275
+ def _preprocess_raw_data(self, idx, elem):
276
+ self._raw_data[idx] = {
277
+ "text": elem["text"],
278
+ "label": elem["labels"],
279
+ "score": elem["metadata"]["score"],
280
+ "relation": elem["metadata"]["relation"],
281
+ }
282
+
283
+ RANDOM_SEED = 1337
284
+
285
+ def _apply_custom_preprocess_function(self, raw_data):
286
+ rng = np.random.default_rng(self.RANDOM_SEED)
287
+ positive_raw_data = [x for x in raw_data if x["relation"] == "positive"]
288
+ negative_unlabelled_raw_data = [
289
+ x for x in raw_data if x["relation"] == "negative" and not x["label"]
290
+ ]
291
+ # TODO move these variables outside
292
+ negative_to_positive_proportion = 2
293
+ first_method_weight = 0.8
294
+ second_method_weight = 1 - first_method_weight
295
+ score_noise_std = 0.001
296
+
297
+ # adjust if there's not enough data in the unlabelled negative subset
298
+ positive_raw_data_len = len(positive_raw_data)
299
+ max_possible_second_method_weight = min(
300
+ (len(negative_unlabelled_raw_data) / positive_raw_data_len), 1
301
+ )
302
+ second_method_weight = min(
303
+ max_possible_second_method_weight, second_method_weight
304
+ )
305
+ first_method_weight = 1 - second_method_weight
306
+
307
+ # 1. Generation of negative examples by mixing positives.
308
+ negative_count = int(
309
+ positive_raw_data_len
310
+ * negative_to_positive_proportion
311
+ * first_method_weight
312
+ )
313
+ pair_indexes = rng.choice(
314
+ negative_count, negative_count, replace=False
315
+ ) # we're doing it this way to maximize randomness, overflowing indices will sadly be repeats, but in case of no overflow, there will be no repetitions
316
+ pair_indexes = pair_indexes % len(
317
+ positive_raw_data
318
+ ) # ensure indices are in range of len(positive_raw_data)
319
+
320
+ for i in tqdm(
321
+ range(negative_count),
322
+ desc="Generating negative examples by mixing positives.",
323
+ ):
324
+ idx = i % positive_raw_data_len
325
+ pair_idx = pair_indexes[idx]
326
+ elem = positive_raw_data[idx]
327
+ pair_elem = positive_raw_data[pair_idx]
328
+ new_elem = {
329
+ "text": elem["text"],
330
+ "label": pair_elem["label"],
331
+ "score": 0,
332
+ "relation": "negative",
333
+ }
334
+ raw_data.append(new_elem)
335
+
336
+ # 2. Generating negative examples by attaching positive labels to unlabelled negative texts.
337
+ negative_count = int(
338
+ negative_count / first_method_weight * second_method_weight
339
+ ) # inversion of the count
340
+ pair_indexes = rng.choice(len(positive_raw_data), negative_count, replace=True)
341
+ negative_indexes = rng.choice(
342
+ len(negative_unlabelled_raw_data),
343
+ negative_count,
344
+ replace=True,
345
+ ) # subset of negatives
346
+ for i in tqdm(
347
+ range(negative_count),
348
+ desc="Generating negative examples by attaching positive labels to unlabelled negative texts.",
349
+ ):
350
+ negative_elem = negative_unlabelled_raw_data[negative_indexes[i]]
351
+ negative_elem["label"] = positive_raw_data[pair_indexes[i]]["label"]
352
+ # no need to append to raw_data, because we are overwriting 'label' of negative_elem by reference
353
+
354
+ # remove unlabelled data
355
+ raw_data = [x for x in raw_data if x["label"]]
356
+
357
+ # expand along the label dimension
358
+ tmp_data = []
359
+ for elem in raw_data:
360
+ for label in elem["label"]:
361
+ tmp_elem = elem.copy()
362
+ tmp_elem["label"] = label
363
+ tmp_data.append(tmp_elem)
364
+ raw_data = tmp_data
365
+
366
+ # add noise to score to mimic a real dataset
367
+ for elem in raw_data:
368
+ if elem["relation"] == "positive":
369
+ elem["score"] -= abs(rng.normal(0, score_noise_std))
370
+ else:
371
+ elem["score"] += abs(rng.normal(0, score_noise_std))
372
+
373
+ return raw_data
374
+
375
+ def _mapping_function(self, batch):
376
+ # tokenized_text = self._tokenizer(batch["text"], **self.tokenizer_kwargs)[
377
+ # "input_ids"
378
+ # ]
379
+ # tokenized_label = self._tokenizer(batch["label"], **self.tokenizer_kwargs)[
380
+ # "input_ids"
381
+ # ]
382
+ # return {
383
+ # "text": tokenized_text,
384
+ # "label": tokenized_label,
385
+ # "score": batch["score"],
386
+ # "relation": batch["relation"],
387
+ # }
388
+ return {
389
+ "sentence1": batch["text"],
390
+ "sentence2": batch["label"],
391
+ "score": batch["score"],
392
+ }
393
+
394
+ def _transform_to_hf_dataset(self, dataset) -> Dataset:
395
+ texts = []
396
+ labels = []
397
+ scores = []
398
+ relations = []
399
+ for item in dataset:
400
+ texts.append(item["text"])
401
+ cl_label = item["label"]
402
+ labels.append(cl_label)
403
+ scores.append(item["score"])
404
+ relations.append(item["relation"])
405
+
406
+ return Dataset.from_dict(
407
+ {"text": texts, "label": labels, "score": scores, "relation": relations}
408
+ )
409
+
410
+
411
+ class DPODataset(DatasetBaseClass):
412
+ MAPPING_ENABLED = False # this type of dataset is not preprocessed (does not utilize preprocessing.dataset.Dataset), not tokenized therefore does not require caching
413
+
414
+ def _preprocess_raw_data(self, idx, elem):
415
+ data_dict = {
416
+ "prompt": elem["prompt"],
417
+ "chosen": elem["chosen"],
418
+ "rejected": elem["rejected"],
419
+ }
420
+ self._raw_data[idx] = data_dict
421
+
422
+ def _mapping_function(self, element):
423
+ return element
424
+
425
+ def _transform_to_hf_dataset(self, dataset) -> Dataset:
426
+ prompt = []
427
+ chosen = []
428
+ rejected = []
429
+ for item in dataset:
430
+ prompt.append(item["prompt"])
431
+ chosen.append(item["chosen"])
432
+ rejected.append(item["rejected"])
433
+ return Dataset.from_dict(
434
+ {"prompt": prompt, "chosen": chosen, "rejected": rejected}
435
+ )
436
+
437
+
438
+ class DialogueAlternativesDataset(DatasetBaseClass):
439
+ BATCHED_TOKENIZATION = False
440
+
441
+ def _preprocess_raw_data(self, idx, elem):
442
+ data_dict = {
443
+ "shared_part": elem["dialogue"],
444
+ "endings": elem["endings"],
445
+ "scores": elem["metadata"]["ending_scores"],
446
+ }
447
+ self._raw_data[idx] = data_dict
448
+
449
+ def _mapping_function(self, element):
450
+ tokenized_endings = {"endings": []}
451
+ shared_part = element["shared_part"]
452
+ for ending in element["endings"]:
453
+ # TODO optimize (batched processing) like here https://github.com/imoneoi/openchat/blob/master/ochat/data/generate_dataset.py
454
+ # TODO invent a way to tokenize the context only once and share it via reference (which is the dialogue before an ending)
455
+ dialogue = shared_part + [ending]
456
+ templated_dialogue: str = self._tokenizer.apply_chat_template(
457
+ dialogue, tokenize=False
458
+ )
459
+ tokenized_dialogue = self._tokenizer(
460
+ templated_dialogue, **self.tokenizer_kwargs
461
+ )
462
+ del tokenized_dialogue["attention_mask"]
463
+ tokenized_endings["endings"].append(tokenized_dialogue)
464
+
465
+ shared_part_end_idx = 0
466
+ dialogue_input_ids = [
467
+ dialogue["input_ids"] for dialogue in tokenized_endings["endings"]
468
+ ]
469
+ for matching_tokens in zip(*dialogue_input_ids):
470
+ if len(set(matching_tokens)) == 1:
471
+ shared_part_end_idx += 1
472
+ else:
473
+ break
474
+
475
+ shared_part = tokenized_endings["endings"][0]["input_ids"][:shared_part_end_idx]
476
+ for dialogue in tokenized_endings["endings"]:
477
+ dialogue["input_ids"] = dialogue["input_ids"][shared_part_end_idx:]
478
+ tokenized_endings["shared_part"] = shared_part
479
+ tokenized_endings["scores"] = element["scores"]
480
+ return tokenized_endings
481
+
482
+ def _transform_to_hf_dataset(self, dataset) -> Dataset:
483
+ shared_part = []
484
+ scores = []
485
+ endings = []
486
+ for item in dataset:
487
+ shared_part.append(item["shared_part"])
488
+ scores.append(item["scores"])
489
+ endings.append(item["endings"])
490
+ return Dataset.from_dict(
491
+ {"shared_part": shared_part, "scores": scores, "endings": endings}
492
+ )
493
+
494
+
495
+ cached_classes_and_names = None
496
+
497
+
498
+ def get_class_from_type_name(type_name: str) -> DatasetBaseClass:
499
+ global cached_classes_and_names
500
+ if cached_classes_and_names is None:
501
+ cached_classes_and_names = {
502
+ k: v
503
+ for k, v in inspect.getmembers(sys.modules[__name__])
504
+ if inspect.isclass(v) and issubclass(v, DatasetBaseClass)
505
+ }
506
+ if type_name in cached_classes_and_names:
507
+ return cached_classes_and_names[type_name]
508
+ else:
509
+ raise KeyError(f"Dataset class of type: {type_name} does not exist.")
@@ -0,0 +1,140 @@
1
+ import os
2
+
3
+ from radlab_data.datasets.dataset import DatasetBaseClass, get_class_from_type_name
4
+ from radlab_data.preprocessing.dataset import SequenceToSequenceDataset, json_loader
5
+
6
+
7
+ class DatasetLoader:
8
+ """
9
+ An interface for dataset loading and tokenization.
10
+
11
+ Usage:
12
+ 1. Initialize the class.
13
+ 2. Use the load method.
14
+ """
15
+
16
+ def __init__(
17
+ self,
18
+ dataset_path,
19
+ tokenizer,
20
+ tokenizer_kwargs=None,
21
+ cache_path="./cache/",
22
+ class_label_field_name=None,
23
+ split_train_valid_test: str = None,
24
+ pre_shuffle=False,
25
+ ) -> None:
26
+ """
27
+ :param dataset_path: Path to dataset
28
+ :param tokenizer: Tokenizer
29
+ :param cache_path: Path to cached data
30
+ :param class_label_field_name: Class label field name
31
+ :param split_train_test: Example format (train:valid:test): "0.4" (train 0.4, valid 0.6) or "0.4:0.6" (train 0.4, valid 0.6) or "0.4:0.1" (train 0.4, valid 0.1, test 0.5) or "0.2:0.3:0.5" (train 0.2, valid 0.3, test 0.5) or None. Create train and test datasets by a given ratio,
32
+ create only train dataset if this variable is None
33
+ :param pre_shuffle: Shuffle the dataset at initialization.
34
+ """
35
+ if not os.path.exists(cache_path):
36
+ os.makedirs(cache_path)
37
+
38
+ self._dataset_path = dataset_path
39
+ self._metadata = self._read_json("metadata.json")
40
+ self._dataset_type = get_class_from_type_name(self._metadata["dataset_type"])
41
+
42
+ if class_label_field_name is None:
43
+ if issubclass(self._dataset_type, SequenceToSequenceDataset):
44
+ self._class_label_field_name = "labels"
45
+ else:
46
+ self._class_label_field_name = "label"
47
+ else:
48
+ self._class_label_field_name = class_label_field_name
49
+
50
+ self._tokenizer = tokenizer
51
+ self._tokenizer_kwargs = tokenizer_kwargs
52
+ self._cache_path = cache_path
53
+ self._cached_dataset_path = self._get_cached_dataset_path()
54
+
55
+ if split_train_valid_test is not None:
56
+ split_train_valid_test = split_train_valid_test.split(":")
57
+ assert len(split_train_valid_test) in [
58
+ 1,
59
+ 2,
60
+ 3,
61
+ ], "Incorrect amount of splits."
62
+ split_train_valid_test = tuple(float(x) for x in split_train_valid_test)
63
+ if len(split_train_valid_test) == 1:
64
+ split_train_valid_test = (
65
+ split_train_valid_test[0],
66
+ 1 - split_train_valid_test[0],
67
+ )
68
+ elif len(split_train_valid_test) == 2 and sum(split_train_valid_test) != 1:
69
+ test_num = 1 - sum(split_train_valid_test)
70
+ split_train_valid_test = (*split_train_valid_test, test_num)
71
+
72
+ for split in split_train_valid_test:
73
+ assert (
74
+ 0 < split < 1
75
+ ), "Split values should be greater than 0 and less than 1"
76
+ assert (
77
+ sum(split_train_valid_test) == 1
78
+ ), "Train, Valid and Test rations must add up to 1."
79
+
80
+ self._split_train_valid_test = split_train_valid_test
81
+ self._pre_shuffle = pre_shuffle
82
+
83
+ def load(self) -> DatasetBaseClass:
84
+ """
85
+ Loads the dataset.
86
+ """
87
+ raw_data = self._read_json("data.json")
88
+
89
+ dataset = self._dataset_type(
90
+ raw_data,
91
+ self._metadata,
92
+ self._cached_dataset_path,
93
+ self._tokenizer,
94
+ self._tokenizer_kwargs,
95
+ self._class_label_field_name,
96
+ self._split_train_valid_test,
97
+ self._pre_shuffle,
98
+ )
99
+
100
+ return dataset
101
+
102
+ def _get_cached_dataset_path(self) -> str:
103
+ import hashlib
104
+ import json as _json
105
+
106
+ date_created = self._metadata["date_created"]
107
+ tokenizer_class_name = self._tokenizer.__class__.__name__
108
+ dataset_name = os.path.basename(self._dataset_path)
109
+ dataset_class_name = self._dataset_type.__name__
110
+ tokenizer_kwargs = _json.dumps(
111
+ self._tokenizer_kwargs, sort_keys=True, default=str
112
+ )
113
+ tokenizer_kwargs_hash = hashlib.sha256(tokenizer_kwargs.encode()).hexdigest()[
114
+ :12
115
+ ]
116
+ key_string = (
117
+ dataset_name
118
+ + "__"
119
+ + dataset_class_name
120
+ + "__"
121
+ + tokenizer_class_name
122
+ + "__"
123
+ + tokenizer_kwargs_hash
124
+ + "__"
125
+ + date_created
126
+ )
127
+ cache_name = key_string + ".cache"
128
+ cached_dataset_path = os.path.join(self._cache_path, cache_name)
129
+ return cached_dataset_path
130
+
131
+ def _read_json(self, file_name) -> dict:
132
+ """
133
+ Loads JSON formatted file and tokenizes allowed fields.
134
+ """
135
+ path_to_load = os.path.join(self._dataset_path, file_name)
136
+ assert os.path.exists(path_to_load), f"{path_to_load} does not exist."
137
+ assert os.path.isfile(path_to_load), f"{path_to_load} is not a file."
138
+ _, extension = os.path.splitext(path_to_load)
139
+ assert extension == ".json", "Only JSON formatted dataset files are supported."
140
+ return json_loader(path_to_load)