radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,726 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from typing import TypeVar
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import sklearn.model_selection
|
|
13
|
+
from radlab_data.utils.special_tokens_rules import (
|
|
14
|
+
RuleApplicator,
|
|
15
|
+
SpecialTokensRulesParser,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
T_dtype = TypeVar("T_dtype")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def json_loader(dataset_path, class_label_field_name=None):
|
|
22
|
+
"""
|
|
23
|
+
Json loader
|
|
24
|
+
:param dataset_path:
|
|
25
|
+
:param class_label_field_name:
|
|
26
|
+
:return:
|
|
27
|
+
"""
|
|
28
|
+
# TODO: proper class_label_field_name handling (see _jsonl_loader)
|
|
29
|
+
with open(dataset_path, mode="r", encoding="utf-8") as f:
|
|
30
|
+
data = json.load(f)
|
|
31
|
+
return data
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def jsonl_loader(dataset_path, class_label_field_name=None):
|
|
35
|
+
"""
|
|
36
|
+
Load jsonl file
|
|
37
|
+
:param dataset_path: Path to jsonl file
|
|
38
|
+
:param class_label_field_name: If given then additionally type of class
|
|
39
|
+
label will be checked
|
|
40
|
+
:return:
|
|
41
|
+
"""
|
|
42
|
+
with open(dataset_path, "r", encoding="utf-8") as f:
|
|
43
|
+
json_list = f.readlines()
|
|
44
|
+
|
|
45
|
+
if class_label_field_name is None:
|
|
46
|
+
return [json.loads(json_str) for json_str in json_list]
|
|
47
|
+
|
|
48
|
+
all_dataset_elems = []
|
|
49
|
+
for json_str in json_list:
|
|
50
|
+
jdata = json.loads(json_str)
|
|
51
|
+
cl_data = jdata[class_label_field_name]
|
|
52
|
+
if type(cl_data) in [str]:
|
|
53
|
+
cl_label = cl_data
|
|
54
|
+
jdata[class_label_field_name] = [cl_label]
|
|
55
|
+
all_dataset_elems.append(jdata)
|
|
56
|
+
return all_dataset_elems
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class SaveConfiguration:
|
|
61
|
+
path: str
|
|
62
|
+
extra_metadata: dict = None
|
|
63
|
+
add_date_created_to_name: bool = True
|
|
64
|
+
save_iob_file: bool = False
|
|
65
|
+
save_text_label_xlsx: bool = False
|
|
66
|
+
iob_window_o_class: int = 3
|
|
67
|
+
show_class_labels_histogram: bool = False
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Dataset(ABC):
|
|
71
|
+
ALLOWED_EXTENSIONS = {".json": json_loader, ".jsonl": jsonl_loader}
|
|
72
|
+
DEFAULT_CLASS_LABEL_FIELD = "label"
|
|
73
|
+
|
|
74
|
+
def _load_raw_data(
|
|
75
|
+
self, dataset_path, class_label_field_name, dataset_extension=None
|
|
76
|
+
):
|
|
77
|
+
is_dir = os.path.isdir(dataset_path)
|
|
78
|
+
if is_dir:
|
|
79
|
+
# if loading dataset from a folder (usually that means multiple
|
|
80
|
+
# files inside, that require merging) the dataset_type must be defined
|
|
81
|
+
assert (
|
|
82
|
+
dataset_extension is not None
|
|
83
|
+
), "Specify dataset_extension when loading a directory."
|
|
84
|
+
if dataset_extension is None:
|
|
85
|
+
# extension will define the dataset_type
|
|
86
|
+
_, dataset_extension = os.path.splitext(dataset_path)
|
|
87
|
+
assert dataset_extension in self.ALLOWED_EXTENSIONS, (
|
|
88
|
+
f"{dataset_extension =} is not supported. Allowed extensions: "
|
|
89
|
+
f"{list(self.ALLOWED_EXTENSIONS.keys())}"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
loader_class_label = None
|
|
93
|
+
if class_label_field_name != self.DEFAULT_CLASS_LABEL_FIELD:
|
|
94
|
+
loader_class_label = class_label_field_name
|
|
95
|
+
|
|
96
|
+
self.class_label_field_name = class_label_field_name
|
|
97
|
+
|
|
98
|
+
data_loader_function = self.ALLOWED_EXTENSIONS[dataset_extension]
|
|
99
|
+
if is_dir:
|
|
100
|
+
files = sorted(
|
|
101
|
+
filename
|
|
102
|
+
for filename in os.listdir(dataset_path)
|
|
103
|
+
if filename.endswith(dataset_extension)
|
|
104
|
+
)
|
|
105
|
+
all_data = []
|
|
106
|
+
for filename in files:
|
|
107
|
+
file_path = os.path.join(dataset_path, filename)
|
|
108
|
+
data = data_loader_function(file_path, loader_class_label)
|
|
109
|
+
if data:
|
|
110
|
+
all_data.append(data)
|
|
111
|
+
self._raw_data = list(itertools.chain.from_iterable(all_data))
|
|
112
|
+
else:
|
|
113
|
+
self._raw_data = data_loader_function(dataset_path, loader_class_label)
|
|
114
|
+
|
|
115
|
+
def _generate_date_created(self):
|
|
116
|
+
return str(datetime.now()).replace(" ", "_")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ClassificationDataset(Dataset, ABC):
|
|
120
|
+
"""
|
|
121
|
+
Constructor parameters:
|
|
122
|
+
dataset_path: str - path to the dataset, can be file or folder
|
|
123
|
+
dataset_extension: str = None - type of the dataset, see self.ALLOWED_TYPES
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
dataset_path: str,
|
|
129
|
+
dataset_extension: str = None,
|
|
130
|
+
class_label_field_name: str = Dataset.DEFAULT_CLASS_LABEL_FIELD,
|
|
131
|
+
class_labels_mapping_file: str = None,
|
|
132
|
+
ignore_labels_not_mapped: bool = True,
|
|
133
|
+
) -> None:
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
:param dataset_path:
|
|
137
|
+
:param dataset_extension:
|
|
138
|
+
:param class_label_field_name:
|
|
139
|
+
:param class_labels_mapping_file: If given, then
|
|
140
|
+
class labels mapping will be done
|
|
141
|
+
:param ignore_labels_not_mapped: If mapping is enabled,
|
|
142
|
+
ignore these class labels, which are not mapped
|
|
143
|
+
"""
|
|
144
|
+
self.date_created = self._generate_date_created()
|
|
145
|
+
self._load_raw_data(dataset_path, class_label_field_name, dataset_extension)
|
|
146
|
+
|
|
147
|
+
self._class_labels_mapping = {}
|
|
148
|
+
self._ignore_labels_not_mapped = ignore_labels_not_mapped
|
|
149
|
+
if class_labels_mapping_file is not None:
|
|
150
|
+
self._prepare_class_labels_mapping(class_labels_mapping_file)
|
|
151
|
+
|
|
152
|
+
self._random_state = None
|
|
153
|
+
self._is_processed = False
|
|
154
|
+
self._process_raw_data()
|
|
155
|
+
self._label_histogram = self._get_label_histogram(self._raw_data)
|
|
156
|
+
self._map_labels()
|
|
157
|
+
|
|
158
|
+
def _get_mapped_label(self, label_str: str) -> str:
|
|
159
|
+
main_label = label_str
|
|
160
|
+
main_label = self._prepare_single_label(label_str=main_label)
|
|
161
|
+
main_label = self._class_labels_mapping.get(main_label, None)
|
|
162
|
+
if main_label is None and not self._ignore_labels_not_mapped:
|
|
163
|
+
main_label = label_str
|
|
164
|
+
return main_label
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def is_processed(self):
|
|
168
|
+
return self._is_processed
|
|
169
|
+
|
|
170
|
+
def __len__(self):
|
|
171
|
+
return len(self._raw_data)
|
|
172
|
+
|
|
173
|
+
def __getitem__(self, index):
|
|
174
|
+
return self._raw_data[index]
|
|
175
|
+
|
|
176
|
+
def _process_raw_data(self):
|
|
177
|
+
assert not self._is_processed
|
|
178
|
+
self._raw_data = [
|
|
179
|
+
{
|
|
180
|
+
"text": x["text"],
|
|
181
|
+
"metadata": {k: v for k, v in x.items() if k != "text"},
|
|
182
|
+
}
|
|
183
|
+
for x in self._raw_data
|
|
184
|
+
]
|
|
185
|
+
self.labels = list(sorted(self._get_labels(self._raw_data)))
|
|
186
|
+
self.id2label = dict(enumerate(self.labels))
|
|
187
|
+
self.label2id = {v: k for k, v in self.id2label.items()}
|
|
188
|
+
self.num_labels = len(self.labels)
|
|
189
|
+
self._is_processed = True
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def _prepare_single_label(label_str):
|
|
193
|
+
return (
|
|
194
|
+
label_str.replace('\\"', "")
|
|
195
|
+
.replace('"', "")
|
|
196
|
+
.replace("'", "")
|
|
197
|
+
.replace(" ", "_")
|
|
198
|
+
.replace("\t", "_")
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def _prepare_class_labels_mapping(self, csv_mapping_filepath):
|
|
202
|
+
df = pd.read_csv(csv_mapping_filepath, sep="\t")
|
|
203
|
+
for idx, row in df.iterrows():
|
|
204
|
+
self._class_labels_mapping[
|
|
205
|
+
self._prepare_single_label(row["source_label"])
|
|
206
|
+
] = self._prepare_single_label(row["target_label"])
|
|
207
|
+
|
|
208
|
+
for source_label, target_label in self._class_labels_mapping.items():
|
|
209
|
+
logging.info(f"Mapped label: {source_label} -> {target_label}")
|
|
210
|
+
|
|
211
|
+
def get_save_dir(self, path, add_date_created_to_name: bool = True) -> str:
|
|
212
|
+
if add_date_created_to_name:
|
|
213
|
+
path = path + "_" + self.date_created
|
|
214
|
+
|
|
215
|
+
if not os.path.exists(path):
|
|
216
|
+
os.makedirs(path)
|
|
217
|
+
|
|
218
|
+
return path
|
|
219
|
+
|
|
220
|
+
def save(
|
|
221
|
+
self,
|
|
222
|
+
save_config: SaveConfiguration,
|
|
223
|
+
) -> str:
|
|
224
|
+
path = self.get_save_dir(save_config.path, save_config.add_date_created_to_name)
|
|
225
|
+
|
|
226
|
+
metadata_to_save = {
|
|
227
|
+
"dataset_type": self.__class__.__name__,
|
|
228
|
+
"date_created": self.date_created,
|
|
229
|
+
"labels": {
|
|
230
|
+
"names": self.labels,
|
|
231
|
+
"id2label": self.id2label,
|
|
232
|
+
"label2id": self.label2id,
|
|
233
|
+
"num_labels": self.num_labels,
|
|
234
|
+
},
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if save_config.extra_metadata is not None:
|
|
238
|
+
metadata_to_save.update(save_config.extra_metadata)
|
|
239
|
+
|
|
240
|
+
if self._class_labels_mapping:
|
|
241
|
+
metadata_to_save["labels"]["labels_mapping"] = self._class_labels_mapping
|
|
242
|
+
|
|
243
|
+
data_to_save = self._raw_data
|
|
244
|
+
data_file_path = os.path.join(path, "data.json")
|
|
245
|
+
with open(data_file_path, mode="w", encoding="utf-8") as f:
|
|
246
|
+
json.dump(obj=data_to_save, fp=f, indent=4, ensure_ascii=False)
|
|
247
|
+
|
|
248
|
+
metadata_file_path = os.path.join(path, "metadata.json")
|
|
249
|
+
with open(metadata_file_path, mode="w", encoding="utf-8") as f:
|
|
250
|
+
json.dump(obj=metadata_to_save, fp=f, indent=4, ensure_ascii=False)
|
|
251
|
+
return path
|
|
252
|
+
|
|
253
|
+
@abstractmethod
|
|
254
|
+
def _get_label_distribution(self, data, dtype: T_dtype) -> T_dtype:
|
|
255
|
+
"""
|
|
256
|
+
This method returns labels distribution of dtype (can be list or set for example).
|
|
257
|
+
"""
|
|
258
|
+
pass
|
|
259
|
+
|
|
260
|
+
@abstractmethod
|
|
261
|
+
def _get_labels(self, data) -> set:
|
|
262
|
+
"""
|
|
263
|
+
This method should utilize the _get_label_distribution.
|
|
264
|
+
"""
|
|
265
|
+
pass
|
|
266
|
+
|
|
267
|
+
@abstractmethod
|
|
268
|
+
def _map_labels(self):
|
|
269
|
+
pass
|
|
270
|
+
|
|
271
|
+
@property
|
|
272
|
+
def raw_data(self):
|
|
273
|
+
return self._raw_data
|
|
274
|
+
|
|
275
|
+
@raw_data.setter
|
|
276
|
+
def raw_data(self, value):
|
|
277
|
+
self._raw_data = value
|
|
278
|
+
|
|
279
|
+
def _get_label_histogram(self, data):
|
|
280
|
+
all_labels = self._get_label_distribution(data, dtype=list)
|
|
281
|
+
if all_labels and not isinstance(all_labels[0], str):
|
|
282
|
+
raise TypeError(
|
|
283
|
+
"Histogram should be generated before mapping labels to types other than str."
|
|
284
|
+
)
|
|
285
|
+
hist = defaultdict(int)
|
|
286
|
+
for label in all_labels:
|
|
287
|
+
hist[label] += 1
|
|
288
|
+
return hist
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class SequenceToSequenceDataset(Dataset):
|
|
292
|
+
def __init__(
|
|
293
|
+
self,
|
|
294
|
+
dataset_path: str,
|
|
295
|
+
specials_tokens_file_path: str = None,
|
|
296
|
+
dataset_extension: str = None,
|
|
297
|
+
class_label_field_name: str = Dataset.DEFAULT_CLASS_LABEL_FIELD,
|
|
298
|
+
) -> None:
|
|
299
|
+
self.date_created = self._generate_date_created()
|
|
300
|
+
self._load_raw_data(dataset_path, class_label_field_name, dataset_extension)
|
|
301
|
+
|
|
302
|
+
self.special_token_info = None
|
|
303
|
+
self.special_tokens = None
|
|
304
|
+
self.special_tokens_rules = None
|
|
305
|
+
if specials_tokens_file_path is not None:
|
|
306
|
+
self.special_token_info = self._load_special_token_info(
|
|
307
|
+
specials_tokens_file_path
|
|
308
|
+
)
|
|
309
|
+
self.special_tokens = self.special_token_info["special_tokens"]
|
|
310
|
+
self.special_tokens_rules = self.special_token_info["special_token_rules"]
|
|
311
|
+
|
|
312
|
+
self._random_state = None
|
|
313
|
+
self._is_processed = False
|
|
314
|
+
|
|
315
|
+
if specials_tokens_file_path is not None:
|
|
316
|
+
self._special_tokens_rules_applicator = self._parse_special_token_rules()
|
|
317
|
+
self._add_special_tokens_to_data()
|
|
318
|
+
|
|
319
|
+
@property
|
|
320
|
+
def is_processed(self):
|
|
321
|
+
return self._is_processed
|
|
322
|
+
|
|
323
|
+
def __len__(self):
|
|
324
|
+
return len(self._raw_data)
|
|
325
|
+
|
|
326
|
+
def __getitem__(self, index):
|
|
327
|
+
return self._raw_data[index]
|
|
328
|
+
|
|
329
|
+
def _load_special_token_info(self, specials_tokens_file_path):
|
|
330
|
+
return json_loader(specials_tokens_file_path)
|
|
331
|
+
|
|
332
|
+
def _parse_special_token_rules(self) -> RuleApplicator:
|
|
333
|
+
parser = SpecialTokensRulesParser(self.special_tokens)
|
|
334
|
+
applicator_trees = parser.parse(self.special_tokens_rules)
|
|
335
|
+
applicator = RuleApplicator(
|
|
336
|
+
rule_trees=applicator_trees,
|
|
337
|
+
special_tokens=self.special_tokens,
|
|
338
|
+
)
|
|
339
|
+
return applicator
|
|
340
|
+
|
|
341
|
+
def _add_special_tokens_to_data(self):
|
|
342
|
+
self._special_tokens_rules_applicator(self.raw_data)
|
|
343
|
+
|
|
344
|
+
def get_save_dir(self, path, add_date_created_to_name: bool = True) -> str:
|
|
345
|
+
if add_date_created_to_name:
|
|
346
|
+
path = path + "_" + self.date_created
|
|
347
|
+
|
|
348
|
+
if not os.path.exists(path):
|
|
349
|
+
os.makedirs(path)
|
|
350
|
+
|
|
351
|
+
return path
|
|
352
|
+
|
|
353
|
+
def save(
|
|
354
|
+
self,
|
|
355
|
+
save_config: SaveConfiguration,
|
|
356
|
+
) -> str:
|
|
357
|
+
path = self.get_save_dir(save_config.path, save_config.add_date_created_to_name)
|
|
358
|
+
|
|
359
|
+
metadata_to_save = {
|
|
360
|
+
"dataset_type": self.__class__.__name__,
|
|
361
|
+
"date_created": self.date_created,
|
|
362
|
+
"special_token_info": self.special_token_info,
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
if save_config.extra_metadata is not None:
|
|
366
|
+
metadata_to_save.update(save_config.extra_metadata)
|
|
367
|
+
|
|
368
|
+
data_to_save = self._raw_data
|
|
369
|
+
data_file_path = os.path.join(path, "data.json")
|
|
370
|
+
with open(data_file_path, mode="w", encoding="utf-8") as f:
|
|
371
|
+
json.dump(obj=data_to_save, fp=f, indent=4, ensure_ascii=False)
|
|
372
|
+
|
|
373
|
+
metadata_file_path = os.path.join(path, "metadata.json")
|
|
374
|
+
with open(metadata_file_path, mode="w", encoding="utf-8") as f:
|
|
375
|
+
json.dump(obj=metadata_to_save, fp=f, indent=4, ensure_ascii=False)
|
|
376
|
+
return path
|
|
377
|
+
|
|
378
|
+
@property
|
|
379
|
+
def raw_data(self):
|
|
380
|
+
return self._raw_data
|
|
381
|
+
|
|
382
|
+
@raw_data.setter
|
|
383
|
+
def raw_data(self, value):
|
|
384
|
+
self._raw_data = value
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
class TextClassificationDataset(ClassificationDataset):
|
|
388
|
+
def __init__(
|
|
389
|
+
self,
|
|
390
|
+
dataset_path: str,
|
|
391
|
+
dataset_extension: str = None,
|
|
392
|
+
class_label_field_name: str = "label",
|
|
393
|
+
class_labels_mapping_file: str = None,
|
|
394
|
+
) -> None:
|
|
395
|
+
super().__init__(
|
|
396
|
+
dataset_path=dataset_path,
|
|
397
|
+
dataset_extension=dataset_extension,
|
|
398
|
+
class_label_field_name=class_label_field_name,
|
|
399
|
+
class_labels_mapping_file=class_labels_mapping_file,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
def _get_label_distribution(self, data, dtype):
|
|
403
|
+
labels = (
|
|
404
|
+
l for elem in data for l in elem["metadata"][self.class_label_field_name]
|
|
405
|
+
)
|
|
406
|
+
if len(self._class_labels_mapping):
|
|
407
|
+
labels = (
|
|
408
|
+
self._get_mapped_label(label_str=l)
|
|
409
|
+
for l in labels
|
|
410
|
+
if l is not None
|
|
411
|
+
if self._get_mapped_label(label_str=l) is not None
|
|
412
|
+
)
|
|
413
|
+
return dtype(labels)
|
|
414
|
+
|
|
415
|
+
def _get_labels(self, data):
|
|
416
|
+
return self._get_label_distribution(data, dtype=set)
|
|
417
|
+
|
|
418
|
+
def _map_labels(self):
|
|
419
|
+
for elem in self._raw_data:
|
|
420
|
+
mapped_labels = []
|
|
421
|
+
for label in elem["metadata"][self.class_label_field_name]:
|
|
422
|
+
if len(self._class_labels_mapping):
|
|
423
|
+
label = self._get_mapped_label(label_str=label)
|
|
424
|
+
if label is None:
|
|
425
|
+
continue
|
|
426
|
+
mapped_label = self.label2id[label]
|
|
427
|
+
mapped_labels.append(mapped_label)
|
|
428
|
+
if not mapped_labels: # empty list converts to None == unlabelled data
|
|
429
|
+
mapped_labels = None
|
|
430
|
+
elem["metadata"][self.class_label_field_name] = mapped_labels
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
class SequenceLabellingDataset(ClassificationDataset):
|
|
434
|
+
def __init__(
|
|
435
|
+
self,
|
|
436
|
+
dataset_path: str,
|
|
437
|
+
dataset_extension: str = None,
|
|
438
|
+
class_label_field_name: str = "label",
|
|
439
|
+
class_labels_mapping_file: str = None,
|
|
440
|
+
) -> None:
|
|
441
|
+
super().__init__(
|
|
442
|
+
dataset_path=dataset_path,
|
|
443
|
+
dataset_extension=dataset_extension,
|
|
444
|
+
class_label_field_name=class_label_field_name,
|
|
445
|
+
class_labels_mapping_file=class_labels_mapping_file,
|
|
446
|
+
)
|
|
447
|
+
self._show_histogram = False
|
|
448
|
+
self._assert_message = True
|
|
449
|
+
self._label_pair_mapping_cache = {}
|
|
450
|
+
|
|
451
|
+
def save(self, save_config: SaveConfiguration) -> str:
|
|
452
|
+
self._show_histogram = save_config.show_class_labels_histogram
|
|
453
|
+
extra_metadata = {}
|
|
454
|
+
path = self.get_save_dir(
|
|
455
|
+
save_config.path,
|
|
456
|
+
add_date_created_to_name=save_config.add_date_created_to_name,
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
if save_config.save_iob_file or save_config.save_text_label_xlsx:
|
|
460
|
+
out_dataset = self._get_iob_data_format(
|
|
461
|
+
self._raw_data, True, save_config.iob_window_o_class
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
if save_config.save_text_label_xlsx:
|
|
465
|
+
text_label_xlsx_path = os.path.join(
|
|
466
|
+
path,
|
|
467
|
+
"annotations__text_label_pairs.xlsx",
|
|
468
|
+
)
|
|
469
|
+
text_label_pairs = self._get_text_label_xlsx_format(out_dataset)
|
|
470
|
+
text_label_pairs_flat = []
|
|
471
|
+
# concat all splits
|
|
472
|
+
for split in text_label_pairs.values():
|
|
473
|
+
text_label_pairs_flat.extend(split)
|
|
474
|
+
df = pd.DataFrame(
|
|
475
|
+
text_label_pairs_flat, columns=["tagged_text", "label"]
|
|
476
|
+
)
|
|
477
|
+
df.to_excel(text_label_xlsx_path, index=False)
|
|
478
|
+
|
|
479
|
+
if save_config.save_iob_file:
|
|
480
|
+
iob_file_path = os.path.join(
|
|
481
|
+
path,
|
|
482
|
+
"data_iob.json",
|
|
483
|
+
)
|
|
484
|
+
with open(iob_file_path, "wt") as fout:
|
|
485
|
+
json.dump(out_dataset, fout, indent=2)
|
|
486
|
+
|
|
487
|
+
cl_hist = self._get_advanced_iob_label_histogram(out_dataset)
|
|
488
|
+
extra_metadata["advanced_iob_class_labels_hist"] = cl_hist
|
|
489
|
+
|
|
490
|
+
if self._show_histogram:
|
|
491
|
+
print(json.dumps(cl_hist, indent=2))
|
|
492
|
+
|
|
493
|
+
extra_metadata["class_labels_hist"] = self._label_histogram
|
|
494
|
+
save_config.extra_metadata = extra_metadata
|
|
495
|
+
return super().save(save_config)
|
|
496
|
+
|
|
497
|
+
def _get_text_label_xlsx_format(self, iob_format_dataset):
|
|
498
|
+
pairs = {}
|
|
499
|
+
for split, split_values in iob_format_dataset.items():
|
|
500
|
+
if split not in pairs:
|
|
501
|
+
pairs[split] = []
|
|
502
|
+
for text_pack in split_values:
|
|
503
|
+
for text in text_pack:
|
|
504
|
+
labels = set([x[2:] for x in text["iob_tags"].split() if x != "O"])
|
|
505
|
+
if len(labels) == 1:
|
|
506
|
+
label = list(labels)[0]
|
|
507
|
+
else:
|
|
508
|
+
raise RuntimeError()
|
|
509
|
+
elem = {"tagged_text": text["tagged_text"], "label": label}
|
|
510
|
+
pairs[split].append(elem)
|
|
511
|
+
return pairs
|
|
512
|
+
|
|
513
|
+
def _get_iob_data_format(
|
|
514
|
+
self,
|
|
515
|
+
dataset: list,
|
|
516
|
+
merge_example_labels: bool = True,
|
|
517
|
+
iob_window_o_class: int = 3,
|
|
518
|
+
):
|
|
519
|
+
all_examples = self._convert_to_iob_examples(
|
|
520
|
+
dataset, merge_example_labels, iob_window_o_class
|
|
521
|
+
)
|
|
522
|
+
train_data, test_data = sklearn.model_selection.train_test_split(
|
|
523
|
+
all_examples,
|
|
524
|
+
test_size=0.05,
|
|
525
|
+
shuffle=True,
|
|
526
|
+
random_state=42 if self._random_state is None else self._random_state,
|
|
527
|
+
)
|
|
528
|
+
out_dataset = {"train": train_data, "test": test_data}
|
|
529
|
+
return out_dataset
|
|
530
|
+
|
|
531
|
+
@staticmethod
|
|
532
|
+
def _get_advanced_iob_label_histogram(dataset_dict: dict):
|
|
533
|
+
"""
|
|
534
|
+
Generates a histogram of class labels distribution
|
|
535
|
+
on the IOB level after tokenization and label
|
|
536
|
+
assignment to each token.
|
|
537
|
+
"""
|
|
538
|
+
hist = {}
|
|
539
|
+
for split, split_dataset in dataset_dict.items():
|
|
540
|
+
hist[split] = {}
|
|
541
|
+
for example in split_dataset:
|
|
542
|
+
examples = [example]
|
|
543
|
+
if type(example) in [list]:
|
|
544
|
+
examples = example
|
|
545
|
+
for e in examples:
|
|
546
|
+
for label in e["iob_tags"].split():
|
|
547
|
+
if label not in hist[split]:
|
|
548
|
+
hist[split][label] = 0
|
|
549
|
+
hist[split][label] += 1
|
|
550
|
+
return hist
|
|
551
|
+
|
|
552
|
+
def _convert_to_iob_examples(
|
|
553
|
+
self,
|
|
554
|
+
data_to_save: list,
|
|
555
|
+
merge_example_labels: bool,
|
|
556
|
+
iob_window_o_class: int = 3,
|
|
557
|
+
):
|
|
558
|
+
all_examples = []
|
|
559
|
+
d_idx = 0
|
|
560
|
+
for d_elem in data_to_save:
|
|
561
|
+
iob_anno, d_idx = self._iob_for_text_labels(
|
|
562
|
+
dataset_elem=d_elem,
|
|
563
|
+
dataset_idx=d_idx,
|
|
564
|
+
merge_example_labels=merge_example_labels,
|
|
565
|
+
iob_window_o_class=iob_window_o_class,
|
|
566
|
+
)
|
|
567
|
+
# all_examples.extend(iob_anno)
|
|
568
|
+
all_examples.append(iob_anno)
|
|
569
|
+
return all_examples
|
|
570
|
+
|
|
571
|
+
def _iob_for_text_labels(
|
|
572
|
+
self,
|
|
573
|
+
dataset_elem: dict,
|
|
574
|
+
dataset_idx: int,
|
|
575
|
+
merge_example_labels: bool = True,
|
|
576
|
+
iob_window_o_class: int = 3,
|
|
577
|
+
):
|
|
578
|
+
out_iob_anno = []
|
|
579
|
+
text_str = dataset_elem["text"]
|
|
580
|
+
text_labels = dataset_elem["metadata"][self.class_label_field_name]
|
|
581
|
+
for labels in text_labels:
|
|
582
|
+
b_label_pos, e_label_pos, _, iob_ann_i_tag = labels
|
|
583
|
+
begin_text = text_str[:b_label_pos]
|
|
584
|
+
anno_text = text_str[b_label_pos:e_label_pos]
|
|
585
|
+
end_text = text_str[e_label_pos:]
|
|
586
|
+
label_str = self._prepare_single_label(
|
|
587
|
+
label_str=self.id2label.get(iob_ann_i_tag)
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
if not len(anno_text.strip()):
|
|
591
|
+
print(
|
|
592
|
+
"Empty text:",
|
|
593
|
+
b_label_pos,
|
|
594
|
+
e_label_pos,
|
|
595
|
+
"\n",
|
|
596
|
+
len(text_str),
|
|
597
|
+
text_str,
|
|
598
|
+
)
|
|
599
|
+
continue
|
|
600
|
+
|
|
601
|
+
iob_anno = [f"{label_str}"] * len(anno_text.split())
|
|
602
|
+
iob_anno[0] = "B" + iob_anno[0][1:]
|
|
603
|
+
o_begin = ["O"] * len(begin_text.split())
|
|
604
|
+
o_end = ["O"] * len(end_text.split())
|
|
605
|
+
|
|
606
|
+
m_text_str = text_str
|
|
607
|
+
if iob_window_o_class > 0:
|
|
608
|
+
o_begin = o_begin[-iob_window_o_class:]
|
|
609
|
+
o_end = o_end[:iob_window_o_class]
|
|
610
|
+
bt_spl = begin_text.split()[-iob_window_o_class:]
|
|
611
|
+
et_spl = end_text.split()[:iob_window_o_class]
|
|
612
|
+
at_spl = anno_text.split()
|
|
613
|
+
m_text_str = " ".join(bt_spl + at_spl + et_spl)
|
|
614
|
+
|
|
615
|
+
end_text = " ".join(et_spl)
|
|
616
|
+
begin_text = " ".join(bt_spl)
|
|
617
|
+
|
|
618
|
+
all_str_labels = o_begin + iob_anno + o_end
|
|
619
|
+
labels_str = " ".join(all_str_labels).strip()
|
|
620
|
+
tagged_text = f"{begin_text}[[{anno_text}]]{end_text}".rstrip()
|
|
621
|
+
|
|
622
|
+
if len(m_text_str.strip().split()) != len(labels_str.split()):
|
|
623
|
+
if self._assert_message:
|
|
624
|
+
assert len(m_text_str.strip().split()) == len(labels_str.split()), (
|
|
625
|
+
f"\n\nDifferent number of words and tags after "
|
|
626
|
+
f"converting text {dataset_idx}!\n\n"
|
|
627
|
+
f"{len(labels_str.split())}: {labels_str}\n\n"
|
|
628
|
+
f"{len(m_text_str.split())}: {m_text_str}\n\n"
|
|
629
|
+
f"{len(tagged_text.split())}: {tagged_text}\n\n"
|
|
630
|
+
)
|
|
631
|
+
else:
|
|
632
|
+
print("Different length of annotation and text, skipping...")
|
|
633
|
+
else:
|
|
634
|
+
out_iob_anno.append(
|
|
635
|
+
{
|
|
636
|
+
"id": dataset_idx,
|
|
637
|
+
"raw_text": m_text_str,
|
|
638
|
+
"iob_tags": labels_str,
|
|
639
|
+
"tagged_text": tagged_text,
|
|
640
|
+
}
|
|
641
|
+
)
|
|
642
|
+
dataset_idx += 1
|
|
643
|
+
if merge_example_labels:
|
|
644
|
+
out_iob_anno = self._merge_text_annotations(text_annotations=out_iob_anno)
|
|
645
|
+
|
|
646
|
+
return out_iob_anno, dataset_idx
|
|
647
|
+
|
|
648
|
+
@staticmethod
|
|
649
|
+
def _merge_text_annotations(text_annotations: list):
|
|
650
|
+
# annotations = [
|
|
651
|
+
# t.split() for t in (a['iob_tags'] for a in text_annotations)
|
|
652
|
+
# ]
|
|
653
|
+
#
|
|
654
|
+
# print(annotations)
|
|
655
|
+
#
|
|
656
|
+
# new_annotations = []
|
|
657
|
+
# for i in range(len(annotations[0])):
|
|
658
|
+
# new_label_str = 'O'
|
|
659
|
+
# for a in annotations:
|
|
660
|
+
# label = a[i]
|
|
661
|
+
# if label != 'O':
|
|
662
|
+
# new_label_str = label
|
|
663
|
+
# break
|
|
664
|
+
# new_annotations.append(new_label_str)
|
|
665
|
+
#
|
|
666
|
+
# new_anno_str = " ".join(new_annotations)
|
|
667
|
+
# new_annotation = text_annotations[0]
|
|
668
|
+
# new_annotation["iob_tags"] = new_anno_str
|
|
669
|
+
#
|
|
670
|
+
# return [new_annotation]
|
|
671
|
+
return text_annotations
|
|
672
|
+
|
|
673
|
+
def _get_label_distribution(self, data, dtype=set):
|
|
674
|
+
original_labels = dtype(
|
|
675
|
+
l[2] for elem in data for l in elem["metadata"][self.class_label_field_name]
|
|
676
|
+
)
|
|
677
|
+
# TODO: why len instead of "if self._class_labels_maping:"
|
|
678
|
+
if len(self._class_labels_mapping):
|
|
679
|
+
original_labels = dtype(
|
|
680
|
+
self._get_mapped_label(label_str=l)
|
|
681
|
+
for l in original_labels
|
|
682
|
+
if l is not None
|
|
683
|
+
if self._get_mapped_label(label_str=l) is not None
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
labels_I = dtype(f"I-{elem}" for elem in original_labels)
|
|
687
|
+
labels_B = dtype(f"B-{elem}" for elem in original_labels)
|
|
688
|
+
outside_labels = dtype("O")
|
|
689
|
+
|
|
690
|
+
if dtype == set:
|
|
691
|
+
return labels_I | labels_B | outside_labels
|
|
692
|
+
elif dtype == list:
|
|
693
|
+
return labels_I + labels_B + outside_labels
|
|
694
|
+
else:
|
|
695
|
+
return labels_I, labels_B, outside_labels
|
|
696
|
+
|
|
697
|
+
def _get_labels(self, data):
|
|
698
|
+
return self._get_label_distribution(data, dtype=set)
|
|
699
|
+
|
|
700
|
+
def _map_label_pair(self, label_str):
|
|
701
|
+
label_pair = self._label_pair_mapping_cache.get(label_str, None)
|
|
702
|
+
if label_pair is None:
|
|
703
|
+
label_pair = (
|
|
704
|
+
self.label2id[f"B-{label_str}"],
|
|
705
|
+
self.label2id[f"I-{label_str}"],
|
|
706
|
+
)
|
|
707
|
+
self._label_pair_mapping_cache[label_str] = label_pair
|
|
708
|
+
return label_pair
|
|
709
|
+
|
|
710
|
+
def _map_labels(self):
|
|
711
|
+
for elem in self._raw_data:
|
|
712
|
+
mapped_labels = []
|
|
713
|
+
for idx, label in enumerate(elem["metadata"][self.class_label_field_name]):
|
|
714
|
+
label_str = label[2]
|
|
715
|
+
if len(self._class_labels_mapping):
|
|
716
|
+
label_str = self._get_mapped_label(label_str)
|
|
717
|
+
|
|
718
|
+
if label_str is None:
|
|
719
|
+
continue
|
|
720
|
+
label_b_and_i = self._map_label_pair(label_str)
|
|
721
|
+
label[2:4] = label_b_and_i
|
|
722
|
+
mapped_labels.append(label)
|
|
723
|
+
|
|
724
|
+
if not mapped_labels: # empty list converts to None == unlabelled data
|
|
725
|
+
mapped_labels = None
|
|
726
|
+
elem["metadata"][self.class_label_field_name] = mapped_labels
|