radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,726 @@
1
+ import itertools
2
+ import json
3
+ import logging
4
+ import os
5
+ from abc import ABC, abstractmethod
6
+ from collections import defaultdict
7
+ from dataclasses import dataclass
8
+ from datetime import datetime
9
+ from typing import TypeVar
10
+
11
+ import pandas as pd
12
+ import sklearn.model_selection
13
+ from radlab_data.utils.special_tokens_rules import (
14
+ RuleApplicator,
15
+ SpecialTokensRulesParser,
16
+ )
17
+
18
+ T_dtype = TypeVar("T_dtype")
19
+
20
+
21
+ def json_loader(dataset_path, class_label_field_name=None):
22
+ """
23
+ Json loader
24
+ :param dataset_path:
25
+ :param class_label_field_name:
26
+ :return:
27
+ """
28
+ # TODO: proper class_label_field_name handling (see _jsonl_loader)
29
+ with open(dataset_path, mode="r", encoding="utf-8") as f:
30
+ data = json.load(f)
31
+ return data
32
+
33
+
34
+ def jsonl_loader(dataset_path, class_label_field_name=None):
35
+ """
36
+ Load jsonl file
37
+ :param dataset_path: Path to jsonl file
38
+ :param class_label_field_name: If given then additionally type of class
39
+ label will be checked
40
+ :return:
41
+ """
42
+ with open(dataset_path, "r", encoding="utf-8") as f:
43
+ json_list = f.readlines()
44
+
45
+ if class_label_field_name is None:
46
+ return [json.loads(json_str) for json_str in json_list]
47
+
48
+ all_dataset_elems = []
49
+ for json_str in json_list:
50
+ jdata = json.loads(json_str)
51
+ cl_data = jdata[class_label_field_name]
52
+ if type(cl_data) in [str]:
53
+ cl_label = cl_data
54
+ jdata[class_label_field_name] = [cl_label]
55
+ all_dataset_elems.append(jdata)
56
+ return all_dataset_elems
57
+
58
+
59
+ @dataclass
60
+ class SaveConfiguration:
61
+ path: str
62
+ extra_metadata: dict = None
63
+ add_date_created_to_name: bool = True
64
+ save_iob_file: bool = False
65
+ save_text_label_xlsx: bool = False
66
+ iob_window_o_class: int = 3
67
+ show_class_labels_histogram: bool = False
68
+
69
+
70
+ class Dataset(ABC):
71
+ ALLOWED_EXTENSIONS = {".json": json_loader, ".jsonl": jsonl_loader}
72
+ DEFAULT_CLASS_LABEL_FIELD = "label"
73
+
74
+ def _load_raw_data(
75
+ self, dataset_path, class_label_field_name, dataset_extension=None
76
+ ):
77
+ is_dir = os.path.isdir(dataset_path)
78
+ if is_dir:
79
+ # if loading dataset from a folder (usually that means multiple
80
+ # files inside, that require merging) the dataset_type must be defined
81
+ assert (
82
+ dataset_extension is not None
83
+ ), "Specify dataset_extension when loading a directory."
84
+ if dataset_extension is None:
85
+ # extension will define the dataset_type
86
+ _, dataset_extension = os.path.splitext(dataset_path)
87
+ assert dataset_extension in self.ALLOWED_EXTENSIONS, (
88
+ f"{dataset_extension =} is not supported. Allowed extensions: "
89
+ f"{list(self.ALLOWED_EXTENSIONS.keys())}"
90
+ )
91
+
92
+ loader_class_label = None
93
+ if class_label_field_name != self.DEFAULT_CLASS_LABEL_FIELD:
94
+ loader_class_label = class_label_field_name
95
+
96
+ self.class_label_field_name = class_label_field_name
97
+
98
+ data_loader_function = self.ALLOWED_EXTENSIONS[dataset_extension]
99
+ if is_dir:
100
+ files = sorted(
101
+ filename
102
+ for filename in os.listdir(dataset_path)
103
+ if filename.endswith(dataset_extension)
104
+ )
105
+ all_data = []
106
+ for filename in files:
107
+ file_path = os.path.join(dataset_path, filename)
108
+ data = data_loader_function(file_path, loader_class_label)
109
+ if data:
110
+ all_data.append(data)
111
+ self._raw_data = list(itertools.chain.from_iterable(all_data))
112
+ else:
113
+ self._raw_data = data_loader_function(dataset_path, loader_class_label)
114
+
115
+ def _generate_date_created(self):
116
+ return str(datetime.now()).replace(" ", "_")
117
+
118
+
119
+ class ClassificationDataset(Dataset, ABC):
120
+ """
121
+ Constructor parameters:
122
+ dataset_path: str - path to the dataset, can be file or folder
123
+ dataset_extension: str = None - type of the dataset, see self.ALLOWED_TYPES
124
+ """
125
+
126
+ def __init__(
127
+ self,
128
+ dataset_path: str,
129
+ dataset_extension: str = None,
130
+ class_label_field_name: str = Dataset.DEFAULT_CLASS_LABEL_FIELD,
131
+ class_labels_mapping_file: str = None,
132
+ ignore_labels_not_mapped: bool = True,
133
+ ) -> None:
134
+ """
135
+
136
+ :param dataset_path:
137
+ :param dataset_extension:
138
+ :param class_label_field_name:
139
+ :param class_labels_mapping_file: If given, then
140
+ class labels mapping will be done
141
+ :param ignore_labels_not_mapped: If mapping is enabled,
142
+ ignore these class labels, which are not mapped
143
+ """
144
+ self.date_created = self._generate_date_created()
145
+ self._load_raw_data(dataset_path, class_label_field_name, dataset_extension)
146
+
147
+ self._class_labels_mapping = {}
148
+ self._ignore_labels_not_mapped = ignore_labels_not_mapped
149
+ if class_labels_mapping_file is not None:
150
+ self._prepare_class_labels_mapping(class_labels_mapping_file)
151
+
152
+ self._random_state = None
153
+ self._is_processed = False
154
+ self._process_raw_data()
155
+ self._label_histogram = self._get_label_histogram(self._raw_data)
156
+ self._map_labels()
157
+
158
+ def _get_mapped_label(self, label_str: str) -> str:
159
+ main_label = label_str
160
+ main_label = self._prepare_single_label(label_str=main_label)
161
+ main_label = self._class_labels_mapping.get(main_label, None)
162
+ if main_label is None and not self._ignore_labels_not_mapped:
163
+ main_label = label_str
164
+ return main_label
165
+
166
+ @property
167
+ def is_processed(self):
168
+ return self._is_processed
169
+
170
+ def __len__(self):
171
+ return len(self._raw_data)
172
+
173
+ def __getitem__(self, index):
174
+ return self._raw_data[index]
175
+
176
+ def _process_raw_data(self):
177
+ assert not self._is_processed
178
+ self._raw_data = [
179
+ {
180
+ "text": x["text"],
181
+ "metadata": {k: v for k, v in x.items() if k != "text"},
182
+ }
183
+ for x in self._raw_data
184
+ ]
185
+ self.labels = list(sorted(self._get_labels(self._raw_data)))
186
+ self.id2label = dict(enumerate(self.labels))
187
+ self.label2id = {v: k for k, v in self.id2label.items()}
188
+ self.num_labels = len(self.labels)
189
+ self._is_processed = True
190
+
191
+ @staticmethod
192
+ def _prepare_single_label(label_str):
193
+ return (
194
+ label_str.replace('\\"', "")
195
+ .replace('"', "")
196
+ .replace("'", "")
197
+ .replace(" ", "_")
198
+ .replace("\t", "_")
199
+ )
200
+
201
+ def _prepare_class_labels_mapping(self, csv_mapping_filepath):
202
+ df = pd.read_csv(csv_mapping_filepath, sep="\t")
203
+ for idx, row in df.iterrows():
204
+ self._class_labels_mapping[
205
+ self._prepare_single_label(row["source_label"])
206
+ ] = self._prepare_single_label(row["target_label"])
207
+
208
+ for source_label, target_label in self._class_labels_mapping.items():
209
+ logging.info(f"Mapped label: {source_label} -> {target_label}")
210
+
211
+ def get_save_dir(self, path, add_date_created_to_name: bool = True) -> str:
212
+ if add_date_created_to_name:
213
+ path = path + "_" + self.date_created
214
+
215
+ if not os.path.exists(path):
216
+ os.makedirs(path)
217
+
218
+ return path
219
+
220
+ def save(
221
+ self,
222
+ save_config: SaveConfiguration,
223
+ ) -> str:
224
+ path = self.get_save_dir(save_config.path, save_config.add_date_created_to_name)
225
+
226
+ metadata_to_save = {
227
+ "dataset_type": self.__class__.__name__,
228
+ "date_created": self.date_created,
229
+ "labels": {
230
+ "names": self.labels,
231
+ "id2label": self.id2label,
232
+ "label2id": self.label2id,
233
+ "num_labels": self.num_labels,
234
+ },
235
+ }
236
+
237
+ if save_config.extra_metadata is not None:
238
+ metadata_to_save.update(save_config.extra_metadata)
239
+
240
+ if self._class_labels_mapping:
241
+ metadata_to_save["labels"]["labels_mapping"] = self._class_labels_mapping
242
+
243
+ data_to_save = self._raw_data
244
+ data_file_path = os.path.join(path, "data.json")
245
+ with open(data_file_path, mode="w", encoding="utf-8") as f:
246
+ json.dump(obj=data_to_save, fp=f, indent=4, ensure_ascii=False)
247
+
248
+ metadata_file_path = os.path.join(path, "metadata.json")
249
+ with open(metadata_file_path, mode="w", encoding="utf-8") as f:
250
+ json.dump(obj=metadata_to_save, fp=f, indent=4, ensure_ascii=False)
251
+ return path
252
+
253
+ @abstractmethod
254
+ def _get_label_distribution(self, data, dtype: T_dtype) -> T_dtype:
255
+ """
256
+ This method returns labels distribution of dtype (can be list or set for example).
257
+ """
258
+ pass
259
+
260
+ @abstractmethod
261
+ def _get_labels(self, data) -> set:
262
+ """
263
+ This method should utilize the _get_label_distribution.
264
+ """
265
+ pass
266
+
267
+ @abstractmethod
268
+ def _map_labels(self):
269
+ pass
270
+
271
+ @property
272
+ def raw_data(self):
273
+ return self._raw_data
274
+
275
+ @raw_data.setter
276
+ def raw_data(self, value):
277
+ self._raw_data = value
278
+
279
+ def _get_label_histogram(self, data):
280
+ all_labels = self._get_label_distribution(data, dtype=list)
281
+ if all_labels and not isinstance(all_labels[0], str):
282
+ raise TypeError(
283
+ "Histogram should be generated before mapping labels to types other than str."
284
+ )
285
+ hist = defaultdict(int)
286
+ for label in all_labels:
287
+ hist[label] += 1
288
+ return hist
289
+
290
+
291
+ class SequenceToSequenceDataset(Dataset):
292
+ def __init__(
293
+ self,
294
+ dataset_path: str,
295
+ specials_tokens_file_path: str = None,
296
+ dataset_extension: str = None,
297
+ class_label_field_name: str = Dataset.DEFAULT_CLASS_LABEL_FIELD,
298
+ ) -> None:
299
+ self.date_created = self._generate_date_created()
300
+ self._load_raw_data(dataset_path, class_label_field_name, dataset_extension)
301
+
302
+ self.special_token_info = None
303
+ self.special_tokens = None
304
+ self.special_tokens_rules = None
305
+ if specials_tokens_file_path is not None:
306
+ self.special_token_info = self._load_special_token_info(
307
+ specials_tokens_file_path
308
+ )
309
+ self.special_tokens = self.special_token_info["special_tokens"]
310
+ self.special_tokens_rules = self.special_token_info["special_token_rules"]
311
+
312
+ self._random_state = None
313
+ self._is_processed = False
314
+
315
+ if specials_tokens_file_path is not None:
316
+ self._special_tokens_rules_applicator = self._parse_special_token_rules()
317
+ self._add_special_tokens_to_data()
318
+
319
+ @property
320
+ def is_processed(self):
321
+ return self._is_processed
322
+
323
+ def __len__(self):
324
+ return len(self._raw_data)
325
+
326
+ def __getitem__(self, index):
327
+ return self._raw_data[index]
328
+
329
+ def _load_special_token_info(self, specials_tokens_file_path):
330
+ return json_loader(specials_tokens_file_path)
331
+
332
+ def _parse_special_token_rules(self) -> RuleApplicator:
333
+ parser = SpecialTokensRulesParser(self.special_tokens)
334
+ applicator_trees = parser.parse(self.special_tokens_rules)
335
+ applicator = RuleApplicator(
336
+ rule_trees=applicator_trees,
337
+ special_tokens=self.special_tokens,
338
+ )
339
+ return applicator
340
+
341
+ def _add_special_tokens_to_data(self):
342
+ self._special_tokens_rules_applicator(self.raw_data)
343
+
344
+ def get_save_dir(self, path, add_date_created_to_name: bool = True) -> str:
345
+ if add_date_created_to_name:
346
+ path = path + "_" + self.date_created
347
+
348
+ if not os.path.exists(path):
349
+ os.makedirs(path)
350
+
351
+ return path
352
+
353
+ def save(
354
+ self,
355
+ save_config: SaveConfiguration,
356
+ ) -> str:
357
+ path = self.get_save_dir(save_config.path, save_config.add_date_created_to_name)
358
+
359
+ metadata_to_save = {
360
+ "dataset_type": self.__class__.__name__,
361
+ "date_created": self.date_created,
362
+ "special_token_info": self.special_token_info,
363
+ }
364
+
365
+ if save_config.extra_metadata is not None:
366
+ metadata_to_save.update(save_config.extra_metadata)
367
+
368
+ data_to_save = self._raw_data
369
+ data_file_path = os.path.join(path, "data.json")
370
+ with open(data_file_path, mode="w", encoding="utf-8") as f:
371
+ json.dump(obj=data_to_save, fp=f, indent=4, ensure_ascii=False)
372
+
373
+ metadata_file_path = os.path.join(path, "metadata.json")
374
+ with open(metadata_file_path, mode="w", encoding="utf-8") as f:
375
+ json.dump(obj=metadata_to_save, fp=f, indent=4, ensure_ascii=False)
376
+ return path
377
+
378
+ @property
379
+ def raw_data(self):
380
+ return self._raw_data
381
+
382
+ @raw_data.setter
383
+ def raw_data(self, value):
384
+ self._raw_data = value
385
+
386
+
387
+ class TextClassificationDataset(ClassificationDataset):
388
+ def __init__(
389
+ self,
390
+ dataset_path: str,
391
+ dataset_extension: str = None,
392
+ class_label_field_name: str = "label",
393
+ class_labels_mapping_file: str = None,
394
+ ) -> None:
395
+ super().__init__(
396
+ dataset_path=dataset_path,
397
+ dataset_extension=dataset_extension,
398
+ class_label_field_name=class_label_field_name,
399
+ class_labels_mapping_file=class_labels_mapping_file,
400
+ )
401
+
402
+ def _get_label_distribution(self, data, dtype):
403
+ labels = (
404
+ l for elem in data for l in elem["metadata"][self.class_label_field_name]
405
+ )
406
+ if len(self._class_labels_mapping):
407
+ labels = (
408
+ self._get_mapped_label(label_str=l)
409
+ for l in labels
410
+ if l is not None
411
+ if self._get_mapped_label(label_str=l) is not None
412
+ )
413
+ return dtype(labels)
414
+
415
+ def _get_labels(self, data):
416
+ return self._get_label_distribution(data, dtype=set)
417
+
418
+ def _map_labels(self):
419
+ for elem in self._raw_data:
420
+ mapped_labels = []
421
+ for label in elem["metadata"][self.class_label_field_name]:
422
+ if len(self._class_labels_mapping):
423
+ label = self._get_mapped_label(label_str=label)
424
+ if label is None:
425
+ continue
426
+ mapped_label = self.label2id[label]
427
+ mapped_labels.append(mapped_label)
428
+ if not mapped_labels: # empty list converts to None == unlabelled data
429
+ mapped_labels = None
430
+ elem["metadata"][self.class_label_field_name] = mapped_labels
431
+
432
+
433
+ class SequenceLabellingDataset(ClassificationDataset):
434
+ def __init__(
435
+ self,
436
+ dataset_path: str,
437
+ dataset_extension: str = None,
438
+ class_label_field_name: str = "label",
439
+ class_labels_mapping_file: str = None,
440
+ ) -> None:
441
+ super().__init__(
442
+ dataset_path=dataset_path,
443
+ dataset_extension=dataset_extension,
444
+ class_label_field_name=class_label_field_name,
445
+ class_labels_mapping_file=class_labels_mapping_file,
446
+ )
447
+ self._show_histogram = False
448
+ self._assert_message = True
449
+ self._label_pair_mapping_cache = {}
450
+
451
+ def save(self, save_config: SaveConfiguration) -> str:
452
+ self._show_histogram = save_config.show_class_labels_histogram
453
+ extra_metadata = {}
454
+ path = self.get_save_dir(
455
+ save_config.path,
456
+ add_date_created_to_name=save_config.add_date_created_to_name,
457
+ )
458
+
459
+ if save_config.save_iob_file or save_config.save_text_label_xlsx:
460
+ out_dataset = self._get_iob_data_format(
461
+ self._raw_data, True, save_config.iob_window_o_class
462
+ )
463
+
464
+ if save_config.save_text_label_xlsx:
465
+ text_label_xlsx_path = os.path.join(
466
+ path,
467
+ "annotations__text_label_pairs.xlsx",
468
+ )
469
+ text_label_pairs = self._get_text_label_xlsx_format(out_dataset)
470
+ text_label_pairs_flat = []
471
+ # concat all splits
472
+ for split in text_label_pairs.values():
473
+ text_label_pairs_flat.extend(split)
474
+ df = pd.DataFrame(
475
+ text_label_pairs_flat, columns=["tagged_text", "label"]
476
+ )
477
+ df.to_excel(text_label_xlsx_path, index=False)
478
+
479
+ if save_config.save_iob_file:
480
+ iob_file_path = os.path.join(
481
+ path,
482
+ "data_iob.json",
483
+ )
484
+ with open(iob_file_path, "wt") as fout:
485
+ json.dump(out_dataset, fout, indent=2)
486
+
487
+ cl_hist = self._get_advanced_iob_label_histogram(out_dataset)
488
+ extra_metadata["advanced_iob_class_labels_hist"] = cl_hist
489
+
490
+ if self._show_histogram:
491
+ print(json.dumps(cl_hist, indent=2))
492
+
493
+ extra_metadata["class_labels_hist"] = self._label_histogram
494
+ save_config.extra_metadata = extra_metadata
495
+ return super().save(save_config)
496
+
497
+ def _get_text_label_xlsx_format(self, iob_format_dataset):
498
+ pairs = {}
499
+ for split, split_values in iob_format_dataset.items():
500
+ if split not in pairs:
501
+ pairs[split] = []
502
+ for text_pack in split_values:
503
+ for text in text_pack:
504
+ labels = set([x[2:] for x in text["iob_tags"].split() if x != "O"])
505
+ if len(labels) == 1:
506
+ label = list(labels)[0]
507
+ else:
508
+ raise RuntimeError()
509
+ elem = {"tagged_text": text["tagged_text"], "label": label}
510
+ pairs[split].append(elem)
511
+ return pairs
512
+
513
+ def _get_iob_data_format(
514
+ self,
515
+ dataset: list,
516
+ merge_example_labels: bool = True,
517
+ iob_window_o_class: int = 3,
518
+ ):
519
+ all_examples = self._convert_to_iob_examples(
520
+ dataset, merge_example_labels, iob_window_o_class
521
+ )
522
+ train_data, test_data = sklearn.model_selection.train_test_split(
523
+ all_examples,
524
+ test_size=0.05,
525
+ shuffle=True,
526
+ random_state=42 if self._random_state is None else self._random_state,
527
+ )
528
+ out_dataset = {"train": train_data, "test": test_data}
529
+ return out_dataset
530
+
531
+ @staticmethod
532
+ def _get_advanced_iob_label_histogram(dataset_dict: dict):
533
+ """
534
+ Generates a histogram of class labels distribution
535
+ on the IOB level after tokenization and label
536
+ assignment to each token.
537
+ """
538
+ hist = {}
539
+ for split, split_dataset in dataset_dict.items():
540
+ hist[split] = {}
541
+ for example in split_dataset:
542
+ examples = [example]
543
+ if type(example) in [list]:
544
+ examples = example
545
+ for e in examples:
546
+ for label in e["iob_tags"].split():
547
+ if label not in hist[split]:
548
+ hist[split][label] = 0
549
+ hist[split][label] += 1
550
+ return hist
551
+
552
+ def _convert_to_iob_examples(
553
+ self,
554
+ data_to_save: list,
555
+ merge_example_labels: bool,
556
+ iob_window_o_class: int = 3,
557
+ ):
558
+ all_examples = []
559
+ d_idx = 0
560
+ for d_elem in data_to_save:
561
+ iob_anno, d_idx = self._iob_for_text_labels(
562
+ dataset_elem=d_elem,
563
+ dataset_idx=d_idx,
564
+ merge_example_labels=merge_example_labels,
565
+ iob_window_o_class=iob_window_o_class,
566
+ )
567
+ # all_examples.extend(iob_anno)
568
+ all_examples.append(iob_anno)
569
+ return all_examples
570
+
571
+ def _iob_for_text_labels(
572
+ self,
573
+ dataset_elem: dict,
574
+ dataset_idx: int,
575
+ merge_example_labels: bool = True,
576
+ iob_window_o_class: int = 3,
577
+ ):
578
+ out_iob_anno = []
579
+ text_str = dataset_elem["text"]
580
+ text_labels = dataset_elem["metadata"][self.class_label_field_name]
581
+ for labels in text_labels:
582
+ b_label_pos, e_label_pos, _, iob_ann_i_tag = labels
583
+ begin_text = text_str[:b_label_pos]
584
+ anno_text = text_str[b_label_pos:e_label_pos]
585
+ end_text = text_str[e_label_pos:]
586
+ label_str = self._prepare_single_label(
587
+ label_str=self.id2label.get(iob_ann_i_tag)
588
+ )
589
+
590
+ if not len(anno_text.strip()):
591
+ print(
592
+ "Empty text:",
593
+ b_label_pos,
594
+ e_label_pos,
595
+ "\n",
596
+ len(text_str),
597
+ text_str,
598
+ )
599
+ continue
600
+
601
+ iob_anno = [f"{label_str}"] * len(anno_text.split())
602
+ iob_anno[0] = "B" + iob_anno[0][1:]
603
+ o_begin = ["O"] * len(begin_text.split())
604
+ o_end = ["O"] * len(end_text.split())
605
+
606
+ m_text_str = text_str
607
+ if iob_window_o_class > 0:
608
+ o_begin = o_begin[-iob_window_o_class:]
609
+ o_end = o_end[:iob_window_o_class]
610
+ bt_spl = begin_text.split()[-iob_window_o_class:]
611
+ et_spl = end_text.split()[:iob_window_o_class]
612
+ at_spl = anno_text.split()
613
+ m_text_str = " ".join(bt_spl + at_spl + et_spl)
614
+
615
+ end_text = " ".join(et_spl)
616
+ begin_text = " ".join(bt_spl)
617
+
618
+ all_str_labels = o_begin + iob_anno + o_end
619
+ labels_str = " ".join(all_str_labels).strip()
620
+ tagged_text = f"{begin_text}[[{anno_text}]]{end_text}".rstrip()
621
+
622
+ if len(m_text_str.strip().split()) != len(labels_str.split()):
623
+ if self._assert_message:
624
+ assert len(m_text_str.strip().split()) == len(labels_str.split()), (
625
+ f"\n\nDifferent number of words and tags after "
626
+ f"converting text {dataset_idx}!\n\n"
627
+ f"{len(labels_str.split())}: {labels_str}\n\n"
628
+ f"{len(m_text_str.split())}: {m_text_str}\n\n"
629
+ f"{len(tagged_text.split())}: {tagged_text}\n\n"
630
+ )
631
+ else:
632
+ print("Different length of annotation and text, skipping...")
633
+ else:
634
+ out_iob_anno.append(
635
+ {
636
+ "id": dataset_idx,
637
+ "raw_text": m_text_str,
638
+ "iob_tags": labels_str,
639
+ "tagged_text": tagged_text,
640
+ }
641
+ )
642
+ dataset_idx += 1
643
+ if merge_example_labels:
644
+ out_iob_anno = self._merge_text_annotations(text_annotations=out_iob_anno)
645
+
646
+ return out_iob_anno, dataset_idx
647
+
648
+ @staticmethod
649
+ def _merge_text_annotations(text_annotations: list):
650
+ # annotations = [
651
+ # t.split() for t in (a['iob_tags'] for a in text_annotations)
652
+ # ]
653
+ #
654
+ # print(annotations)
655
+ #
656
+ # new_annotations = []
657
+ # for i in range(len(annotations[0])):
658
+ # new_label_str = 'O'
659
+ # for a in annotations:
660
+ # label = a[i]
661
+ # if label != 'O':
662
+ # new_label_str = label
663
+ # break
664
+ # new_annotations.append(new_label_str)
665
+ #
666
+ # new_anno_str = " ".join(new_annotations)
667
+ # new_annotation = text_annotations[0]
668
+ # new_annotation["iob_tags"] = new_anno_str
669
+ #
670
+ # return [new_annotation]
671
+ return text_annotations
672
+
673
+ def _get_label_distribution(self, data, dtype=set):
674
+ original_labels = dtype(
675
+ l[2] for elem in data for l in elem["metadata"][self.class_label_field_name]
676
+ )
677
+ # TODO: why len instead of "if self._class_labels_maping:"
678
+ if len(self._class_labels_mapping):
679
+ original_labels = dtype(
680
+ self._get_mapped_label(label_str=l)
681
+ for l in original_labels
682
+ if l is not None
683
+ if self._get_mapped_label(label_str=l) is not None
684
+ )
685
+
686
+ labels_I = dtype(f"I-{elem}" for elem in original_labels)
687
+ labels_B = dtype(f"B-{elem}" for elem in original_labels)
688
+ outside_labels = dtype("O")
689
+
690
+ if dtype == set:
691
+ return labels_I | labels_B | outside_labels
692
+ elif dtype == list:
693
+ return labels_I + labels_B + outside_labels
694
+ else:
695
+ return labels_I, labels_B, outside_labels
696
+
697
+ def _get_labels(self, data):
698
+ return self._get_label_distribution(data, dtype=set)
699
+
700
+ def _map_label_pair(self, label_str):
701
+ label_pair = self._label_pair_mapping_cache.get(label_str, None)
702
+ if label_pair is None:
703
+ label_pair = (
704
+ self.label2id[f"B-{label_str}"],
705
+ self.label2id[f"I-{label_str}"],
706
+ )
707
+ self._label_pair_mapping_cache[label_str] = label_pair
708
+ return label_pair
709
+
710
+ def _map_labels(self):
711
+ for elem in self._raw_data:
712
+ mapped_labels = []
713
+ for idx, label in enumerate(elem["metadata"][self.class_label_field_name]):
714
+ label_str = label[2]
715
+ if len(self._class_labels_mapping):
716
+ label_str = self._get_mapped_label(label_str)
717
+
718
+ if label_str is None:
719
+ continue
720
+ label_b_and_i = self._map_label_pair(label_str)
721
+ label[2:4] = label_b_and_i
722
+ mapped_labels.append(label)
723
+
724
+ if not mapped_labels: # empty list converts to None == unlabelled data
725
+ mapped_labels = None
726
+ elem["metadata"][self.class_label_field_name] = mapped_labels