radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
from radlab_data.preprocessing.dataset import ClassificationDataset
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PipelineElementBase(ABC):
|
|
8
|
+
ALLOWED_DATASET_TYPES: tuple = ()
|
|
9
|
+
ACTION_MAPPING: dict = {}
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
def __call__(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
16
|
+
assert self._dataset_type_check(
|
|
17
|
+
dataset
|
|
18
|
+
), f"{self.__class__.__name__} pipeline element does not support dataset of type {dataset.__class__.__name__}."
|
|
19
|
+
return self._process_data(dataset)
|
|
20
|
+
|
|
21
|
+
def _dataset_type_check(self, dataset: ClassificationDataset) -> bool:
|
|
22
|
+
if self.ALLOWED_DATASET_TYPES:
|
|
23
|
+
return isinstance(dataset, self.ALLOWED_DATASET_TYPES)
|
|
24
|
+
else:
|
|
25
|
+
# ALLOWED_DATASET_TYPES empty => any type allowed
|
|
26
|
+
return True
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Pipeline:
|
|
30
|
+
def __init__(self, modules: List[PipelineElementBase]) -> None:
|
|
31
|
+
self.modules = modules
|
|
32
|
+
|
|
33
|
+
def __call__(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
34
|
+
module_count = len(self.modules)
|
|
35
|
+
for idx, module in enumerate(self.modules, 1):
|
|
36
|
+
print(f"[{idx}/{module_count}] Pipeline: {module.__class__.__name__}")
|
|
37
|
+
dataset = module(dataset)
|
|
38
|
+
return dataset
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
import spacy
|
|
5
|
+
from radlab_data.preprocessing.dataset import (
|
|
6
|
+
ClassificationDataset,
|
|
7
|
+
Dataset,
|
|
8
|
+
SequenceLabellingDataset,
|
|
9
|
+
SequenceToSequenceDataset,
|
|
10
|
+
TextClassificationDataset,
|
|
11
|
+
)
|
|
12
|
+
from radlab_data.preprocessing.pipeline import PipelineElementBase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SplitLabels(PipelineElementBase):
|
|
16
|
+
def __init__(self) -> None:
|
|
17
|
+
super().__init__()
|
|
18
|
+
|
|
19
|
+
def _process_data(
|
|
20
|
+
self, dataset: SequenceToSequenceDataset
|
|
21
|
+
) -> SequenceToSequenceDataset:
|
|
22
|
+
new_data = []
|
|
23
|
+
for elem in dataset._raw_data:
|
|
24
|
+
new_elems = []
|
|
25
|
+
labels = elem[dataset.class_label_field_name]
|
|
26
|
+
if len(labels) > 1:
|
|
27
|
+
for label in labels:
|
|
28
|
+
new_elem = copy.deepcopy(elem)
|
|
29
|
+
new_elem[dataset.class_label_field_name] = [label]
|
|
30
|
+
new_elems.append(new_elem)
|
|
31
|
+
else:
|
|
32
|
+
new_elems.append(elem)
|
|
33
|
+
new_data.extend(new_elems)
|
|
34
|
+
dataset._raw_data = new_data
|
|
35
|
+
return dataset
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class RemoveUnlabelledData(PipelineElementBase):
|
|
39
|
+
def __init__(self) -> None:
|
|
40
|
+
super().__init__()
|
|
41
|
+
|
|
42
|
+
def _process_data(self, dataset: Dataset) -> Dataset:
|
|
43
|
+
if isinstance(dataset, ClassificationDataset):
|
|
44
|
+
dataset._raw_data = list(
|
|
45
|
+
elem
|
|
46
|
+
for elem in dataset.raw_data
|
|
47
|
+
if elem["metadata"][dataset.class_label_field_name]
|
|
48
|
+
)
|
|
49
|
+
return dataset
|
|
50
|
+
elif isinstance(dataset, SequenceToSequenceDataset):
|
|
51
|
+
dataset._raw_data = list(
|
|
52
|
+
elem
|
|
53
|
+
for elem in dataset.raw_data
|
|
54
|
+
if elem[dataset.class_label_field_name]
|
|
55
|
+
)
|
|
56
|
+
return dataset
|
|
57
|
+
else:
|
|
58
|
+
raise TypeError(
|
|
59
|
+
f"{self.__class__.__name__} does not support dataset of type "
|
|
60
|
+
f"{dataset.__class__.__name__}."
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RemoveDuplicates(PipelineElementBase):
|
|
65
|
+
def __init__(self) -> None:
|
|
66
|
+
super().__init__()
|
|
67
|
+
|
|
68
|
+
def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
69
|
+
exists = set()
|
|
70
|
+
|
|
71
|
+
def generator():
|
|
72
|
+
removed = 0
|
|
73
|
+
for elem in dataset.raw_data:
|
|
74
|
+
representation_hash = hash(elem["text"])
|
|
75
|
+
if representation_hash not in exists:
|
|
76
|
+
exists.add(representation_hash)
|
|
77
|
+
yield elem
|
|
78
|
+
else:
|
|
79
|
+
removed += 1
|
|
80
|
+
print(f"Removed {removed} duplicates.")
|
|
81
|
+
|
|
82
|
+
dataset._raw_data = list(generator())
|
|
83
|
+
return dataset
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class SplitSentences(PipelineElementBase):
|
|
87
|
+
ALLOWED_DATASET_TYPES: tuple = (
|
|
88
|
+
SequenceLabellingDataset,
|
|
89
|
+
TextClassificationDataset,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def __init__(self) -> None:
|
|
93
|
+
self.nlp = spacy.load("pl_core_news_lg")
|
|
94
|
+
self.ACTION_MAPPING: dict = {
|
|
95
|
+
SequenceLabellingDataset: self._action_sequence_labelling_dataset,
|
|
96
|
+
TextClassificationDataset: self._action_text_classification_dataset,
|
|
97
|
+
}
|
|
98
|
+
super().__init__()
|
|
99
|
+
|
|
100
|
+
def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
101
|
+
action = self.ACTION_MAPPING[type(dataset)]
|
|
102
|
+
new_data = []
|
|
103
|
+
for elem in dataset.raw_data:
|
|
104
|
+
output = action(dataset, elem)
|
|
105
|
+
new_data.extend(output)
|
|
106
|
+
dataset.raw_data = new_data
|
|
107
|
+
return dataset
|
|
108
|
+
|
|
109
|
+
def _action_sequence_labelling_dataset(self, dataset, element) -> list:
|
|
110
|
+
text = element["text"]
|
|
111
|
+
labels = element["metadata"][dataset.class_label_field_name]
|
|
112
|
+
sentences = self.nlp(text).sents
|
|
113
|
+
output = []
|
|
114
|
+
|
|
115
|
+
for sentence in sentences:
|
|
116
|
+
sentence_start_char = sentence.start_char
|
|
117
|
+
sentence_last_char = sentence.end_char - 1
|
|
118
|
+
sentence_labels = []
|
|
119
|
+
sentence_text = sentence.text
|
|
120
|
+
|
|
121
|
+
# text stripping
|
|
122
|
+
original_text_len = len(sentence_text)
|
|
123
|
+
sentence_text = sentence_text.lstrip()
|
|
124
|
+
if not sentence_text:
|
|
125
|
+
continue
|
|
126
|
+
chars_lost = original_text_len - len(sentence_text)
|
|
127
|
+
sentence_start_char += chars_lost
|
|
128
|
+
|
|
129
|
+
original_text_len = len(sentence_text)
|
|
130
|
+
sentence_text = sentence_text.rstrip()
|
|
131
|
+
if not sentence_text:
|
|
132
|
+
continue
|
|
133
|
+
chars_lost = original_text_len - len(sentence_text)
|
|
134
|
+
sentence_last_char -= chars_lost
|
|
135
|
+
|
|
136
|
+
for label in labels:
|
|
137
|
+
label_start_char, label_end_char, *data = label
|
|
138
|
+
label_last_char = label_end_char - 1
|
|
139
|
+
|
|
140
|
+
if (
|
|
141
|
+
label_start_char > sentence_last_char
|
|
142
|
+
or label_last_char < sentence_start_char
|
|
143
|
+
):
|
|
144
|
+
continue
|
|
145
|
+
intersection_start = max(sentence_start_char, label_start_char)
|
|
146
|
+
intersection_end = min(sentence_last_char, label_last_char)
|
|
147
|
+
|
|
148
|
+
# shift to local coordinates
|
|
149
|
+
intersection_start -= sentence_start_char
|
|
150
|
+
intersection_end -= sentence_start_char
|
|
151
|
+
|
|
152
|
+
new_label_data = [intersection_start, intersection_end + 1, *data]
|
|
153
|
+
sentence_labels.append(new_label_data)
|
|
154
|
+
|
|
155
|
+
new_elem = copy.deepcopy(element)
|
|
156
|
+
new_elem["text"] = sentence_text
|
|
157
|
+
new_elem["metadata"][dataset.class_label_field_name] = sentence_labels
|
|
158
|
+
output.append(new_elem)
|
|
159
|
+
return output
|
|
160
|
+
|
|
161
|
+
def _action_text_classification_dataset(self, dataset, element) -> list:
|
|
162
|
+
text = element["text"]
|
|
163
|
+
sentences = [x.text.strip() for x in self.nlp(text).sents]
|
|
164
|
+
output = []
|
|
165
|
+
for sent in sentences:
|
|
166
|
+
if not sent:
|
|
167
|
+
# remove empty sentences
|
|
168
|
+
continue
|
|
169
|
+
new_elem = copy.deepcopy(element)
|
|
170
|
+
new_elem["text"] = sent
|
|
171
|
+
output.append(new_elem)
|
|
172
|
+
return output
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class RemoveLongIOBAnnotation(PipelineElementBase):
|
|
176
|
+
def __init__(self, ans_factor: float = 0.85) -> None:
|
|
177
|
+
"""
|
|
178
|
+
:param ans_factor: Annotation to sentence factor
|
|
179
|
+
"""
|
|
180
|
+
super().__init__()
|
|
181
|
+
self.anno_sent_perc_len = ans_factor
|
|
182
|
+
|
|
183
|
+
def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
184
|
+
new_dataset = []
|
|
185
|
+
for elem in dataset.raw_data:
|
|
186
|
+
if not len(elem["metadata"][dataset.class_label_field_name]):
|
|
187
|
+
continue
|
|
188
|
+
new_annos = []
|
|
189
|
+
text_str_len = len(elem["text"])
|
|
190
|
+
for annotation in elem["metadata"][dataset.class_label_field_name]:
|
|
191
|
+
anno_len = annotation[1] - annotation[0]
|
|
192
|
+
if anno_len / text_str_len >= self.anno_sent_perc_len:
|
|
193
|
+
continue
|
|
194
|
+
new_annos.append(annotation)
|
|
195
|
+
elem["metadata"][dataset.class_label_field_name] = new_annos
|
|
196
|
+
new_dataset.append(elem)
|
|
197
|
+
dataset.raw_data = new_dataset
|
|
198
|
+
return dataset
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class AlignAnnotationToWordBoundaries(PipelineElementBase):
|
|
202
|
+
def __init__(self, align_begin: bool = True, align_end: bool = True) -> None:
|
|
203
|
+
"""
|
|
204
|
+
:param ans_factor: Annotation to sentence factor
|
|
205
|
+
"""
|
|
206
|
+
super().__init__()
|
|
207
|
+
self._align_begin = align_begin
|
|
208
|
+
self._align_end = align_end
|
|
209
|
+
|
|
210
|
+
def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
|
|
211
|
+
# No alignment is needed!
|
|
212
|
+
if not self._align_begin and not self._align_end:
|
|
213
|
+
return dataset
|
|
214
|
+
new_dataset = []
|
|
215
|
+
for elem in dataset.raw_data:
|
|
216
|
+
if not len(elem["metadata"][dataset.class_label_field_name]):
|
|
217
|
+
continue
|
|
218
|
+
new_annotations = []
|
|
219
|
+
text_str = elem["text"]
|
|
220
|
+
for annotation in elem["metadata"][dataset.class_label_field_name]:
|
|
221
|
+
# TODO: Tutaj jest sytuacja, że po "odsiewaniu" labelek
|
|
222
|
+
# TODO: zrobione jest obejście na odisanie przykładów, bez labelki
|
|
223
|
+
# TODO: czyli bez kompletu beg_pos, end_pos, atag_b, atag_e
|
|
224
|
+
if len(annotation) < 4:
|
|
225
|
+
continue
|
|
226
|
+
a_beg_pos, a_end_pos, *data = annotation
|
|
227
|
+
a_last_pos = a_end_pos - 1
|
|
228
|
+
text_len = len(text_str)
|
|
229
|
+
|
|
230
|
+
if not ((0 <= a_beg_pos < text_len) and (0 <= a_last_pos < text_len)):
|
|
231
|
+
warnings.warn(
|
|
232
|
+
f"Annotation boundaries incorrect ({annotation = }, {text_len = }, {text_str[:64] = }). Skipping annotation."
|
|
233
|
+
)
|
|
234
|
+
continue
|
|
235
|
+
|
|
236
|
+
if self._align_begin:
|
|
237
|
+
a_beg_pos = self._align_begin_annotation_position(
|
|
238
|
+
a_beg_pos, text_str, strategy="most_word"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
if self._align_end:
|
|
242
|
+
a_last_pos = self._align_end_annotation_position(
|
|
243
|
+
a_last_pos, text_str, strategy="most_word"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
a_end_pos = a_last_pos + 1
|
|
247
|
+
new_annotations.append([a_beg_pos, a_end_pos, *data])
|
|
248
|
+
elem["metadata"][dataset.class_label_field_name] = new_annotations
|
|
249
|
+
new_dataset.append(elem)
|
|
250
|
+
dataset.raw_data = new_dataset
|
|
251
|
+
return dataset
|
|
252
|
+
|
|
253
|
+
def _align_begin_annotation_position(
|
|
254
|
+
self, position: int, text_str: str, strategy: str = None
|
|
255
|
+
):
|
|
256
|
+
if strategy == "most_word":
|
|
257
|
+
return self._align_to_most_word(position, text_str, direction="right")
|
|
258
|
+
|
|
259
|
+
for corr_pos in range(position, -1, -1):
|
|
260
|
+
if text_str[corr_pos].isspace():
|
|
261
|
+
return corr_pos
|
|
262
|
+
return 0
|
|
263
|
+
|
|
264
|
+
def _align_end_annotation_position(
|
|
265
|
+
self, position: int, text_str: str, strategy: str = None
|
|
266
|
+
):
|
|
267
|
+
"""
|
|
268
|
+
Depends on given strategy annotation will be aligned to mostly annotated
|
|
269
|
+
word on begin/end position of annotation
|
|
270
|
+
|
|
271
|
+
:param position:
|
|
272
|
+
:param text_str:
|
|
273
|
+
:param strategy:
|
|
274
|
+
:return:
|
|
275
|
+
"""
|
|
276
|
+
if strategy == "most_word":
|
|
277
|
+
return self._align_to_most_word(position, text_str, direction="left")
|
|
278
|
+
|
|
279
|
+
for corr_pos in range(position, len(text_str)):
|
|
280
|
+
if text_str[corr_pos].isspace():
|
|
281
|
+
return corr_pos
|
|
282
|
+
return len(text_str) - 1
|
|
283
|
+
|
|
284
|
+
def _align_to_most_word(self, position: int, text_str: str, direction: str) -> int:
|
|
285
|
+
"""
|
|
286
|
+
Align annotation to most annotated part of word.
|
|
287
|
+
Depends on direction (left, right) begin position will be moved to
|
|
288
|
+
begin/end position of word in case when most word will (or will not)
|
|
289
|
+
be annotated.
|
|
290
|
+
|
|
291
|
+
:param position: Starting position
|
|
292
|
+
:param text_str: Text to move annotation
|
|
293
|
+
:param direction: l/r if dir
|
|
294
|
+
:return: Position of annotation point
|
|
295
|
+
"""
|
|
296
|
+
|
|
297
|
+
direction_mapping = {"left": 0, "l": 0, "right": 1, "r": 1}
|
|
298
|
+
if direction.lower() not in direction_mapping.keys():
|
|
299
|
+
raise Exception("Direction have to be one of: {left, right, l, r}")
|
|
300
|
+
|
|
301
|
+
assert 0 <= position < len(text_str)
|
|
302
|
+
if text_str[position].isspace() or position == 0:
|
|
303
|
+
return position
|
|
304
|
+
|
|
305
|
+
binary_direction = direction_mapping[direction.lower()]
|
|
306
|
+
first_space_left = position
|
|
307
|
+
first_space_right = position
|
|
308
|
+
|
|
309
|
+
while not text_str[first_space_left].isspace():
|
|
310
|
+
first_space_left -= 1
|
|
311
|
+
if first_space_left < 0:
|
|
312
|
+
first_space_left = 0
|
|
313
|
+
break
|
|
314
|
+
while not text_str[first_space_right].isspace():
|
|
315
|
+
first_space_right += 1
|
|
316
|
+
if first_space_right >= len(text_str):
|
|
317
|
+
first_space_right = len(text_str) - 1
|
|
318
|
+
break
|
|
319
|
+
|
|
320
|
+
chars_to_left = position - first_space_left
|
|
321
|
+
chars_to_right = first_space_right - position
|
|
322
|
+
|
|
323
|
+
return_position = 0
|
|
324
|
+
if binary_direction == 0: # left
|
|
325
|
+
if chars_to_left >= chars_to_right:
|
|
326
|
+
return_position = (
|
|
327
|
+
first_space_right - 1
|
|
328
|
+
) # -1/+1 because we want the word before/after space, not exactly the space
|
|
329
|
+
else:
|
|
330
|
+
return_position = first_space_left + 1
|
|
331
|
+
else: # right
|
|
332
|
+
if chars_to_right >= chars_to_left:
|
|
333
|
+
return_position = first_space_left + 1
|
|
334
|
+
else:
|
|
335
|
+
return_position = first_space_right - 1
|
|
336
|
+
|
|
337
|
+
if return_position < 0:
|
|
338
|
+
return 0
|
|
339
|
+
elif return_position >= len(text_str):
|
|
340
|
+
return len(text_str) - 1
|
|
341
|
+
else:
|
|
342
|
+
return return_position
|
|
File without changes
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
|
|
3
|
+
from radlab_data.text.loaders.pdf import PDFLoader
|
|
4
|
+
from radlab_data.text.loaders.txt import TXTLoader
|
|
5
|
+
from radlab_data.text.loaders.markdown import MarkdownLoader
|
|
6
|
+
from radlab_data.text.loaders.docx import DOCXLoader
|
|
7
|
+
from radlab_data.text.loaders.html import HTMLLoader
|
|
8
|
+
from radlab_data.text.loaders.input_txt import InputTextLoader
|
|
9
|
+
|
|
10
|
+
from radlab_data.text.loaders.config import (
|
|
11
|
+
AVAILABLE_FILE_EXTENSIONS,
|
|
12
|
+
AVAILABLE_FILE_EXTENSIONS_STR,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Document:
|
|
17
|
+
"""
|
|
18
|
+
Definition of any loaded document (pdf, docx, txt, md, html). It is general definition
|
|
19
|
+
which may be used to read content from any file with supported extensions.
|
|
20
|
+
|
|
21
|
+
The full list of supported extension to read files, are defined into
|
|
22
|
+
radlab_data.text.loaders.config.AVAILABLE_FILE_EXTENSIONS
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
EXTENSION_TO_LOADER = {
|
|
26
|
+
"txt": TXTLoader,
|
|
27
|
+
"md": MarkdownLoader,
|
|
28
|
+
"pdf": PDFLoader,
|
|
29
|
+
"docx": DOCXLoader,
|
|
30
|
+
"html": HTMLLoader,
|
|
31
|
+
"input_text": InputTextLoader,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
file_path: str,
|
|
37
|
+
relative_file_path: str = None,
|
|
38
|
+
file_name: str = None,
|
|
39
|
+
prepare_proper_pages: bool = False,
|
|
40
|
+
merge_document_pages: bool = False,
|
|
41
|
+
clear_texts: bool = False,
|
|
42
|
+
use_text_denoiser: bool = False,
|
|
43
|
+
max_tokens_in_chunk: int = None,
|
|
44
|
+
number_of_overlap_tokens: int = None,
|
|
45
|
+
check_text_language: bool = False,
|
|
46
|
+
document_category: str = None,
|
|
47
|
+
document_content: str = None,
|
|
48
|
+
) -> None:
|
|
49
|
+
"""
|
|
50
|
+
:param file_path: Path to file to read
|
|
51
|
+
:param relative_file_path: Relative path to file
|
|
52
|
+
:param file_name: Name of read file
|
|
53
|
+
:param prepare_proper_pages: IF set to True, then all texts/sentences
|
|
54
|
+
from the same page will be merged to single page.
|
|
55
|
+
:param merge_document_pages: If set to True then all pages from document
|
|
56
|
+
will be merged to single page, then document will contain single page.
|
|
57
|
+
:param clear_texts: If set to True, then each text will be
|
|
58
|
+
:param max_tokens_in_chunk: If given then each text will be split
|
|
59
|
+
to chunks with given max number of tokens in single chunk
|
|
60
|
+
:param number_of_overlap_tokens: If given then each generated chunk
|
|
61
|
+
will contain number_of_overlap_tokens from the previous chunk.
|
|
62
|
+
:param check_text_language: If options is set to True, then language
|
|
63
|
+
of each text will be checked and stored to metadata
|
|
64
|
+
:param document_category: Category of document
|
|
65
|
+
:param document_content: Content of document
|
|
66
|
+
"""
|
|
67
|
+
self.file_path = file_path
|
|
68
|
+
self.relative_file_path = relative_file_path
|
|
69
|
+
self.file_name = file_name
|
|
70
|
+
self.file_type = None
|
|
71
|
+
|
|
72
|
+
# Loaded document
|
|
73
|
+
self._document = None
|
|
74
|
+
self._document_as_dict = None
|
|
75
|
+
|
|
76
|
+
# Some global options passed to each document loader
|
|
77
|
+
self.global_reader_options = {
|
|
78
|
+
"merge_to_proper_pages": prepare_proper_pages,
|
|
79
|
+
"merge_document_pages": merge_document_pages,
|
|
80
|
+
"clear_texts": clear_texts,
|
|
81
|
+
"use_text_denoiser": use_text_denoiser,
|
|
82
|
+
"check_text_language": check_text_language,
|
|
83
|
+
"max_tokens_in_chunk": max_tokens_in_chunk,
|
|
84
|
+
"number_of_overlap_tokens": number_of_overlap_tokens,
|
|
85
|
+
"relative_file_path": relative_file_path,
|
|
86
|
+
"document_category": document_category,
|
|
87
|
+
"document_content": document_content,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
def __iter__(self):
|
|
91
|
+
"""
|
|
92
|
+
Added iterator over document pages
|
|
93
|
+
:return: Iterator over document pages
|
|
94
|
+
"""
|
|
95
|
+
for p in self._document:
|
|
96
|
+
yield p
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def as_dict(self):
|
|
100
|
+
return self._document_as_dict
|
|
101
|
+
|
|
102
|
+
def load(self):
|
|
103
|
+
"""
|
|
104
|
+
Load the document. Prepare document loader and load the document.
|
|
105
|
+
:return: loaded document
|
|
106
|
+
"""
|
|
107
|
+
assert self.file_path is not None
|
|
108
|
+
if self.file_name is None:
|
|
109
|
+
self.file_name = os.path.basename(self.file_path)
|
|
110
|
+
assert self.file_name is not None and len(self.file_name) > 0
|
|
111
|
+
|
|
112
|
+
self.file_type = self._resolve_file_type()
|
|
113
|
+
document_loader = self.EXTENSION_TO_LOADER[self.file_type](
|
|
114
|
+
self.file_path, options=self.global_reader_options
|
|
115
|
+
)
|
|
116
|
+
self._document = document_loader.load()
|
|
117
|
+
|
|
118
|
+
# Preprocess document if is needed
|
|
119
|
+
document_loader.merge_all_pages()
|
|
120
|
+
document_loader.clear_doc_pages()
|
|
121
|
+
document_loader.split_to_chunks()
|
|
122
|
+
document_loader.check_language()
|
|
123
|
+
|
|
124
|
+
# convert preprocessed document as dict
|
|
125
|
+
self._document_as_dict = document_loader.as_dict()
|
|
126
|
+
|
|
127
|
+
return self._document
|
|
128
|
+
|
|
129
|
+
def _resolve_file_type(self):
|
|
130
|
+
file_extension = self.file_path.split(".")[-1].lower().strip()
|
|
131
|
+
if file_extension not in AVAILABLE_FILE_EXTENSIONS:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"Unsupported file extension {file_extension}. "
|
|
134
|
+
f"Available extensions are: {AVAILABLE_FILE_EXTENSIONS_STR}"
|
|
135
|
+
)
|
|
136
|
+
return file_extension
|
|
File without changes
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class DocumentPage:
|
|
6
|
+
"""
|
|
7
|
+
Single page (fragment) of a loaded document.
|
|
8
|
+
|
|
9
|
+
Objects of this class are created by document loaders and consumed by
|
|
10
|
+
`LoaderI` (merging, clearing, chunking, converting to dictionary).
|
|
11
|
+
Both fields are mutable in place, pages are compared by their content
|
|
12
|
+
and metadata, and instances can be pickled and deep copied, so they can
|
|
13
|
+
be safely used with multiprocessing.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
page_content: str = ""
|
|
17
|
+
metadata: dict = field(default_factory=dict)
|
|
18
|
+
|
|
19
|
+
def to_dict(self) -> dict:
|
|
20
|
+
"""
|
|
21
|
+
Return page content and metadata as a plain dictionary
|
|
22
|
+
:return: Dictionary with `page_content` and `metadata` keys
|
|
23
|
+
"""
|
|
24
|
+
return {
|
|
25
|
+
"page_content": self.page_content,
|
|
26
|
+
"metadata": self.metadata,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Encodings used to read a text file when it cannot be decoded with the
|
|
31
|
+
# requested one. Single byte encodings which decode every byte sequence
|
|
32
|
+
# (like `latin-1`) are intentionally not listed here, because they would
|
|
33
|
+
# silently accept garbage instead of reporting a problem.
|
|
34
|
+
FALLBACK_ENCODINGS = ["utf-8-sig", "cp1250", "cp1252"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_text_file(
|
|
38
|
+
filepath: str, encoding: str = "utf-8", autodetect_encoding: bool = True
|
|
39
|
+
) -> str:
|
|
40
|
+
"""
|
|
41
|
+
Read the whole text file into a single string.
|
|
42
|
+
|
|
43
|
+
At first the file is read with the given `encoding`. When it fails and
|
|
44
|
+
`autodetect_encoding` is set to True, encodings from `FALLBACK_ENCODINGS`
|
|
45
|
+
are tried one after another.
|
|
46
|
+
|
|
47
|
+
:param filepath: Path to the text file to read
|
|
48
|
+
:param encoding: Encoding used to read the file
|
|
49
|
+
:param autodetect_encoding: If True, fallback encodings will be tried
|
|
50
|
+
when the file cannot be decoded with the given `encoding`
|
|
51
|
+
:return: Content of the file
|
|
52
|
+
"""
|
|
53
|
+
candidate_encodings = [encoding]
|
|
54
|
+
if autodetect_encoding:
|
|
55
|
+
candidate_encodings += [enc for enc in FALLBACK_ENCODINGS if enc != encoding]
|
|
56
|
+
|
|
57
|
+
decoding_error = None
|
|
58
|
+
for candidate_encoding in candidate_encodings:
|
|
59
|
+
try:
|
|
60
|
+
with open(filepath, "r", encoding=candidate_encoding) as text_file:
|
|
61
|
+
return text_file.read()
|
|
62
|
+
except UnicodeDecodeError as error:
|
|
63
|
+
decoding_error = error
|
|
64
|
+
|
|
65
|
+
raise RuntimeError(f"Error loading {filepath}") from decoding_error
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import docx
|
|
2
|
+
|
|
3
|
+
from radlab_data.text.loaders.base import DocumentPage
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from radlab_data.text.loaders.loader_i import LoaderI
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DOCXLoader(LoaderI):
|
|
10
|
+
def __init__(self, filepath: str, options: dict = None) -> None:
|
|
11
|
+
"""
|
|
12
|
+
:param filepath: Path to docx file
|
|
13
|
+
"""
|
|
14
|
+
if options is None:
|
|
15
|
+
options = {}
|
|
16
|
+
|
|
17
|
+
LoaderI.__init__(self, filepath=filepath, options=options)
|
|
18
|
+
self.document = docx.Document(filepath)
|
|
19
|
+
|
|
20
|
+
def load(self) -> list:
|
|
21
|
+
"""
|
|
22
|
+
Loads content from docx.
|
|
23
|
+
:return:
|
|
24
|
+
"""
|
|
25
|
+
if len(self.doc_pages):
|
|
26
|
+
return self.doc_pages
|
|
27
|
+
|
|
28
|
+
page_text = ""
|
|
29
|
+
page_number = 0
|
|
30
|
+
is_new_page = False
|
|
31
|
+
proper_pages = self.options.get("merge_to_proper_pages", False)
|
|
32
|
+
for p in self.document.paragraphs:
|
|
33
|
+
if p.text is None or not len(p.text.strip()):
|
|
34
|
+
continue
|
|
35
|
+
|
|
36
|
+
is_new_page = self._is_new_page(p)
|
|
37
|
+
if is_new_page:
|
|
38
|
+
page_number += 1
|
|
39
|
+
|
|
40
|
+
if proper_pages:
|
|
41
|
+
if is_new_page:
|
|
42
|
+
lgc_d = DocumentPage(
|
|
43
|
+
page_content=page_text,
|
|
44
|
+
metadata={"source": self.filepath, "page": page_number},
|
|
45
|
+
)
|
|
46
|
+
self.doc_pages.append(lgc_d)
|
|
47
|
+
page_text = p.text.strip()
|
|
48
|
+
else:
|
|
49
|
+
page_text += "\n" + p.text.strip()
|
|
50
|
+
else:
|
|
51
|
+
page_text = p.text.strip()
|
|
52
|
+
lgc_d = DocumentPage(
|
|
53
|
+
page_content=page_text,
|
|
54
|
+
metadata={"source": self.filepath, "page": page_number},
|
|
55
|
+
)
|
|
56
|
+
self.doc_pages.append(lgc_d)
|
|
57
|
+
if not is_new_page and len(page_text.strip()):
|
|
58
|
+
page_number += 1
|
|
59
|
+
lgc_d = DocumentPage(
|
|
60
|
+
page_content=page_text,
|
|
61
|
+
metadata={"source": self.filepath, "page": page_number},
|
|
62
|
+
)
|
|
63
|
+
self.doc_pages.append(lgc_d)
|
|
64
|
+
|
|
65
|
+
table_number = 0
|
|
66
|
+
for table in self.document.tables:
|
|
67
|
+
table_number += 1
|
|
68
|
+
|
|
69
|
+
for row_number, row in enumerate(table.rows):
|
|
70
|
+
for column_number, cell in enumerate(row.cells):
|
|
71
|
+
if cell.text is None or not len(cell.text.strip()):
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
cell_text = cell.text.strip()
|
|
75
|
+
lgc_d = DocumentPage(
|
|
76
|
+
page_content=cell_text,
|
|
77
|
+
metadata={
|
|
78
|
+
"source": self.filepath,
|
|
79
|
+
"table": table_number,
|
|
80
|
+
"row": row_number,
|
|
81
|
+
"column": column_number,
|
|
82
|
+
},
|
|
83
|
+
)
|
|
84
|
+
self.doc_pages.append(lgc_d)
|
|
85
|
+
return self.doc_pages
|
|
86
|
+
|
|
87
|
+
@staticmethod
|
|
88
|
+
def _is_new_page(paragraph) -> bool:
|
|
89
|
+
is_new_page = False
|
|
90
|
+
for run in paragraph.runs:
|
|
91
|
+
# Soft and hard page wrapping
|
|
92
|
+
if "lastRenderedPageBreak" in run._element.xml or (
|
|
93
|
+
"w:br" in run._element.xml and 'type="page"' in run._element.xml
|
|
94
|
+
):
|
|
95
|
+
is_new_page = True
|
|
96
|
+
return is_new_page
|