radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,38 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import List
3
+
4
+ from radlab_data.preprocessing.dataset import ClassificationDataset
5
+
6
+
7
+ class PipelineElementBase(ABC):
8
+ ALLOWED_DATASET_TYPES: tuple = ()
9
+ ACTION_MAPPING: dict = {}
10
+
11
+ @abstractmethod
12
+ def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
13
+ pass
14
+
15
+ def __call__(self, dataset: ClassificationDataset) -> ClassificationDataset:
16
+ assert self._dataset_type_check(
17
+ dataset
18
+ ), f"{self.__class__.__name__} pipeline element does not support dataset of type {dataset.__class__.__name__}."
19
+ return self._process_data(dataset)
20
+
21
+ def _dataset_type_check(self, dataset: ClassificationDataset) -> bool:
22
+ if self.ALLOWED_DATASET_TYPES:
23
+ return isinstance(dataset, self.ALLOWED_DATASET_TYPES)
24
+ else:
25
+ # ALLOWED_DATASET_TYPES empty => any type allowed
26
+ return True
27
+
28
+
29
+ class Pipeline:
30
+ def __init__(self, modules: List[PipelineElementBase]) -> None:
31
+ self.modules = modules
32
+
33
+ def __call__(self, dataset: ClassificationDataset) -> ClassificationDataset:
34
+ module_count = len(self.modules)
35
+ for idx, module in enumerate(self.modules, 1):
36
+ print(f"[{idx}/{module_count}] Pipeline: {module.__class__.__name__}")
37
+ dataset = module(dataset)
38
+ return dataset
@@ -0,0 +1,342 @@
1
+ import copy
2
+ import warnings
3
+
4
+ import spacy
5
+ from radlab_data.preprocessing.dataset import (
6
+ ClassificationDataset,
7
+ Dataset,
8
+ SequenceLabellingDataset,
9
+ SequenceToSequenceDataset,
10
+ TextClassificationDataset,
11
+ )
12
+ from radlab_data.preprocessing.pipeline import PipelineElementBase
13
+
14
+
15
+ class SplitLabels(PipelineElementBase):
16
+ def __init__(self) -> None:
17
+ super().__init__()
18
+
19
+ def _process_data(
20
+ self, dataset: SequenceToSequenceDataset
21
+ ) -> SequenceToSequenceDataset:
22
+ new_data = []
23
+ for elem in dataset._raw_data:
24
+ new_elems = []
25
+ labels = elem[dataset.class_label_field_name]
26
+ if len(labels) > 1:
27
+ for label in labels:
28
+ new_elem = copy.deepcopy(elem)
29
+ new_elem[dataset.class_label_field_name] = [label]
30
+ new_elems.append(new_elem)
31
+ else:
32
+ new_elems.append(elem)
33
+ new_data.extend(new_elems)
34
+ dataset._raw_data = new_data
35
+ return dataset
36
+
37
+
38
+ class RemoveUnlabelledData(PipelineElementBase):
39
+ def __init__(self) -> None:
40
+ super().__init__()
41
+
42
+ def _process_data(self, dataset: Dataset) -> Dataset:
43
+ if isinstance(dataset, ClassificationDataset):
44
+ dataset._raw_data = list(
45
+ elem
46
+ for elem in dataset.raw_data
47
+ if elem["metadata"][dataset.class_label_field_name]
48
+ )
49
+ return dataset
50
+ elif isinstance(dataset, SequenceToSequenceDataset):
51
+ dataset._raw_data = list(
52
+ elem
53
+ for elem in dataset.raw_data
54
+ if elem[dataset.class_label_field_name]
55
+ )
56
+ return dataset
57
+ else:
58
+ raise TypeError(
59
+ f"{self.__class__.__name__} does not support dataset of type "
60
+ f"{dataset.__class__.__name__}."
61
+ )
62
+
63
+
64
+ class RemoveDuplicates(PipelineElementBase):
65
+ def __init__(self) -> None:
66
+ super().__init__()
67
+
68
+ def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
69
+ exists = set()
70
+
71
+ def generator():
72
+ removed = 0
73
+ for elem in dataset.raw_data:
74
+ representation_hash = hash(elem["text"])
75
+ if representation_hash not in exists:
76
+ exists.add(representation_hash)
77
+ yield elem
78
+ else:
79
+ removed += 1
80
+ print(f"Removed {removed} duplicates.")
81
+
82
+ dataset._raw_data = list(generator())
83
+ return dataset
84
+
85
+
86
+ class SplitSentences(PipelineElementBase):
87
+ ALLOWED_DATASET_TYPES: tuple = (
88
+ SequenceLabellingDataset,
89
+ TextClassificationDataset,
90
+ )
91
+
92
+ def __init__(self) -> None:
93
+ self.nlp = spacy.load("pl_core_news_lg")
94
+ self.ACTION_MAPPING: dict = {
95
+ SequenceLabellingDataset: self._action_sequence_labelling_dataset,
96
+ TextClassificationDataset: self._action_text_classification_dataset,
97
+ }
98
+ super().__init__()
99
+
100
+ def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
101
+ action = self.ACTION_MAPPING[type(dataset)]
102
+ new_data = []
103
+ for elem in dataset.raw_data:
104
+ output = action(dataset, elem)
105
+ new_data.extend(output)
106
+ dataset.raw_data = new_data
107
+ return dataset
108
+
109
+ def _action_sequence_labelling_dataset(self, dataset, element) -> list:
110
+ text = element["text"]
111
+ labels = element["metadata"][dataset.class_label_field_name]
112
+ sentences = self.nlp(text).sents
113
+ output = []
114
+
115
+ for sentence in sentences:
116
+ sentence_start_char = sentence.start_char
117
+ sentence_last_char = sentence.end_char - 1
118
+ sentence_labels = []
119
+ sentence_text = sentence.text
120
+
121
+ # text stripping
122
+ original_text_len = len(sentence_text)
123
+ sentence_text = sentence_text.lstrip()
124
+ if not sentence_text:
125
+ continue
126
+ chars_lost = original_text_len - len(sentence_text)
127
+ sentence_start_char += chars_lost
128
+
129
+ original_text_len = len(sentence_text)
130
+ sentence_text = sentence_text.rstrip()
131
+ if not sentence_text:
132
+ continue
133
+ chars_lost = original_text_len - len(sentence_text)
134
+ sentence_last_char -= chars_lost
135
+
136
+ for label in labels:
137
+ label_start_char, label_end_char, *data = label
138
+ label_last_char = label_end_char - 1
139
+
140
+ if (
141
+ label_start_char > sentence_last_char
142
+ or label_last_char < sentence_start_char
143
+ ):
144
+ continue
145
+ intersection_start = max(sentence_start_char, label_start_char)
146
+ intersection_end = min(sentence_last_char, label_last_char)
147
+
148
+ # shift to local coordinates
149
+ intersection_start -= sentence_start_char
150
+ intersection_end -= sentence_start_char
151
+
152
+ new_label_data = [intersection_start, intersection_end + 1, *data]
153
+ sentence_labels.append(new_label_data)
154
+
155
+ new_elem = copy.deepcopy(element)
156
+ new_elem["text"] = sentence_text
157
+ new_elem["metadata"][dataset.class_label_field_name] = sentence_labels
158
+ output.append(new_elem)
159
+ return output
160
+
161
+ def _action_text_classification_dataset(self, dataset, element) -> list:
162
+ text = element["text"]
163
+ sentences = [x.text.strip() for x in self.nlp(text).sents]
164
+ output = []
165
+ for sent in sentences:
166
+ if not sent:
167
+ # remove empty sentences
168
+ continue
169
+ new_elem = copy.deepcopy(element)
170
+ new_elem["text"] = sent
171
+ output.append(new_elem)
172
+ return output
173
+
174
+
175
+ class RemoveLongIOBAnnotation(PipelineElementBase):
176
+ def __init__(self, ans_factor: float = 0.85) -> None:
177
+ """
178
+ :param ans_factor: Annotation to sentence factor
179
+ """
180
+ super().__init__()
181
+ self.anno_sent_perc_len = ans_factor
182
+
183
+ def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
184
+ new_dataset = []
185
+ for elem in dataset.raw_data:
186
+ if not len(elem["metadata"][dataset.class_label_field_name]):
187
+ continue
188
+ new_annos = []
189
+ text_str_len = len(elem["text"])
190
+ for annotation in elem["metadata"][dataset.class_label_field_name]:
191
+ anno_len = annotation[1] - annotation[0]
192
+ if anno_len / text_str_len >= self.anno_sent_perc_len:
193
+ continue
194
+ new_annos.append(annotation)
195
+ elem["metadata"][dataset.class_label_field_name] = new_annos
196
+ new_dataset.append(elem)
197
+ dataset.raw_data = new_dataset
198
+ return dataset
199
+
200
+
201
+ class AlignAnnotationToWordBoundaries(PipelineElementBase):
202
+ def __init__(self, align_begin: bool = True, align_end: bool = True) -> None:
203
+ """
204
+ :param ans_factor: Annotation to sentence factor
205
+ """
206
+ super().__init__()
207
+ self._align_begin = align_begin
208
+ self._align_end = align_end
209
+
210
+ def _process_data(self, dataset: ClassificationDataset) -> ClassificationDataset:
211
+ # No alignment is needed!
212
+ if not self._align_begin and not self._align_end:
213
+ return dataset
214
+ new_dataset = []
215
+ for elem in dataset.raw_data:
216
+ if not len(elem["metadata"][dataset.class_label_field_name]):
217
+ continue
218
+ new_annotations = []
219
+ text_str = elem["text"]
220
+ for annotation in elem["metadata"][dataset.class_label_field_name]:
221
+ # TODO: Tutaj jest sytuacja, że po "odsiewaniu" labelek
222
+ # TODO: zrobione jest obejście na odisanie przykładów, bez labelki
223
+ # TODO: czyli bez kompletu beg_pos, end_pos, atag_b, atag_e
224
+ if len(annotation) < 4:
225
+ continue
226
+ a_beg_pos, a_end_pos, *data = annotation
227
+ a_last_pos = a_end_pos - 1
228
+ text_len = len(text_str)
229
+
230
+ if not ((0 <= a_beg_pos < text_len) and (0 <= a_last_pos < text_len)):
231
+ warnings.warn(
232
+ f"Annotation boundaries incorrect ({annotation = }, {text_len = }, {text_str[:64] = }). Skipping annotation."
233
+ )
234
+ continue
235
+
236
+ if self._align_begin:
237
+ a_beg_pos = self._align_begin_annotation_position(
238
+ a_beg_pos, text_str, strategy="most_word"
239
+ )
240
+
241
+ if self._align_end:
242
+ a_last_pos = self._align_end_annotation_position(
243
+ a_last_pos, text_str, strategy="most_word"
244
+ )
245
+
246
+ a_end_pos = a_last_pos + 1
247
+ new_annotations.append([a_beg_pos, a_end_pos, *data])
248
+ elem["metadata"][dataset.class_label_field_name] = new_annotations
249
+ new_dataset.append(elem)
250
+ dataset.raw_data = new_dataset
251
+ return dataset
252
+
253
+ def _align_begin_annotation_position(
254
+ self, position: int, text_str: str, strategy: str = None
255
+ ):
256
+ if strategy == "most_word":
257
+ return self._align_to_most_word(position, text_str, direction="right")
258
+
259
+ for corr_pos in range(position, -1, -1):
260
+ if text_str[corr_pos].isspace():
261
+ return corr_pos
262
+ return 0
263
+
264
+ def _align_end_annotation_position(
265
+ self, position: int, text_str: str, strategy: str = None
266
+ ):
267
+ """
268
+ Depends on given strategy annotation will be aligned to mostly annotated
269
+ word on begin/end position of annotation
270
+
271
+ :param position:
272
+ :param text_str:
273
+ :param strategy:
274
+ :return:
275
+ """
276
+ if strategy == "most_word":
277
+ return self._align_to_most_word(position, text_str, direction="left")
278
+
279
+ for corr_pos in range(position, len(text_str)):
280
+ if text_str[corr_pos].isspace():
281
+ return corr_pos
282
+ return len(text_str) - 1
283
+
284
+ def _align_to_most_word(self, position: int, text_str: str, direction: str) -> int:
285
+ """
286
+ Align annotation to most annotated part of word.
287
+ Depends on direction (left, right) begin position will be moved to
288
+ begin/end position of word in case when most word will (or will not)
289
+ be annotated.
290
+
291
+ :param position: Starting position
292
+ :param text_str: Text to move annotation
293
+ :param direction: l/r if dir
294
+ :return: Position of annotation point
295
+ """
296
+
297
+ direction_mapping = {"left": 0, "l": 0, "right": 1, "r": 1}
298
+ if direction.lower() not in direction_mapping.keys():
299
+ raise Exception("Direction have to be one of: {left, right, l, r}")
300
+
301
+ assert 0 <= position < len(text_str)
302
+ if text_str[position].isspace() or position == 0:
303
+ return position
304
+
305
+ binary_direction = direction_mapping[direction.lower()]
306
+ first_space_left = position
307
+ first_space_right = position
308
+
309
+ while not text_str[first_space_left].isspace():
310
+ first_space_left -= 1
311
+ if first_space_left < 0:
312
+ first_space_left = 0
313
+ break
314
+ while not text_str[first_space_right].isspace():
315
+ first_space_right += 1
316
+ if first_space_right >= len(text_str):
317
+ first_space_right = len(text_str) - 1
318
+ break
319
+
320
+ chars_to_left = position - first_space_left
321
+ chars_to_right = first_space_right - position
322
+
323
+ return_position = 0
324
+ if binary_direction == 0: # left
325
+ if chars_to_left >= chars_to_right:
326
+ return_position = (
327
+ first_space_right - 1
328
+ ) # -1/+1 because we want the word before/after space, not exactly the space
329
+ else:
330
+ return_position = first_space_left + 1
331
+ else: # right
332
+ if chars_to_right >= chars_to_left:
333
+ return_position = first_space_left + 1
334
+ else:
335
+ return_position = first_space_right - 1
336
+
337
+ if return_position < 0:
338
+ return 0
339
+ elif return_position >= len(text_str):
340
+ return len(text_str) - 1
341
+ else:
342
+ return return_position
File without changes
@@ -0,0 +1,136 @@
1
+ import os.path
2
+
3
+ from radlab_data.text.loaders.pdf import PDFLoader
4
+ from radlab_data.text.loaders.txt import TXTLoader
5
+ from radlab_data.text.loaders.markdown import MarkdownLoader
6
+ from radlab_data.text.loaders.docx import DOCXLoader
7
+ from radlab_data.text.loaders.html import HTMLLoader
8
+ from radlab_data.text.loaders.input_txt import InputTextLoader
9
+
10
+ from radlab_data.text.loaders.config import (
11
+ AVAILABLE_FILE_EXTENSIONS,
12
+ AVAILABLE_FILE_EXTENSIONS_STR,
13
+ )
14
+
15
+
16
+ class Document:
17
+ """
18
+ Definition of any loaded document (pdf, docx, txt, md, html). It is general definition
19
+ which may be used to read content from any file with supported extensions.
20
+
21
+ The full list of supported extension to read files, are defined into
22
+ radlab_data.text.loaders.config.AVAILABLE_FILE_EXTENSIONS
23
+ """
24
+
25
+ EXTENSION_TO_LOADER = {
26
+ "txt": TXTLoader,
27
+ "md": MarkdownLoader,
28
+ "pdf": PDFLoader,
29
+ "docx": DOCXLoader,
30
+ "html": HTMLLoader,
31
+ "input_text": InputTextLoader,
32
+ }
33
+
34
+ def __init__(
35
+ self,
36
+ file_path: str,
37
+ relative_file_path: str = None,
38
+ file_name: str = None,
39
+ prepare_proper_pages: bool = False,
40
+ merge_document_pages: bool = False,
41
+ clear_texts: bool = False,
42
+ use_text_denoiser: bool = False,
43
+ max_tokens_in_chunk: int = None,
44
+ number_of_overlap_tokens: int = None,
45
+ check_text_language: bool = False,
46
+ document_category: str = None,
47
+ document_content: str = None,
48
+ ) -> None:
49
+ """
50
+ :param file_path: Path to file to read
51
+ :param relative_file_path: Relative path to file
52
+ :param file_name: Name of read file
53
+ :param prepare_proper_pages: IF set to True, then all texts/sentences
54
+ from the same page will be merged to single page.
55
+ :param merge_document_pages: If set to True then all pages from document
56
+ will be merged to single page, then document will contain single page.
57
+ :param clear_texts: If set to True, then each text will be
58
+ :param max_tokens_in_chunk: If given then each text will be split
59
+ to chunks with given max number of tokens in single chunk
60
+ :param number_of_overlap_tokens: If given then each generated chunk
61
+ will contain number_of_overlap_tokens from the previous chunk.
62
+ :param check_text_language: If options is set to True, then language
63
+ of each text will be checked and stored to metadata
64
+ :param document_category: Category of document
65
+ :param document_content: Content of document
66
+ """
67
+ self.file_path = file_path
68
+ self.relative_file_path = relative_file_path
69
+ self.file_name = file_name
70
+ self.file_type = None
71
+
72
+ # Loaded document
73
+ self._document = None
74
+ self._document_as_dict = None
75
+
76
+ # Some global options passed to each document loader
77
+ self.global_reader_options = {
78
+ "merge_to_proper_pages": prepare_proper_pages,
79
+ "merge_document_pages": merge_document_pages,
80
+ "clear_texts": clear_texts,
81
+ "use_text_denoiser": use_text_denoiser,
82
+ "check_text_language": check_text_language,
83
+ "max_tokens_in_chunk": max_tokens_in_chunk,
84
+ "number_of_overlap_tokens": number_of_overlap_tokens,
85
+ "relative_file_path": relative_file_path,
86
+ "document_category": document_category,
87
+ "document_content": document_content,
88
+ }
89
+
90
+ def __iter__(self):
91
+ """
92
+ Added iterator over document pages
93
+ :return: Iterator over document pages
94
+ """
95
+ for p in self._document:
96
+ yield p
97
+
98
+ @property
99
+ def as_dict(self):
100
+ return self._document_as_dict
101
+
102
+ def load(self):
103
+ """
104
+ Load the document. Prepare document loader and load the document.
105
+ :return: loaded document
106
+ """
107
+ assert self.file_path is not None
108
+ if self.file_name is None:
109
+ self.file_name = os.path.basename(self.file_path)
110
+ assert self.file_name is not None and len(self.file_name) > 0
111
+
112
+ self.file_type = self._resolve_file_type()
113
+ document_loader = self.EXTENSION_TO_LOADER[self.file_type](
114
+ self.file_path, options=self.global_reader_options
115
+ )
116
+ self._document = document_loader.load()
117
+
118
+ # Preprocess document if is needed
119
+ document_loader.merge_all_pages()
120
+ document_loader.clear_doc_pages()
121
+ document_loader.split_to_chunks()
122
+ document_loader.check_language()
123
+
124
+ # convert preprocessed document as dict
125
+ self._document_as_dict = document_loader.as_dict()
126
+
127
+ return self._document
128
+
129
+ def _resolve_file_type(self):
130
+ file_extension = self.file_path.split(".")[-1].lower().strip()
131
+ if file_extension not in AVAILABLE_FILE_EXTENSIONS:
132
+ raise ValueError(
133
+ f"Unsupported file extension {file_extension}. "
134
+ f"Available extensions are: {AVAILABLE_FILE_EXTENSIONS_STR}"
135
+ )
136
+ return file_extension
File without changes
@@ -0,0 +1,65 @@
1
+ from dataclasses import dataclass, field
2
+
3
+
4
+ @dataclass
5
+ class DocumentPage:
6
+ """
7
+ Single page (fragment) of a loaded document.
8
+
9
+ Objects of this class are created by document loaders and consumed by
10
+ `LoaderI` (merging, clearing, chunking, converting to dictionary).
11
+ Both fields are mutable in place, pages are compared by their content
12
+ and metadata, and instances can be pickled and deep copied, so they can
13
+ be safely used with multiprocessing.
14
+ """
15
+
16
+ page_content: str = ""
17
+ metadata: dict = field(default_factory=dict)
18
+
19
+ def to_dict(self) -> dict:
20
+ """
21
+ Return page content and metadata as a plain dictionary
22
+ :return: Dictionary with `page_content` and `metadata` keys
23
+ """
24
+ return {
25
+ "page_content": self.page_content,
26
+ "metadata": self.metadata,
27
+ }
28
+
29
+
30
+ # Encodings used to read a text file when it cannot be decoded with the
31
+ # requested one. Single byte encodings which decode every byte sequence
32
+ # (like `latin-1`) are intentionally not listed here, because they would
33
+ # silently accept garbage instead of reporting a problem.
34
+ FALLBACK_ENCODINGS = ["utf-8-sig", "cp1250", "cp1252"]
35
+
36
+
37
+ def read_text_file(
38
+ filepath: str, encoding: str = "utf-8", autodetect_encoding: bool = True
39
+ ) -> str:
40
+ """
41
+ Read the whole text file into a single string.
42
+
43
+ At first the file is read with the given `encoding`. When it fails and
44
+ `autodetect_encoding` is set to True, encodings from `FALLBACK_ENCODINGS`
45
+ are tried one after another.
46
+
47
+ :param filepath: Path to the text file to read
48
+ :param encoding: Encoding used to read the file
49
+ :param autodetect_encoding: If True, fallback encodings will be tried
50
+ when the file cannot be decoded with the given `encoding`
51
+ :return: Content of the file
52
+ """
53
+ candidate_encodings = [encoding]
54
+ if autodetect_encoding:
55
+ candidate_encodings += [enc for enc in FALLBACK_ENCODINGS if enc != encoding]
56
+
57
+ decoding_error = None
58
+ for candidate_encoding in candidate_encodings:
59
+ try:
60
+ with open(filepath, "r", encoding=candidate_encoding) as text_file:
61
+ return text_file.read()
62
+ except UnicodeDecodeError as error:
63
+ decoding_error = error
64
+
65
+ raise RuntimeError(f"Error loading {filepath}") from decoding_error
@@ -0,0 +1,4 @@
1
+ TOKENIZER_PATH = "radlab/polish-fast-tokenizer"
2
+
3
+ AVAILABLE_FILE_EXTENSIONS = ["txt", "md", "pdf", "docx", "html", "input_text"]
4
+ AVAILABLE_FILE_EXTENSIONS_STR = ", ".join(AVAILABLE_FILE_EXTENSIONS)
@@ -0,0 +1,96 @@
1
+ import docx
2
+
3
+ from radlab_data.text.loaders.base import DocumentPage
4
+
5
+
6
+ from radlab_data.text.loaders.loader_i import LoaderI
7
+
8
+
9
+ class DOCXLoader(LoaderI):
10
+ def __init__(self, filepath: str, options: dict = None) -> None:
11
+ """
12
+ :param filepath: Path to docx file
13
+ """
14
+ if options is None:
15
+ options = {}
16
+
17
+ LoaderI.__init__(self, filepath=filepath, options=options)
18
+ self.document = docx.Document(filepath)
19
+
20
+ def load(self) -> list:
21
+ """
22
+ Loads content from docx.
23
+ :return:
24
+ """
25
+ if len(self.doc_pages):
26
+ return self.doc_pages
27
+
28
+ page_text = ""
29
+ page_number = 0
30
+ is_new_page = False
31
+ proper_pages = self.options.get("merge_to_proper_pages", False)
32
+ for p in self.document.paragraphs:
33
+ if p.text is None or not len(p.text.strip()):
34
+ continue
35
+
36
+ is_new_page = self._is_new_page(p)
37
+ if is_new_page:
38
+ page_number += 1
39
+
40
+ if proper_pages:
41
+ if is_new_page:
42
+ lgc_d = DocumentPage(
43
+ page_content=page_text,
44
+ metadata={"source": self.filepath, "page": page_number},
45
+ )
46
+ self.doc_pages.append(lgc_d)
47
+ page_text = p.text.strip()
48
+ else:
49
+ page_text += "\n" + p.text.strip()
50
+ else:
51
+ page_text = p.text.strip()
52
+ lgc_d = DocumentPage(
53
+ page_content=page_text,
54
+ metadata={"source": self.filepath, "page": page_number},
55
+ )
56
+ self.doc_pages.append(lgc_d)
57
+ if not is_new_page and len(page_text.strip()):
58
+ page_number += 1
59
+ lgc_d = DocumentPage(
60
+ page_content=page_text,
61
+ metadata={"source": self.filepath, "page": page_number},
62
+ )
63
+ self.doc_pages.append(lgc_d)
64
+
65
+ table_number = 0
66
+ for table in self.document.tables:
67
+ table_number += 1
68
+
69
+ for row_number, row in enumerate(table.rows):
70
+ for column_number, cell in enumerate(row.cells):
71
+ if cell.text is None or not len(cell.text.strip()):
72
+ continue
73
+
74
+ cell_text = cell.text.strip()
75
+ lgc_d = DocumentPage(
76
+ page_content=cell_text,
77
+ metadata={
78
+ "source": self.filepath,
79
+ "table": table_number,
80
+ "row": row_number,
81
+ "column": column_number,
82
+ },
83
+ )
84
+ self.doc_pages.append(lgc_d)
85
+ return self.doc_pages
86
+
87
+ @staticmethod
88
+ def _is_new_page(paragraph) -> bool:
89
+ is_new_page = False
90
+ for run in paragraph.runs:
91
+ # Soft and hard page wrapping
92
+ if "lastRenderedPageBreak" in run._element.xml or (
93
+ "w:br" in run._element.xml and 'type="page"' in run._element.xml
94
+ ):
95
+ is_new_page = True
96
+ return is_new_page