radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from radlab_data.text.loaders.base import DocumentPage, read_text_file
|
|
2
|
+
from radlab_data.text.loaders.loader_i import LoaderI
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TXTLoader(LoaderI):
|
|
6
|
+
"""
|
|
7
|
+
Loader for plain text files. The whole file is loaded as a single page.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
filepath: str,
|
|
13
|
+
options: dict = None,
|
|
14
|
+
encoding: str = "utf-8",
|
|
15
|
+
autodetect_encoding: bool = True,
|
|
16
|
+
) -> None:
|
|
17
|
+
"""
|
|
18
|
+
:param filepath: Path to text file
|
|
19
|
+
:param options: Options shared with other loaders (see `LoaderI`)
|
|
20
|
+
:param encoding: Encoding used to read the file
|
|
21
|
+
:param autodetect_encoding: If the file cannot be decoded with
|
|
22
|
+
`encoding`, encodings from `FALLBACK_ENCODINGS` are tried
|
|
23
|
+
"""
|
|
24
|
+
LoaderI.__init__(self, filepath=filepath, options=options)
|
|
25
|
+
self.encoding = encoding
|
|
26
|
+
self.autodetect_encoding = autodetect_encoding
|
|
27
|
+
|
|
28
|
+
def load(self) -> list:
|
|
29
|
+
"""
|
|
30
|
+
Load the text file.
|
|
31
|
+
:return: List with the single document page holding the file content
|
|
32
|
+
"""
|
|
33
|
+
text = read_text_file(
|
|
34
|
+
self.filepath,
|
|
35
|
+
encoding=self.encoding,
|
|
36
|
+
autodetect_encoding=self.autodetect_encoding,
|
|
37
|
+
)
|
|
38
|
+
self.doc_pages = [
|
|
39
|
+
DocumentPage(
|
|
40
|
+
page_content=text,
|
|
41
|
+
metadata={"source": str(self.filepath)},
|
|
42
|
+
)
|
|
43
|
+
]
|
|
44
|
+
return self.doc_pages
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
from typing import Dict, List
|
|
6
|
+
from docx import Document as DOCXDocument
|
|
7
|
+
|
|
8
|
+
from radlab_data.text.processors.splitter import SentenceSplitter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PairElemsAbstract(abc.ABC):
|
|
12
|
+
FIRST_PAIR_ELEM = None
|
|
13
|
+
SECOND_PAIR_ELEM = None
|
|
14
|
+
NAME = None
|
|
15
|
+
ELEMS_TO_FILE_POSTFIX = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FirstSecondPairElems(PairElemsAbstract):
|
|
19
|
+
FIRST_PAIR_ELEM = "first"
|
|
20
|
+
SECOND_PAIR_ELEM = "second"
|
|
21
|
+
NAME = f"{FIRST_PAIR_ELEM}_{SECOND_PAIR_ELEM}"
|
|
22
|
+
ELEMS_TO_FILE_POSTFIX = {
|
|
23
|
+
FIRST_PAIR_ELEM: "_FIRST.docx",
|
|
24
|
+
SECOND_PAIR_ELEM: "_SECOND.docx",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class FirstSecondNumberPairElems(PairElemsAbstract):
|
|
29
|
+
FIRST_PAIR_ELEM = "1"
|
|
30
|
+
SECOND_PAIR_ELEM = "2"
|
|
31
|
+
NAME = f"{FIRST_PAIR_ELEM}_{SECOND_PAIR_ELEM}"
|
|
32
|
+
ELEMS_TO_FILE_POSTFIX = {
|
|
33
|
+
FIRST_PAIR_ELEM: "_1.docx",
|
|
34
|
+
SECOND_PAIR_ELEM: "_2.docx",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
AVAILABLE_POSTFIXES = {
|
|
39
|
+
FirstSecondPairElems.NAME: FirstSecondPairElems,
|
|
40
|
+
FirstSecondNumberPairElems.NAME: FirstSecondNumberPairElems,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class DirectoryPairsReader:
|
|
45
|
+
AVAILABLE_POSTFIXES_STR = ", ".join(list(AVAILABLE_POSTFIXES.keys()))
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
dir_path: str | None = None,
|
|
50
|
+
postfix_configuration_name: str = FirstSecondNumberPairElems.NAME,
|
|
51
|
+
):
|
|
52
|
+
"""
|
|
53
|
+
Set directory path to read documents
|
|
54
|
+
:param dir_path: Path to directory with documents to pair them
|
|
55
|
+
"""
|
|
56
|
+
if postfix_configuration_name not in AVAILABLE_POSTFIXES:
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"Invalid postfix configuration: {postfix_configuration_name}. "
|
|
59
|
+
f"Postfix configuration must be one of: "
|
|
60
|
+
f"{self.AVAILABLE_POSTFIXES_STR}"
|
|
61
|
+
)
|
|
62
|
+
self.postfix: PairElemsAbstract = AVAILABLE_POSTFIXES[
|
|
63
|
+
postfix_configuration_name
|
|
64
|
+
]()
|
|
65
|
+
self._postfix_configuration = postfix_configuration_name
|
|
66
|
+
|
|
67
|
+
self._documents_dir_path: str = dir_path
|
|
68
|
+
|
|
69
|
+
# Base filename to documents pair (paths to both documents)
|
|
70
|
+
self._document_pairs: Dict[str, Dict[str, str]] = {}
|
|
71
|
+
self._document_pairs_content: Dict[str, Dict[List[str], List[str]]] = {}
|
|
72
|
+
|
|
73
|
+
# base file name to dict: first/second to list of sentences
|
|
74
|
+
self._document_pairs_sentences: Dict[str, Dict[List[str], List[str]]] = {}
|
|
75
|
+
|
|
76
|
+
self._sentence_splitter = SentenceSplitter()
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def document_pairs_sentences(self):
|
|
80
|
+
return self._document_pairs_sentences
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def base_name_to_paths(self):
|
|
84
|
+
return self._document_pairs
|
|
85
|
+
|
|
86
|
+
def prepare_pairs(self, dir_path: str | None = None) -> Dict[str, Dict[str, str]]:
|
|
87
|
+
"""
|
|
88
|
+
List files from given dir_path and prepare pairs of documents
|
|
89
|
+
|
|
90
|
+
:param dir_path: Path to directory containing files (documents)
|
|
91
|
+
:return: Dictionary with paths of pairs of documents
|
|
92
|
+
"""
|
|
93
|
+
if dir_path is None:
|
|
94
|
+
dir_path = self._documents_dir_path
|
|
95
|
+
else:
|
|
96
|
+
self._documents_dir_path = dir_path
|
|
97
|
+
assert dir_path is not None, f"Directory with documents pairs is not provided"
|
|
98
|
+
|
|
99
|
+
self._document_pairs.clear()
|
|
100
|
+
self._document_pairs_content.clear()
|
|
101
|
+
for file_name in os.listdir(dir_path):
|
|
102
|
+
if not self._is_valid_filename(filename=file_name):
|
|
103
|
+
logging.info(
|
|
104
|
+
f"Skipping {file_name} - is not valid "
|
|
105
|
+
f"{self._postfix_configuration} file name"
|
|
106
|
+
)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
base_file_name = self._filename_without_postfix(file_name)
|
|
110
|
+
if base_file_name not in self._document_pairs:
|
|
111
|
+
self._document_pairs[base_file_name] = {
|
|
112
|
+
self.postfix.FIRST_PAIR_ELEM: "",
|
|
113
|
+
self.postfix.SECOND_PAIR_ELEM: "",
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
full_file_path = os.path.join(dir_path, file_name)
|
|
117
|
+
if self._is_first_filename(file_name):
|
|
118
|
+
self._document_pairs[base_file_name][
|
|
119
|
+
self.postfix.FIRST_PAIR_ELEM
|
|
120
|
+
] = full_file_path
|
|
121
|
+
else:
|
|
122
|
+
self._document_pairs[base_file_name][
|
|
123
|
+
self.postfix.SECOND_PAIR_ELEM
|
|
124
|
+
] = full_file_path
|
|
125
|
+
return self._document_pairs
|
|
126
|
+
|
|
127
|
+
def load_documents(self, unique_rows_in_cell: bool = False) -> None:
|
|
128
|
+
"""
|
|
129
|
+
Loads documents from self._documents_pairs as read documents
|
|
130
|
+
:param unique_rows_in_cell: If set to True, items from
|
|
131
|
+
single row will be deduplicated
|
|
132
|
+
:return:
|
|
133
|
+
"""
|
|
134
|
+
assert len(self._document_pairs) > 0, "No documents provided to load!"
|
|
135
|
+
|
|
136
|
+
self._document_pairs_content.clear()
|
|
137
|
+
|
|
138
|
+
with tqdm(total=len(self._document_pairs), desc="Loading files") as pbar:
|
|
139
|
+
for base_doc_name, doc_pair in self._document_pairs.items():
|
|
140
|
+
first_paragraphs, second_paragraphs = None, None
|
|
141
|
+
|
|
142
|
+
f_file = doc_pair[self.postfix.FIRST_PAIR_ELEM]
|
|
143
|
+
s_file = doc_pair[self.postfix.SECOND_PAIR_ELEM]
|
|
144
|
+
|
|
145
|
+
if f_file.endswith(".docx"):
|
|
146
|
+
first_paragraphs = self._load_docx_file_content(
|
|
147
|
+
file_path=f_file, unique_rows_in_cell=unique_rows_in_cell
|
|
148
|
+
)
|
|
149
|
+
second_paragraphs = self._load_docx_file_content(
|
|
150
|
+
file_path=s_file, unique_rows_in_cell=unique_rows_in_cell
|
|
151
|
+
)
|
|
152
|
+
else:
|
|
153
|
+
logging.info(f"Skipping file with not supported extension {f_file}")
|
|
154
|
+
|
|
155
|
+
self._document_pairs_content[base_doc_name] = {
|
|
156
|
+
self.postfix.FIRST_PAIR_ELEM: first_paragraphs,
|
|
157
|
+
self.postfix.SECOND_PAIR_ELEM: second_paragraphs,
|
|
158
|
+
}
|
|
159
|
+
pbar.update()
|
|
160
|
+
return self._split_documents_content_to_sentences()
|
|
161
|
+
|
|
162
|
+
def _split_documents_content_to_sentences(self) -> None:
|
|
163
|
+
assert (
|
|
164
|
+
len(self._document_pairs_content) > 0
|
|
165
|
+
), "No documents provided to split content to sentences!"
|
|
166
|
+
|
|
167
|
+
self._document_pairs_sentences.clear()
|
|
168
|
+
with tqdm(
|
|
169
|
+
total=len(self._document_pairs_content),
|
|
170
|
+
desc="Splitting contents to sentences",
|
|
171
|
+
) as pbar:
|
|
172
|
+
for base_doc_name, doc_pair in self._document_pairs_content.items():
|
|
173
|
+
first_sentences = self._split_paragraphs_to_sentences(
|
|
174
|
+
doc_pair[self.postfix.FIRST_PAIR_ELEM]
|
|
175
|
+
)
|
|
176
|
+
second_sentences = self._split_paragraphs_to_sentences(
|
|
177
|
+
doc_pair[self.postfix.SECOND_PAIR_ELEM]
|
|
178
|
+
)
|
|
179
|
+
if not len(second_sentences) or not len(first_sentences):
|
|
180
|
+
logging.info(
|
|
181
|
+
f"Skipping empty document {base_doc_name} "
|
|
182
|
+
f"[len(first_sentences) = {len(first_sentences)}, "
|
|
183
|
+
f"len(second_sentences) = {len(second_sentences)}]"
|
|
184
|
+
)
|
|
185
|
+
continue
|
|
186
|
+
|
|
187
|
+
self._document_pairs_sentences[base_doc_name] = {
|
|
188
|
+
self.postfix.FIRST_PAIR_ELEM: first_sentences,
|
|
189
|
+
self.postfix.SECOND_PAIR_ELEM: second_sentences,
|
|
190
|
+
}
|
|
191
|
+
pbar.update()
|
|
192
|
+
return None
|
|
193
|
+
|
|
194
|
+
def _split_paragraphs_to_sentences(self, paragraphs: list, min_sent_len: int = 1):
|
|
195
|
+
all_sentences = []
|
|
196
|
+
for paragraph_text in paragraphs:
|
|
197
|
+
sentences = self._sentence_splitter.split(
|
|
198
|
+
paragraph_text, return_stats_dict=False
|
|
199
|
+
)
|
|
200
|
+
all_sentences.extend(sentences)
|
|
201
|
+
return all_sentences
|
|
202
|
+
|
|
203
|
+
@staticmethod
|
|
204
|
+
def _load_docx_file_content(
|
|
205
|
+
file_path: str, unique_rows_in_cell: bool = False
|
|
206
|
+
) -> List[str]:
|
|
207
|
+
"""
|
|
208
|
+
Loads content from paragraphs and tables.
|
|
209
|
+
:param file_path: path to file to load content
|
|
210
|
+
:param unique_rows_in_cell: If set to True, items from
|
|
211
|
+
single row will be deduplicated
|
|
212
|
+
:return: List of read texts
|
|
213
|
+
"""
|
|
214
|
+
docx_loader = DOCXDocument(file_path)
|
|
215
|
+
par_texts = [p.text for p in docx_loader.paragraphs if len(p.text.strip())]
|
|
216
|
+
tab_texts = []
|
|
217
|
+
for table in docx_loader.tables:
|
|
218
|
+
for row in table.rows:
|
|
219
|
+
cell_texts = [
|
|
220
|
+
cell.text.strip() for cell in row.cells if cell.text.strip()
|
|
221
|
+
]
|
|
222
|
+
if unique_rows_in_cell:
|
|
223
|
+
cell_texts = list(dict.fromkeys(cell_texts))
|
|
224
|
+
tab_texts.extend(cell_texts)
|
|
225
|
+
return par_texts + tab_texts
|
|
226
|
+
|
|
227
|
+
def _filename_without_postfix(self, filename: str) -> str:
|
|
228
|
+
for postfix in self.postfix.ELEMS_TO_FILE_POSTFIX.values():
|
|
229
|
+
filename = filename.replace(postfix, "")
|
|
230
|
+
return filename
|
|
231
|
+
|
|
232
|
+
def _is_valid_filename(self, filename: str) -> bool:
|
|
233
|
+
for postfix in self.postfix.ELEMS_TO_FILE_POSTFIX.values():
|
|
234
|
+
if postfix in filename:
|
|
235
|
+
return True
|
|
236
|
+
return False
|
|
237
|
+
|
|
238
|
+
def _is_first_filename(self, filename: str) -> bool:
|
|
239
|
+
return filename.strip().endswith(
|
|
240
|
+
self.postfix.ELEMS_TO_FILE_POSTFIX[self.postfix.FIRST_PAIR_ELEM]
|
|
241
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TextProcessor(abc.ABC):
|
|
5
|
+
"""
|
|
6
|
+
Base class to all text processors
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
def __init__(self):
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
# def __call__(self, *args, **kwargs):
|
|
13
|
+
# self.apply()
|
|
14
|
+
|
|
15
|
+
@abc.abstractmethod
|
|
16
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
17
|
+
"""
|
|
18
|
+
Abstract method to process any text and return processed text.
|
|
19
|
+
:param text: Raw text
|
|
20
|
+
:param args: Optionally arguments for text processor
|
|
21
|
+
:return: Processed text as raw text
|
|
22
|
+
"""
|
|
23
|
+
pass
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import abc
|
|
3
|
+
|
|
4
|
+
from radlab_data.text.processors.base import TextProcessor
|
|
5
|
+
from radlab_data.text.processors.regex_defs import (
|
|
6
|
+
T_REPL,
|
|
7
|
+
REPL_W,
|
|
8
|
+
GENERAL_CLEANERS_REGEX,
|
|
9
|
+
ABBREV_REPLACE_MAP,
|
|
10
|
+
ADD_NEW_LINE_IN_CASE,
|
|
11
|
+
SENTENCE_END_CHARS,
|
|
12
|
+
ENUMERATION_REGEX,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TextCleaner(TextProcessor, abc.ABC):
|
|
17
|
+
"""
|
|
18
|
+
Base class for all text cleaners
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
super().__init__()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Strip(TextProcessor):
|
|
26
|
+
"""
|
|
27
|
+
Simple text processor to strip given text
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self):
|
|
31
|
+
super().__init__()
|
|
32
|
+
|
|
33
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
34
|
+
"""
|
|
35
|
+
Strip text and return new text after strip()
|
|
36
|
+
:param text: Text to strip
|
|
37
|
+
:param args: Additional arguments (not used)
|
|
38
|
+
:return: Text after strip
|
|
39
|
+
"""
|
|
40
|
+
return text.strip()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class RemovePhrase(TextCleaner):
|
|
44
|
+
"""
|
|
45
|
+
Main class to remove exact match of phrase into given text.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, phrases):
|
|
49
|
+
super().__init__()
|
|
50
|
+
self.phrases_to_remove = phrases if phrases is not None else []
|
|
51
|
+
|
|
52
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
53
|
+
"""
|
|
54
|
+
Remove all `self.phrases_to_remove` from given `text`
|
|
55
|
+
:param text: Text to remove phrases
|
|
56
|
+
:param args: Additional arguments (not used)
|
|
57
|
+
:return: Text without phrases
|
|
58
|
+
"""
|
|
59
|
+
text_clear = text
|
|
60
|
+
for ph in self.phrases_to_remove:
|
|
61
|
+
text_clear = text_clear.replace(ph, "")
|
|
62
|
+
return text_clear
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class NormalizeAbbreviations(TextCleaner):
|
|
66
|
+
"""
|
|
67
|
+
Replace abbreviations with it normalized form, something line replace
|
|
68
|
+
phrase to another phrase.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(self, action=None):
|
|
72
|
+
super().__init__()
|
|
73
|
+
|
|
74
|
+
self.args = None
|
|
75
|
+
if action is not None:
|
|
76
|
+
self.args = {"action": action}
|
|
77
|
+
|
|
78
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
79
|
+
"""
|
|
80
|
+
Abbreviations normalization
|
|
81
|
+
:param text: Text to normalize
|
|
82
|
+
:param args: Additional arguments
|
|
83
|
+
:return:
|
|
84
|
+
"""
|
|
85
|
+
if args is None:
|
|
86
|
+
args = self.args
|
|
87
|
+
|
|
88
|
+
normalize = True
|
|
89
|
+
if args is not None and "action" in args:
|
|
90
|
+
normalize = "norm" in args["action"].lower()
|
|
91
|
+
clear_text = text
|
|
92
|
+
for ab, abn in ABBREV_REPLACE_MAP:
|
|
93
|
+
if normalize:
|
|
94
|
+
clear_text = clear_text.replace(ab, abn)
|
|
95
|
+
else:
|
|
96
|
+
clear_text = clear_text.replace(abn, ab)
|
|
97
|
+
return clear_text
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class GeneralLookImprover(TextCleaner):
|
|
101
|
+
"""
|
|
102
|
+
Improve general text look
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
def __init__(self):
|
|
106
|
+
super().__init__()
|
|
107
|
+
|
|
108
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
109
|
+
"""
|
|
110
|
+
Improve general look of the given `text`
|
|
111
|
+
:param text:
|
|
112
|
+
:param args:
|
|
113
|
+
:return:
|
|
114
|
+
"""
|
|
115
|
+
for regex in GENERAL_CLEANERS_REGEX.values():
|
|
116
|
+
text = re.sub(regex.get(T_REPL), regex.get(REPL_W), text)
|
|
117
|
+
return text
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class RemoveEnums(TextCleaner):
|
|
121
|
+
"""
|
|
122
|
+
Remove enums from text
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
def __init__(self):
|
|
126
|
+
super().__init__()
|
|
127
|
+
|
|
128
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
129
|
+
"""
|
|
130
|
+
Remove enumerations from the text
|
|
131
|
+
:param text:
|
|
132
|
+
:param args:
|
|
133
|
+
:return:
|
|
134
|
+
"""
|
|
135
|
+
for regex in ENUMERATION_REGEX.values():
|
|
136
|
+
text = re.sub(regex.get(T_REPL), regex.get(REPL_W), text)
|
|
137
|
+
return text
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class RemoveQuestionMarks(TextCleaner):
|
|
141
|
+
"""
|
|
142
|
+
Remove qmarks from text
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
def __init__(self):
|
|
146
|
+
super().__init__()
|
|
147
|
+
|
|
148
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
149
|
+
"""
|
|
150
|
+
Remove question marks from the text
|
|
151
|
+
:param text:
|
|
152
|
+
:param args:
|
|
153
|
+
:return:
|
|
154
|
+
"""
|
|
155
|
+
normalized_text = ""
|
|
156
|
+
for line in text.split("\n"):
|
|
157
|
+
line = line.strip()
|
|
158
|
+
if len(line) < 5:
|
|
159
|
+
# n_txt += "\"
|
|
160
|
+
continue
|
|
161
|
+
line_s = line[0:-1].replace("?", "")
|
|
162
|
+
normalized_text += line_s + line[-1] + "\n\n"
|
|
163
|
+
return normalized_text
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class ProperLinesConstructor(TextCleaner):
|
|
167
|
+
"""
|
|
168
|
+
Prepare proper lines to split text to sentences
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
def __init__(self, force_add_dot: bool = False):
|
|
172
|
+
super().__init__()
|
|
173
|
+
self.force_add_dot = force_add_dot
|
|
174
|
+
|
|
175
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
176
|
+
"""
|
|
177
|
+
Prepare text with proper lines.
|
|
178
|
+
:param text:
|
|
179
|
+
:param args:
|
|
180
|
+
:return:
|
|
181
|
+
"""
|
|
182
|
+
prev_line = ""
|
|
183
|
+
normalized_text = ""
|
|
184
|
+
first_line = True
|
|
185
|
+
for line in text.split("\n"):
|
|
186
|
+
line = line.strip()
|
|
187
|
+
if len(line):
|
|
188
|
+
if first_line:
|
|
189
|
+
first_line = False
|
|
190
|
+
normalized_text = self._sentence_upper_start(line)
|
|
191
|
+
else:
|
|
192
|
+
new_sentence = not len(prev_line)
|
|
193
|
+
if new_sentence:
|
|
194
|
+
normalized_text = self._add_dot_at_the_end(
|
|
195
|
+
normalized_text, self.force_add_dot
|
|
196
|
+
)
|
|
197
|
+
normalized_text += "\n\n"
|
|
198
|
+
normalized_text += " " + self._sentence_upper_start(line)
|
|
199
|
+
prev_line = line
|
|
200
|
+
if self.force_add_dot:
|
|
201
|
+
return self._norm_text(normalized_text)
|
|
202
|
+
return normalized_text
|
|
203
|
+
|
|
204
|
+
def _norm_text(self, norm_text: str):
|
|
205
|
+
"""
|
|
206
|
+
Little text normalisation
|
|
207
|
+
:param norm_text:
|
|
208
|
+
:return:
|
|
209
|
+
"""
|
|
210
|
+
norm_text = norm_text.replace(":-", ": ")
|
|
211
|
+
norm_text = norm_text.replace(": -", ": ")
|
|
212
|
+
norm_text = norm_text.replace(",,", ",")
|
|
213
|
+
norm_text = norm_text.replace(" .", ".")
|
|
214
|
+
norm_text = norm_text.replace(" ", " ")
|
|
215
|
+
norm_text = norm_text.replace(" ", " ")
|
|
216
|
+
return norm_text
|
|
217
|
+
|
|
218
|
+
def _sentence_upper_start(self, sentence):
|
|
219
|
+
sentence = sentence.strip()
|
|
220
|
+
n_sent = sentence[0].upper()
|
|
221
|
+
if len(sentence) > 1:
|
|
222
|
+
n_sent += sentence[1:]
|
|
223
|
+
return n_sent
|
|
224
|
+
|
|
225
|
+
def _add_dot_at_the_end(self, norm_text_clear, force_add_dot=False):
|
|
226
|
+
"""
|
|
227
|
+
:param norm_text_clear:
|
|
228
|
+
:param force_add_dot:
|
|
229
|
+
:return:
|
|
230
|
+
"""
|
|
231
|
+
norm_text_clear = norm_text_clear.strip()
|
|
232
|
+
last_char = norm_text_clear[-1]
|
|
233
|
+
if last_char not in SENTENCE_END_CHARS:
|
|
234
|
+
norm_text_clear += "."
|
|
235
|
+
elif force_add_dot:
|
|
236
|
+
if last_char in ":,-":
|
|
237
|
+
norm_text_clear = norm_text_clear[0:-1] + "."
|
|
238
|
+
elif last_char in ')]"':
|
|
239
|
+
norm_text_clear += "."
|
|
240
|
+
return norm_text_clear
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class SimpleTextPreprocessor(TextCleaner):
|
|
244
|
+
"""
|
|
245
|
+
Prepare text to next steps
|
|
246
|
+
"""
|
|
247
|
+
|
|
248
|
+
def __init__(self, force_add_dot: bool = None):
|
|
249
|
+
super().__init__()
|
|
250
|
+
self._text = None
|
|
251
|
+
self.force_add_dot = force_add_dot
|
|
252
|
+
|
|
253
|
+
self._plc = ProperLinesConstructor(force_add_dot=False)
|
|
254
|
+
|
|
255
|
+
def apply(self, text: str, args: dict = None) -> str:
|
|
256
|
+
"""
|
|
257
|
+
Preprocess given text
|
|
258
|
+
:param text: Text to prepare
|
|
259
|
+
:param args: Additional arguments
|
|
260
|
+
:return: Prepared text
|
|
261
|
+
"""
|
|
262
|
+
self._text = text
|
|
263
|
+
# self._add_new_line_at_start()
|
|
264
|
+
self._text = self._plc.apply(self._text)
|
|
265
|
+
self._ws_clear()
|
|
266
|
+
return self._text
|
|
267
|
+
|
|
268
|
+
def _add_new_line_at_start(self) -> str:
|
|
269
|
+
for nline_in_case in ADD_NEW_LINE_IN_CASE.values():
|
|
270
|
+
self._text = re.sub(
|
|
271
|
+
nline_in_case.get(T_REPL), nline_in_case.get(REPL_W), self._text
|
|
272
|
+
)
|
|
273
|
+
self._text = self._text.replace(": -", ":\n-")
|
|
274
|
+
self._text = self._text.replace(":-", ":\n-")
|
|
275
|
+
return self._text
|
|
276
|
+
|
|
277
|
+
def _ws_clear(self):
|
|
278
|
+
self._text = self._text.replace("? ?", "?")
|
|
279
|
+
self._text = self._text.replace("\t", " ")
|
|
280
|
+
self._text = self._text.replace(" ", " ")
|
|
281
|
+
self._text = self._text.replace(" ", " ")
|
|
282
|
+
self._text = self._text.replace(" ", " ")
|
|
283
|
+
self._text = self._text.replace(" ", " ")
|
|
284
|
+
self._text = self._text.replace(": ,", ":")
|
|
285
|
+
self._text = self._text.replace(" ,", ",")
|
|
286
|
+
return self._text
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class FullChainProcessor:
|
|
290
|
+
"""
|
|
291
|
+
Chain of Text Processors
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
def __init__(self, phrases_to_remove):
|
|
295
|
+
self.chain = [
|
|
296
|
+
RemovePhrase(phrases=phrases_to_remove),
|
|
297
|
+
GeneralLookImprover(),
|
|
298
|
+
SimpleTextPreprocessor(),
|
|
299
|
+
NormalizeAbbreviations(action="norma"),
|
|
300
|
+
RemoveEnums(),
|
|
301
|
+
RemoveQuestionMarks(),
|
|
302
|
+
NormalizeAbbreviations(action="restore"),
|
|
303
|
+
ProperLinesConstructor(force_add_dot=False),
|
|
304
|
+
]
|
|
305
|
+
|
|
306
|
+
def apply(self, text: str) -> str:
|
|
307
|
+
"""
|
|
308
|
+
Apply whole processors chain
|
|
309
|
+
:return: Processed text
|
|
310
|
+
"""
|
|
311
|
+
for processor in self.chain:
|
|
312
|
+
text = processor.apply(text)
|
|
313
|
+
return text
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
class CleanerChainProcessor:
|
|
317
|
+
"""
|
|
318
|
+
Chain of Text Processors
|
|
319
|
+
"""
|
|
320
|
+
|
|
321
|
+
def __init__(self, phrases_to_remove):
|
|
322
|
+
self.chain = [
|
|
323
|
+
RemovePhrase(phrases=phrases_to_remove),
|
|
324
|
+
GeneralLookImprover(),
|
|
325
|
+
NormalizeAbbreviations(action="norma"),
|
|
326
|
+
RemoveEnums(),
|
|
327
|
+
RemoveQuestionMarks(),
|
|
328
|
+
NormalizeAbbreviations(action="restore"),
|
|
329
|
+
]
|
|
330
|
+
|
|
331
|
+
def apply(self, text: str) -> str:
|
|
332
|
+
"""
|
|
333
|
+
Apply whole processors chain
|
|
334
|
+
:return: Processed text
|
|
335
|
+
"""
|
|
336
|
+
for processor in self.chain:
|
|
337
|
+
text = processor.apply(text)
|
|
338
|
+
return text
|