radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,44 @@
1
+ from radlab_data.text.loaders.base import DocumentPage, read_text_file
2
+ from radlab_data.text.loaders.loader_i import LoaderI
3
+
4
+
5
+ class TXTLoader(LoaderI):
6
+ """
7
+ Loader for plain text files. The whole file is loaded as a single page.
8
+ """
9
+
10
+ def __init__(
11
+ self,
12
+ filepath: str,
13
+ options: dict = None,
14
+ encoding: str = "utf-8",
15
+ autodetect_encoding: bool = True,
16
+ ) -> None:
17
+ """
18
+ :param filepath: Path to text file
19
+ :param options: Options shared with other loaders (see `LoaderI`)
20
+ :param encoding: Encoding used to read the file
21
+ :param autodetect_encoding: If the file cannot be decoded with
22
+ `encoding`, encodings from `FALLBACK_ENCODINGS` are tried
23
+ """
24
+ LoaderI.__init__(self, filepath=filepath, options=options)
25
+ self.encoding = encoding
26
+ self.autodetect_encoding = autodetect_encoding
27
+
28
+ def load(self) -> list:
29
+ """
30
+ Load the text file.
31
+ :return: List with the single document page holding the file content
32
+ """
33
+ text = read_text_file(
34
+ self.filepath,
35
+ encoding=self.encoding,
36
+ autodetect_encoding=self.autodetect_encoding,
37
+ )
38
+ self.doc_pages = [
39
+ DocumentPage(
40
+ page_content=text,
41
+ metadata={"source": str(self.filepath)},
42
+ )
43
+ ]
44
+ return self.doc_pages
@@ -0,0 +1,241 @@
1
+ import abc
2
+ import logging
3
+ import os
4
+ from tqdm import tqdm
5
+ from typing import Dict, List
6
+ from docx import Document as DOCXDocument
7
+
8
+ from radlab_data.text.processors.splitter import SentenceSplitter
9
+
10
+
11
+ class PairElemsAbstract(abc.ABC):
12
+ FIRST_PAIR_ELEM = None
13
+ SECOND_PAIR_ELEM = None
14
+ NAME = None
15
+ ELEMS_TO_FILE_POSTFIX = None
16
+
17
+
18
+ class FirstSecondPairElems(PairElemsAbstract):
19
+ FIRST_PAIR_ELEM = "first"
20
+ SECOND_PAIR_ELEM = "second"
21
+ NAME = f"{FIRST_PAIR_ELEM}_{SECOND_PAIR_ELEM}"
22
+ ELEMS_TO_FILE_POSTFIX = {
23
+ FIRST_PAIR_ELEM: "_FIRST.docx",
24
+ SECOND_PAIR_ELEM: "_SECOND.docx",
25
+ }
26
+
27
+
28
+ class FirstSecondNumberPairElems(PairElemsAbstract):
29
+ FIRST_PAIR_ELEM = "1"
30
+ SECOND_PAIR_ELEM = "2"
31
+ NAME = f"{FIRST_PAIR_ELEM}_{SECOND_PAIR_ELEM}"
32
+ ELEMS_TO_FILE_POSTFIX = {
33
+ FIRST_PAIR_ELEM: "_1.docx",
34
+ SECOND_PAIR_ELEM: "_2.docx",
35
+ }
36
+
37
+
38
+ AVAILABLE_POSTFIXES = {
39
+ FirstSecondPairElems.NAME: FirstSecondPairElems,
40
+ FirstSecondNumberPairElems.NAME: FirstSecondNumberPairElems,
41
+ }
42
+
43
+
44
+ class DirectoryPairsReader:
45
+ AVAILABLE_POSTFIXES_STR = ", ".join(list(AVAILABLE_POSTFIXES.keys()))
46
+
47
+ def __init__(
48
+ self,
49
+ dir_path: str | None = None,
50
+ postfix_configuration_name: str = FirstSecondNumberPairElems.NAME,
51
+ ):
52
+ """
53
+ Set directory path to read documents
54
+ :param dir_path: Path to directory with documents to pair them
55
+ """
56
+ if postfix_configuration_name not in AVAILABLE_POSTFIXES:
57
+ raise ValueError(
58
+ f"Invalid postfix configuration: {postfix_configuration_name}. "
59
+ f"Postfix configuration must be one of: "
60
+ f"{self.AVAILABLE_POSTFIXES_STR}"
61
+ )
62
+ self.postfix: PairElemsAbstract = AVAILABLE_POSTFIXES[
63
+ postfix_configuration_name
64
+ ]()
65
+ self._postfix_configuration = postfix_configuration_name
66
+
67
+ self._documents_dir_path: str = dir_path
68
+
69
+ # Base filename to documents pair (paths to both documents)
70
+ self._document_pairs: Dict[str, Dict[str, str]] = {}
71
+ self._document_pairs_content: Dict[str, Dict[List[str], List[str]]] = {}
72
+
73
+ # base file name to dict: first/second to list of sentences
74
+ self._document_pairs_sentences: Dict[str, Dict[List[str], List[str]]] = {}
75
+
76
+ self._sentence_splitter = SentenceSplitter()
77
+
78
+ @property
79
+ def document_pairs_sentences(self):
80
+ return self._document_pairs_sentences
81
+
82
+ @property
83
+ def base_name_to_paths(self):
84
+ return self._document_pairs
85
+
86
+ def prepare_pairs(self, dir_path: str | None = None) -> Dict[str, Dict[str, str]]:
87
+ """
88
+ List files from given dir_path and prepare pairs of documents
89
+
90
+ :param dir_path: Path to directory containing files (documents)
91
+ :return: Dictionary with paths of pairs of documents
92
+ """
93
+ if dir_path is None:
94
+ dir_path = self._documents_dir_path
95
+ else:
96
+ self._documents_dir_path = dir_path
97
+ assert dir_path is not None, f"Directory with documents pairs is not provided"
98
+
99
+ self._document_pairs.clear()
100
+ self._document_pairs_content.clear()
101
+ for file_name in os.listdir(dir_path):
102
+ if not self._is_valid_filename(filename=file_name):
103
+ logging.info(
104
+ f"Skipping {file_name} - is not valid "
105
+ f"{self._postfix_configuration} file name"
106
+ )
107
+ continue
108
+
109
+ base_file_name = self._filename_without_postfix(file_name)
110
+ if base_file_name not in self._document_pairs:
111
+ self._document_pairs[base_file_name] = {
112
+ self.postfix.FIRST_PAIR_ELEM: "",
113
+ self.postfix.SECOND_PAIR_ELEM: "",
114
+ }
115
+
116
+ full_file_path = os.path.join(dir_path, file_name)
117
+ if self._is_first_filename(file_name):
118
+ self._document_pairs[base_file_name][
119
+ self.postfix.FIRST_PAIR_ELEM
120
+ ] = full_file_path
121
+ else:
122
+ self._document_pairs[base_file_name][
123
+ self.postfix.SECOND_PAIR_ELEM
124
+ ] = full_file_path
125
+ return self._document_pairs
126
+
127
+ def load_documents(self, unique_rows_in_cell: bool = False) -> None:
128
+ """
129
+ Loads documents from self._documents_pairs as read documents
130
+ :param unique_rows_in_cell: If set to True, items from
131
+ single row will be deduplicated
132
+ :return:
133
+ """
134
+ assert len(self._document_pairs) > 0, "No documents provided to load!"
135
+
136
+ self._document_pairs_content.clear()
137
+
138
+ with tqdm(total=len(self._document_pairs), desc="Loading files") as pbar:
139
+ for base_doc_name, doc_pair in self._document_pairs.items():
140
+ first_paragraphs, second_paragraphs = None, None
141
+
142
+ f_file = doc_pair[self.postfix.FIRST_PAIR_ELEM]
143
+ s_file = doc_pair[self.postfix.SECOND_PAIR_ELEM]
144
+
145
+ if f_file.endswith(".docx"):
146
+ first_paragraphs = self._load_docx_file_content(
147
+ file_path=f_file, unique_rows_in_cell=unique_rows_in_cell
148
+ )
149
+ second_paragraphs = self._load_docx_file_content(
150
+ file_path=s_file, unique_rows_in_cell=unique_rows_in_cell
151
+ )
152
+ else:
153
+ logging.info(f"Skipping file with not supported extension {f_file}")
154
+
155
+ self._document_pairs_content[base_doc_name] = {
156
+ self.postfix.FIRST_PAIR_ELEM: first_paragraphs,
157
+ self.postfix.SECOND_PAIR_ELEM: second_paragraphs,
158
+ }
159
+ pbar.update()
160
+ return self._split_documents_content_to_sentences()
161
+
162
+ def _split_documents_content_to_sentences(self) -> None:
163
+ assert (
164
+ len(self._document_pairs_content) > 0
165
+ ), "No documents provided to split content to sentences!"
166
+
167
+ self._document_pairs_sentences.clear()
168
+ with tqdm(
169
+ total=len(self._document_pairs_content),
170
+ desc="Splitting contents to sentences",
171
+ ) as pbar:
172
+ for base_doc_name, doc_pair in self._document_pairs_content.items():
173
+ first_sentences = self._split_paragraphs_to_sentences(
174
+ doc_pair[self.postfix.FIRST_PAIR_ELEM]
175
+ )
176
+ second_sentences = self._split_paragraphs_to_sentences(
177
+ doc_pair[self.postfix.SECOND_PAIR_ELEM]
178
+ )
179
+ if not len(second_sentences) or not len(first_sentences):
180
+ logging.info(
181
+ f"Skipping empty document {base_doc_name} "
182
+ f"[len(first_sentences) = {len(first_sentences)}, "
183
+ f"len(second_sentences) = {len(second_sentences)}]"
184
+ )
185
+ continue
186
+
187
+ self._document_pairs_sentences[base_doc_name] = {
188
+ self.postfix.FIRST_PAIR_ELEM: first_sentences,
189
+ self.postfix.SECOND_PAIR_ELEM: second_sentences,
190
+ }
191
+ pbar.update()
192
+ return None
193
+
194
+ def _split_paragraphs_to_sentences(self, paragraphs: list, min_sent_len: int = 1):
195
+ all_sentences = []
196
+ for paragraph_text in paragraphs:
197
+ sentences = self._sentence_splitter.split(
198
+ paragraph_text, return_stats_dict=False
199
+ )
200
+ all_sentences.extend(sentences)
201
+ return all_sentences
202
+
203
+ @staticmethod
204
+ def _load_docx_file_content(
205
+ file_path: str, unique_rows_in_cell: bool = False
206
+ ) -> List[str]:
207
+ """
208
+ Loads content from paragraphs and tables.
209
+ :param file_path: path to file to load content
210
+ :param unique_rows_in_cell: If set to True, items from
211
+ single row will be deduplicated
212
+ :return: List of read texts
213
+ """
214
+ docx_loader = DOCXDocument(file_path)
215
+ par_texts = [p.text for p in docx_loader.paragraphs if len(p.text.strip())]
216
+ tab_texts = []
217
+ for table in docx_loader.tables:
218
+ for row in table.rows:
219
+ cell_texts = [
220
+ cell.text.strip() for cell in row.cells if cell.text.strip()
221
+ ]
222
+ if unique_rows_in_cell:
223
+ cell_texts = list(dict.fromkeys(cell_texts))
224
+ tab_texts.extend(cell_texts)
225
+ return par_texts + tab_texts
226
+
227
+ def _filename_without_postfix(self, filename: str) -> str:
228
+ for postfix in self.postfix.ELEMS_TO_FILE_POSTFIX.values():
229
+ filename = filename.replace(postfix, "")
230
+ return filename
231
+
232
+ def _is_valid_filename(self, filename: str) -> bool:
233
+ for postfix in self.postfix.ELEMS_TO_FILE_POSTFIX.values():
234
+ if postfix in filename:
235
+ return True
236
+ return False
237
+
238
+ def _is_first_filename(self, filename: str) -> bool:
239
+ return filename.strip().endswith(
240
+ self.postfix.ELEMS_TO_FILE_POSTFIX[self.postfix.FIRST_PAIR_ELEM]
241
+ )
File without changes
@@ -0,0 +1,23 @@
1
+ import abc
2
+
3
+
4
+ class TextProcessor(abc.ABC):
5
+ """
6
+ Base class to all text processors
7
+ """
8
+
9
+ def __init__(self):
10
+ pass
11
+
12
+ # def __call__(self, *args, **kwargs):
13
+ # self.apply()
14
+
15
+ @abc.abstractmethod
16
+ def apply(self, text: str, args: dict = None) -> str:
17
+ """
18
+ Abstract method to process any text and return processed text.
19
+ :param text: Raw text
20
+ :param args: Optionally arguments for text processor
21
+ :return: Processed text as raw text
22
+ """
23
+ pass
@@ -0,0 +1,338 @@
1
+ import re
2
+ import abc
3
+
4
+ from radlab_data.text.processors.base import TextProcessor
5
+ from radlab_data.text.processors.regex_defs import (
6
+ T_REPL,
7
+ REPL_W,
8
+ GENERAL_CLEANERS_REGEX,
9
+ ABBREV_REPLACE_MAP,
10
+ ADD_NEW_LINE_IN_CASE,
11
+ SENTENCE_END_CHARS,
12
+ ENUMERATION_REGEX,
13
+ )
14
+
15
+
16
+ class TextCleaner(TextProcessor, abc.ABC):
17
+ """
18
+ Base class for all text cleaners
19
+ """
20
+
21
+ def __init__(self):
22
+ super().__init__()
23
+
24
+
25
+ class Strip(TextProcessor):
26
+ """
27
+ Simple text processor to strip given text
28
+ """
29
+
30
+ def __init__(self):
31
+ super().__init__()
32
+
33
+ def apply(self, text: str, args: dict = None) -> str:
34
+ """
35
+ Strip text and return new text after strip()
36
+ :param text: Text to strip
37
+ :param args: Additional arguments (not used)
38
+ :return: Text after strip
39
+ """
40
+ return text.strip()
41
+
42
+
43
+ class RemovePhrase(TextCleaner):
44
+ """
45
+ Main class to remove exact match of phrase into given text.
46
+ """
47
+
48
+ def __init__(self, phrases):
49
+ super().__init__()
50
+ self.phrases_to_remove = phrases if phrases is not None else []
51
+
52
+ def apply(self, text: str, args: dict = None) -> str:
53
+ """
54
+ Remove all `self.phrases_to_remove` from given `text`
55
+ :param text: Text to remove phrases
56
+ :param args: Additional arguments (not used)
57
+ :return: Text without phrases
58
+ """
59
+ text_clear = text
60
+ for ph in self.phrases_to_remove:
61
+ text_clear = text_clear.replace(ph, "")
62
+ return text_clear
63
+
64
+
65
+ class NormalizeAbbreviations(TextCleaner):
66
+ """
67
+ Replace abbreviations with it normalized form, something line replace
68
+ phrase to another phrase.
69
+ """
70
+
71
+ def __init__(self, action=None):
72
+ super().__init__()
73
+
74
+ self.args = None
75
+ if action is not None:
76
+ self.args = {"action": action}
77
+
78
+ def apply(self, text: str, args: dict = None) -> str:
79
+ """
80
+ Abbreviations normalization
81
+ :param text: Text to normalize
82
+ :param args: Additional arguments
83
+ :return:
84
+ """
85
+ if args is None:
86
+ args = self.args
87
+
88
+ normalize = True
89
+ if args is not None and "action" in args:
90
+ normalize = "norm" in args["action"].lower()
91
+ clear_text = text
92
+ for ab, abn in ABBREV_REPLACE_MAP:
93
+ if normalize:
94
+ clear_text = clear_text.replace(ab, abn)
95
+ else:
96
+ clear_text = clear_text.replace(abn, ab)
97
+ return clear_text
98
+
99
+
100
+ class GeneralLookImprover(TextCleaner):
101
+ """
102
+ Improve general text look
103
+ """
104
+
105
+ def __init__(self):
106
+ super().__init__()
107
+
108
+ def apply(self, text: str, args: dict = None) -> str:
109
+ """
110
+ Improve general look of the given `text`
111
+ :param text:
112
+ :param args:
113
+ :return:
114
+ """
115
+ for regex in GENERAL_CLEANERS_REGEX.values():
116
+ text = re.sub(regex.get(T_REPL), regex.get(REPL_W), text)
117
+ return text
118
+
119
+
120
+ class RemoveEnums(TextCleaner):
121
+ """
122
+ Remove enums from text
123
+ """
124
+
125
+ def __init__(self):
126
+ super().__init__()
127
+
128
+ def apply(self, text: str, args: dict = None) -> str:
129
+ """
130
+ Remove enumerations from the text
131
+ :param text:
132
+ :param args:
133
+ :return:
134
+ """
135
+ for regex in ENUMERATION_REGEX.values():
136
+ text = re.sub(regex.get(T_REPL), regex.get(REPL_W), text)
137
+ return text
138
+
139
+
140
+ class RemoveQuestionMarks(TextCleaner):
141
+ """
142
+ Remove qmarks from text
143
+ """
144
+
145
+ def __init__(self):
146
+ super().__init__()
147
+
148
+ def apply(self, text: str, args: dict = None) -> str:
149
+ """
150
+ Remove question marks from the text
151
+ :param text:
152
+ :param args:
153
+ :return:
154
+ """
155
+ normalized_text = ""
156
+ for line in text.split("\n"):
157
+ line = line.strip()
158
+ if len(line) < 5:
159
+ # n_txt += "\"
160
+ continue
161
+ line_s = line[0:-1].replace("?", "")
162
+ normalized_text += line_s + line[-1] + "\n\n"
163
+ return normalized_text
164
+
165
+
166
+ class ProperLinesConstructor(TextCleaner):
167
+ """
168
+ Prepare proper lines to split text to sentences
169
+ """
170
+
171
+ def __init__(self, force_add_dot: bool = False):
172
+ super().__init__()
173
+ self.force_add_dot = force_add_dot
174
+
175
+ def apply(self, text: str, args: dict = None) -> str:
176
+ """
177
+ Prepare text with proper lines.
178
+ :param text:
179
+ :param args:
180
+ :return:
181
+ """
182
+ prev_line = ""
183
+ normalized_text = ""
184
+ first_line = True
185
+ for line in text.split("\n"):
186
+ line = line.strip()
187
+ if len(line):
188
+ if first_line:
189
+ first_line = False
190
+ normalized_text = self._sentence_upper_start(line)
191
+ else:
192
+ new_sentence = not len(prev_line)
193
+ if new_sentence:
194
+ normalized_text = self._add_dot_at_the_end(
195
+ normalized_text, self.force_add_dot
196
+ )
197
+ normalized_text += "\n\n"
198
+ normalized_text += " " + self._sentence_upper_start(line)
199
+ prev_line = line
200
+ if self.force_add_dot:
201
+ return self._norm_text(normalized_text)
202
+ return normalized_text
203
+
204
+ def _norm_text(self, norm_text: str):
205
+ """
206
+ Little text normalisation
207
+ :param norm_text:
208
+ :return:
209
+ """
210
+ norm_text = norm_text.replace(":-", ": ")
211
+ norm_text = norm_text.replace(": -", ": ")
212
+ norm_text = norm_text.replace(",,", ",")
213
+ norm_text = norm_text.replace(" .", ".")
214
+ norm_text = norm_text.replace(" ", " ")
215
+ norm_text = norm_text.replace(" ", " ")
216
+ return norm_text
217
+
218
+ def _sentence_upper_start(self, sentence):
219
+ sentence = sentence.strip()
220
+ n_sent = sentence[0].upper()
221
+ if len(sentence) > 1:
222
+ n_sent += sentence[1:]
223
+ return n_sent
224
+
225
+ def _add_dot_at_the_end(self, norm_text_clear, force_add_dot=False):
226
+ """
227
+ :param norm_text_clear:
228
+ :param force_add_dot:
229
+ :return:
230
+ """
231
+ norm_text_clear = norm_text_clear.strip()
232
+ last_char = norm_text_clear[-1]
233
+ if last_char not in SENTENCE_END_CHARS:
234
+ norm_text_clear += "."
235
+ elif force_add_dot:
236
+ if last_char in ":,-":
237
+ norm_text_clear = norm_text_clear[0:-1] + "."
238
+ elif last_char in ')]"':
239
+ norm_text_clear += "."
240
+ return norm_text_clear
241
+
242
+
243
+ class SimpleTextPreprocessor(TextCleaner):
244
+ """
245
+ Prepare text to next steps
246
+ """
247
+
248
+ def __init__(self, force_add_dot: bool = None):
249
+ super().__init__()
250
+ self._text = None
251
+ self.force_add_dot = force_add_dot
252
+
253
+ self._plc = ProperLinesConstructor(force_add_dot=False)
254
+
255
+ def apply(self, text: str, args: dict = None) -> str:
256
+ """
257
+ Preprocess given text
258
+ :param text: Text to prepare
259
+ :param args: Additional arguments
260
+ :return: Prepared text
261
+ """
262
+ self._text = text
263
+ # self._add_new_line_at_start()
264
+ self._text = self._plc.apply(self._text)
265
+ self._ws_clear()
266
+ return self._text
267
+
268
+ def _add_new_line_at_start(self) -> str:
269
+ for nline_in_case in ADD_NEW_LINE_IN_CASE.values():
270
+ self._text = re.sub(
271
+ nline_in_case.get(T_REPL), nline_in_case.get(REPL_W), self._text
272
+ )
273
+ self._text = self._text.replace(": -", ":\n-")
274
+ self._text = self._text.replace(":-", ":\n-")
275
+ return self._text
276
+
277
+ def _ws_clear(self):
278
+ self._text = self._text.replace("? ?", "?")
279
+ self._text = self._text.replace("\t", " ")
280
+ self._text = self._text.replace(" ", " ")
281
+ self._text = self._text.replace(" ", " ")
282
+ self._text = self._text.replace(" ", " ")
283
+ self._text = self._text.replace(" ", " ")
284
+ self._text = self._text.replace(": ,", ":")
285
+ self._text = self._text.replace(" ,", ",")
286
+ return self._text
287
+
288
+
289
+ class FullChainProcessor:
290
+ """
291
+ Chain of Text Processors
292
+ """
293
+
294
+ def __init__(self, phrases_to_remove):
295
+ self.chain = [
296
+ RemovePhrase(phrases=phrases_to_remove),
297
+ GeneralLookImprover(),
298
+ SimpleTextPreprocessor(),
299
+ NormalizeAbbreviations(action="norma"),
300
+ RemoveEnums(),
301
+ RemoveQuestionMarks(),
302
+ NormalizeAbbreviations(action="restore"),
303
+ ProperLinesConstructor(force_add_dot=False),
304
+ ]
305
+
306
+ def apply(self, text: str) -> str:
307
+ """
308
+ Apply whole processors chain
309
+ :return: Processed text
310
+ """
311
+ for processor in self.chain:
312
+ text = processor.apply(text)
313
+ return text
314
+
315
+
316
+ class CleanerChainProcessor:
317
+ """
318
+ Chain of Text Processors
319
+ """
320
+
321
+ def __init__(self, phrases_to_remove):
322
+ self.chain = [
323
+ RemovePhrase(phrases=phrases_to_remove),
324
+ GeneralLookImprover(),
325
+ NormalizeAbbreviations(action="norma"),
326
+ RemoveEnums(),
327
+ RemoveQuestionMarks(),
328
+ NormalizeAbbreviations(action="restore"),
329
+ ]
330
+
331
+ def apply(self, text: str) -> str:
332
+ """
333
+ Apply whole processors chain
334
+ :return: Processed text
335
+ """
336
+ for processor in self.chain:
337
+ text = processor.apply(text)
338
+ return text