radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BaseMapper(abc.ABC):
|
|
5
|
+
def __init__(self):
|
|
6
|
+
self._obj1 = None
|
|
7
|
+
self._obj2 = None
|
|
8
|
+
self._mapping = None
|
|
9
|
+
|
|
10
|
+
@abc.abstractmethod
|
|
11
|
+
def apply(self, obj1, obj2) -> list:
|
|
12
|
+
"""
|
|
13
|
+
Abstract method to apply mapping obj1 on obj2
|
|
14
|
+
:param obj1: Object to map
|
|
15
|
+
:param obj2: On this object obj1 will be mapped
|
|
16
|
+
:return: List (self._mapping) of mapping
|
|
17
|
+
"""
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RawSentencesToTextMapper(BaseMapper):
|
|
22
|
+
ESTIMATED_POSITION_LEN_CHARS_BIAS = 4
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
simple: bool = True,
|
|
27
|
+
lower: bool = True,
|
|
28
|
+
remove_dot_when_mapping: bool = False,
|
|
29
|
+
exact_sentence_match: bool = True,
|
|
30
|
+
):
|
|
31
|
+
"""
|
|
32
|
+
Actually only simple mapping is available. Simple means, that only first
|
|
33
|
+
and the last sentence words are used to determine sentence position into
|
|
34
|
+
the whole text, text and the sentences are also lowered.
|
|
35
|
+
Mapping objects:
|
|
36
|
+
* self._obj1 have to be set as the list of raw sentences
|
|
37
|
+
* self._obj2 have to be set as the whole raw text
|
|
38
|
+
:param simple - if set to True, then simple mapping will be used
|
|
39
|
+
:param lower - if set to True, then compared texts will be converted to lower
|
|
40
|
+
:param remove_dot_when_mapping - In case when mapped sentence is not found
|
|
41
|
+
then mapping without dot at sentence end will be done
|
|
42
|
+
:param exact_sentence_match - If set to True, then exact sentence
|
|
43
|
+
will be match instead of heuristic
|
|
44
|
+
:exception - Raise Exception when not simple method will be used.
|
|
45
|
+
"""
|
|
46
|
+
if not simple:
|
|
47
|
+
raise Exception("Only simple mapping is supporting now!")
|
|
48
|
+
super().__init__()
|
|
49
|
+
self._lower = lower
|
|
50
|
+
self._simple_map = simple
|
|
51
|
+
self._exact_sentence_match = exact_sentence_match
|
|
52
|
+
self._remove_dot_when_mapping = remove_dot_when_mapping
|
|
53
|
+
|
|
54
|
+
def apply(self, sentences: list, text: str):
|
|
55
|
+
self._obj1 = sentences
|
|
56
|
+
self._obj2 = text
|
|
57
|
+
self._mapping = []
|
|
58
|
+
|
|
59
|
+
end_position = 0
|
|
60
|
+
for s in sentences:
|
|
61
|
+
begin_position, end_position = self.map_sentence_to_text(
|
|
62
|
+
s, text, end_position
|
|
63
|
+
)
|
|
64
|
+
self._mapping.append([begin_position, end_position])
|
|
65
|
+
return self._mapping
|
|
66
|
+
|
|
67
|
+
def map_sentence_to_text(
|
|
68
|
+
self, sentence: str, text: str, begin_position: int
|
|
69
|
+
) -> (int, int):
|
|
70
|
+
"""
|
|
71
|
+
Map the single sentence on the given text, mapping procedure starts
|
|
72
|
+
from given begin_position. If begin_position is smaller than 0,
|
|
73
|
+
then mapping will not be done (assert).
|
|
74
|
+
|
|
75
|
+
:param sentence:
|
|
76
|
+
:param text:
|
|
77
|
+
:param begin_position:
|
|
78
|
+
:return:
|
|
79
|
+
"""
|
|
80
|
+
assert begin_position >= 0, "Begin position cannot be smaller than 0!"
|
|
81
|
+
if self._exact_sentence_match:
|
|
82
|
+
return self.__exact_map_sentence_to_text(
|
|
83
|
+
sentence=sentence, text=text, begin_position=begin_position
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
_epos = (
|
|
87
|
+
begin_position + len(sentence) + self.ESTIMATED_POSITION_LEN_CHARS_BIAS
|
|
88
|
+
)
|
|
89
|
+
if _epos > len(text):
|
|
90
|
+
_epos = len(text)
|
|
91
|
+
|
|
92
|
+
begin_position -= 1
|
|
93
|
+
if begin_position < 0:
|
|
94
|
+
begin_position = 0
|
|
95
|
+
|
|
96
|
+
context = text[begin_position:_epos]
|
|
97
|
+
if self._lower:
|
|
98
|
+
context = context.lower()
|
|
99
|
+
sentence = sentence.lower()
|
|
100
|
+
|
|
101
|
+
spl_sent = sentence.split()
|
|
102
|
+
w_first, w_last = spl_sent[0], spl_sent[-1]
|
|
103
|
+
b_pos = self.__find_word_position(w_first, context, 0, None)
|
|
104
|
+
e_pos = self.__find_word_position(
|
|
105
|
+
word=w_last,
|
|
106
|
+
context=context,
|
|
107
|
+
start_position=b_pos,
|
|
108
|
+
estimated_len=len(sentence),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
if e_pos == -1 and self._remove_dot_when_mapping:
|
|
112
|
+
if w_last[-1] == ".":
|
|
113
|
+
w_last = w_last[:-1]
|
|
114
|
+
e_pos = self.__find_word_position(
|
|
115
|
+
word=w_last,
|
|
116
|
+
context=context,
|
|
117
|
+
start_position=b_pos,
|
|
118
|
+
estimated_len=len(sentence),
|
|
119
|
+
)
|
|
120
|
+
return begin_position + b_pos, e_pos + begin_position
|
|
121
|
+
|
|
122
|
+
def __exact_map_sentence_to_text(
|
|
123
|
+
self, sentence: str, text: str, begin_position: int
|
|
124
|
+
) -> (int, int):
|
|
125
|
+
"""
|
|
126
|
+
Map the single sentence on the given text, mapping procedure starts
|
|
127
|
+
from given begin_position. If begin_position is smaller than 0,
|
|
128
|
+
then mapping will not be done (assert).
|
|
129
|
+
|
|
130
|
+
:param sentence:
|
|
131
|
+
:param text:
|
|
132
|
+
:param begin_position:
|
|
133
|
+
:return:
|
|
134
|
+
"""
|
|
135
|
+
b_position = begin_position - 5
|
|
136
|
+
if b_position < 0:
|
|
137
|
+
b_position = 0
|
|
138
|
+
find_sentence = sentence
|
|
139
|
+
context = text[b_position:]
|
|
140
|
+
if self._lower:
|
|
141
|
+
context = context.lower()
|
|
142
|
+
find_sentence = find_sentence.lower()
|
|
143
|
+
find_position = context.lower().find(find_sentence.lower())
|
|
144
|
+
if find_position < 0:
|
|
145
|
+
return 0, len(sentence)
|
|
146
|
+
start_position = b_position + find_position
|
|
147
|
+
end_position = start_position + len(sentence)
|
|
148
|
+
return start_position, end_position
|
|
149
|
+
|
|
150
|
+
def __find_word_position(
|
|
151
|
+
self,
|
|
152
|
+
word: str,
|
|
153
|
+
context: str,
|
|
154
|
+
start_position: int,
|
|
155
|
+
estimated_len: int = None,
|
|
156
|
+
):
|
|
157
|
+
if estimated_len is None:
|
|
158
|
+
return context[start_position:].find(word)
|
|
159
|
+
|
|
160
|
+
w_pos = context.rfind(word)
|
|
161
|
+
w_sent_len = (w_pos - start_position) + len(word)
|
|
162
|
+
if w_sent_len < 0:
|
|
163
|
+
return -1
|
|
164
|
+
|
|
165
|
+
if abs(estimated_len - w_sent_len) <= self.ESTIMATED_POSITION_LEN_CHARS_BIAS:
|
|
166
|
+
return w_pos + len(word)
|
|
167
|
+
return -1
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
T_REPL = "to_replace"
|
|
2
|
+
REPL_W = "replace_with"
|
|
3
|
+
|
|
4
|
+
# Some general regext to apply to get first stage of clear text
|
|
5
|
+
GENERAL_CLEANERS_REGEX = {
|
|
6
|
+
"repl_space_quot_space_to_comma": {
|
|
7
|
+
T_REPL: r" \? ",
|
|
8
|
+
REPL_W: ", ",
|
|
9
|
+
},
|
|
10
|
+
"repl_colon_comma_to_colon": {T_REPL: r":,", REPL_W: ":"},
|
|
11
|
+
"repl_colon_quote_to_colon": {T_REPL: r": \?", REPL_W: ":"},
|
|
12
|
+
"repl_quote_dot_to_dot": {T_REPL: r"\? \.", REPL_W: "."},
|
|
13
|
+
"repl_quote_dot_to_dot2": {T_REPL: r"\?\.", REPL_W: "."},
|
|
14
|
+
"repl_comma_dash_to_comma": {T_REPL: r", \-", REPL_W: ","},
|
|
15
|
+
"repl_dd_comma_to_comma": {T_REPL: r", ,", REPL_W: ","},
|
|
16
|
+
"repl_d_comma_to_comma": {T_REPL: r",,", REPL_W: ","},
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
# Abbreviations replacement
|
|
20
|
+
ABBREV_REPLACE_MAP = [
|
|
21
|
+
("np.", "np_"),
|
|
22
|
+
("ul.", "ul_"),
|
|
23
|
+
("tj.", "tj_"),
|
|
24
|
+
("inż.", "inż_"),
|
|
25
|
+
("S.A.", "S_A_"),
|
|
26
|
+
("s.a.", "s_a_"),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
# www/url
|
|
30
|
+
REMOVE_URL = [
|
|
31
|
+
(
|
|
32
|
+
"((http|https)\:\/\/)?[a-zA-Z0-9\.\/\?\:@\-_=#]"
|
|
33
|
+
"+\.([a-zA-Z]){2,6}([a-zA-Z0-9\.\&\/\?\:@\-_=#])*",
|
|
34
|
+
"WWW_URL",
|
|
35
|
+
),
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
# Add new line in some cases
|
|
39
|
+
ADD_NEW_LINE_IN_CASE = {
|
|
40
|
+
"repl_beg_chat_num": {T_REPL: r"^\(\d+\)", REPL_W: ""},
|
|
41
|
+
"repl_in_chat_num": {T_REPL: r"\(\d+\)", REPL_W: "\n"},
|
|
42
|
+
"repl_dot_newline1": {
|
|
43
|
+
T_REPL: r'(\s[A-zęąŃńćĆóÓźŹżŻśŚłŁ"()]+[0-9]{0,20}\.)',
|
|
44
|
+
REPL_W: r"\1\n\n",
|
|
45
|
+
},
|
|
46
|
+
"repl_dot_newline2": {T_REPL: r"(\s[0-9]+\.)", REPL_W: r"\1\n\n"},
|
|
47
|
+
"repl_dot_newline3": {T_REPL: r"(\s[A-Z]+\.)", REPL_W: r"\1\n\n"},
|
|
48
|
+
"repl_char_eum_in": {T_REPL: r"\s[a-z]\)", REPL_W: "\n\n"},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
# These characters may occur on the sentence end
|
|
52
|
+
SENTENCE_END_CHARS = "!@#$%^&*()_-+={[}];:'\"<,>./?"
|
|
53
|
+
|
|
54
|
+
# Enumerations
|
|
55
|
+
ENUMERATION_REGEX = {
|
|
56
|
+
"enum_dot_standard": {T_REPL: r"(\s)[0-9]\.\s", REPL_W: r"\1"},
|
|
57
|
+
"enum_round_brackets": {T_REPL: r"(\s)[0-9]\)\s", REPL_W: r"\1"},
|
|
58
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import spacy
|
|
2
|
+
|
|
3
|
+
# The spaCy model is loaded lazily on the first use, because importing this
|
|
4
|
+
# module (for example to use only the pair reader) should not require the
|
|
5
|
+
# Polish model to be installed and should not take several seconds.
|
|
6
|
+
MODEL_NAME = "pl_core_news_lg"
|
|
7
|
+
_nlp = None
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_nlp():
|
|
11
|
+
"""
|
|
12
|
+
Return the spaCy pipeline used for sentence splitting. The model is
|
|
13
|
+
loaded on the first call and cached for the next ones.
|
|
14
|
+
:return: Loaded spaCy pipeline
|
|
15
|
+
"""
|
|
16
|
+
global _nlp
|
|
17
|
+
if _nlp is None:
|
|
18
|
+
_nlp = spacy.load(MODEL_NAME)
|
|
19
|
+
return _nlp
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SentenceSplitter:
|
|
23
|
+
min_sentence_len = 1
|
|
24
|
+
|
|
25
|
+
def split(
|
|
26
|
+
self,
|
|
27
|
+
text: str,
|
|
28
|
+
return_stats_dict: bool = False,
|
|
29
|
+
split_words_with_ws: bool = True,
|
|
30
|
+
):
|
|
31
|
+
"""
|
|
32
|
+
Split text to sentences and optionally return statistics
|
|
33
|
+
|
|
34
|
+
:param text: Text given as string to split to sentences
|
|
35
|
+
:param return_stats_dict: If set, then dictionary with stats will be returned
|
|
36
|
+
:param split_words_with_ws: If set, then number of words will be calculated
|
|
37
|
+
using standard str.split() method
|
|
38
|
+
:return: List of sentences (+ optionally statistics dict)
|
|
39
|
+
"""
|
|
40
|
+
doc = get_nlp()(text)
|
|
41
|
+
|
|
42
|
+
sentences = []
|
|
43
|
+
text_stats = {
|
|
44
|
+
"number_of_sentences": 0,
|
|
45
|
+
"number_of_words": 0,
|
|
46
|
+
"number_of_characters": len(text),
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
for s in doc.sents:
|
|
50
|
+
if return_stats_dict:
|
|
51
|
+
text_stats["number_of_sentences"] += 1
|
|
52
|
+
if split_words_with_ws:
|
|
53
|
+
text_stats["number_of_words"] += len(s.text.strip().split())
|
|
54
|
+
else:
|
|
55
|
+
text_stats["number_of_words"] += len(s)
|
|
56
|
+
|
|
57
|
+
sentence = s.text.strip()
|
|
58
|
+
if len(sentence.split()) >= self.min_sentence_len:
|
|
59
|
+
sentences.append(sentence)
|
|
60
|
+
return (sentences, text_stats) if return_stats_dict else sentences
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from multiprocessing import Pool
|
|
3
|
+
|
|
4
|
+
import tqdm
|
|
5
|
+
import pathlib
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from radlab_data.text.document import Document
|
|
9
|
+
from radlab_data.text.loaders.config import AVAILABLE_FILE_EXTENSIONS
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DirectoryFileReader:
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
main_dir_path: str,
|
|
16
|
+
read_sub_dirs: bool = True,
|
|
17
|
+
accept_extensions: list = None,
|
|
18
|
+
processes_count: int = None,
|
|
19
|
+
):
|
|
20
|
+
"""
|
|
21
|
+
:param main_dir_path: Path to main directory to load files
|
|
22
|
+
:param read_sub_dirs: If set to True, then will be done listing
|
|
23
|
+
all files in main_dir_path with subdirectories
|
|
24
|
+
:param accept_extensions: List of accepted extension, defaults to
|
|
25
|
+
`AVAILABLE_FILE_EXTENSIONS`
|
|
26
|
+
:param processes_count: If number of processes is greater than 1,
|
|
27
|
+
will use given number of processes to load listed files.
|
|
28
|
+
"""
|
|
29
|
+
self.main_dir_path = main_dir_path
|
|
30
|
+
self.deep_read = read_sub_dirs
|
|
31
|
+
self.accept_extensions = (
|
|
32
|
+
list(AVAILABLE_FILE_EXTENSIONS)
|
|
33
|
+
if accept_extensions is None
|
|
34
|
+
else accept_extensions
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
self._processes_count = processes_count
|
|
38
|
+
self._use_multiprocessing = processes_count is not None and processes_count > 1
|
|
39
|
+
self._files_in_directory = self._list_files()
|
|
40
|
+
self._all_documents_in_directory = []
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def documents(self) -> list:
|
|
44
|
+
return self._all_documents_in_directory
|
|
45
|
+
|
|
46
|
+
def load(
|
|
47
|
+
self,
|
|
48
|
+
prepare_proper_pages: bool = False,
|
|
49
|
+
merge_document_pages: bool = False,
|
|
50
|
+
clear_texts: bool = False,
|
|
51
|
+
use_text_denoiser: bool = False,
|
|
52
|
+
max_tokens_in_chunk: int = None,
|
|
53
|
+
number_of_overlap_tokens: int = None,
|
|
54
|
+
check_text_language: bool = False,
|
|
55
|
+
) -> None:
|
|
56
|
+
"""
|
|
57
|
+
Loads all documents listed in `self._files_in_directory`
|
|
58
|
+
:param prepare_proper_pages: IF set to True, then all texts/sentences
|
|
59
|
+
from the same page will be merged to single page.
|
|
60
|
+
:param merge_document_pages: If set to True then all pages from document
|
|
61
|
+
will be merged to single page, then document will contain single page.
|
|
62
|
+
:param clear_texts: If set to True, then each text will be clear
|
|
63
|
+
:param use_text_denoiser: If set to True, then each text will be denoised
|
|
64
|
+
:param max_tokens_in_chunk: If given then each text will be split
|
|
65
|
+
to chunks with given max number of tokens in single chunk
|
|
66
|
+
:param number_of_overlap_tokens: If given then each generated chunk
|
|
67
|
+
will contain number_of_overlap_tokens from the previous chunk.
|
|
68
|
+
:param check_text_language: If options is set to True, then language
|
|
69
|
+
of each text will be checked and stored to metadata
|
|
70
|
+
:return: None
|
|
71
|
+
"""
|
|
72
|
+
logging.info(f"processes_count={self._processes_count}")
|
|
73
|
+
logging.info(f"prepare_proper_pages={prepare_proper_pages}")
|
|
74
|
+
logging.info(f"merge_document_pages={merge_document_pages}")
|
|
75
|
+
logging.info(f"clear_texts={clear_texts}")
|
|
76
|
+
logging.info(f"use_text_denoiser={use_text_denoiser}")
|
|
77
|
+
logging.info(f"check_text_language={check_text_language}")
|
|
78
|
+
logging.info(f"max_tokens_in_chunk={max_tokens_in_chunk}")
|
|
79
|
+
logging.info(f"number_of_overlap_tokens={number_of_overlap_tokens}")
|
|
80
|
+
|
|
81
|
+
if self._use_multiprocessing:
|
|
82
|
+
return self._load_mt(
|
|
83
|
+
prepare_proper_pages=prepare_proper_pages,
|
|
84
|
+
merge_document_pages=merge_document_pages,
|
|
85
|
+
clear_texts=clear_texts,
|
|
86
|
+
use_text_denoiser=use_text_denoiser,
|
|
87
|
+
max_tokens_in_chunk=max_tokens_in_chunk,
|
|
88
|
+
number_of_overlap_tokens=number_of_overlap_tokens,
|
|
89
|
+
check_text_language=check_text_language,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
with tqdm.tqdm(
|
|
93
|
+
total=len(self._files_in_directory), desc="Loading files"
|
|
94
|
+
) as pbar:
|
|
95
|
+
for file_path in self._files_in_directory:
|
|
96
|
+
try:
|
|
97
|
+
doc = Document(
|
|
98
|
+
file_path=file_path,
|
|
99
|
+
relative_file_path=self._relative_file_path(file_path),
|
|
100
|
+
prepare_proper_pages=prepare_proper_pages,
|
|
101
|
+
merge_document_pages=merge_document_pages,
|
|
102
|
+
clear_texts=clear_texts,
|
|
103
|
+
use_text_denoiser=use_text_denoiser,
|
|
104
|
+
max_tokens_in_chunk=max_tokens_in_chunk,
|
|
105
|
+
number_of_overlap_tokens=number_of_overlap_tokens,
|
|
106
|
+
check_text_language=check_text_language,
|
|
107
|
+
)
|
|
108
|
+
doc.load()
|
|
109
|
+
self._all_documents_in_directory.append(doc)
|
|
110
|
+
except Exception:
|
|
111
|
+
logging.warning(
|
|
112
|
+
f"Problem with loading file {file_path}", exc_info=True
|
|
113
|
+
)
|
|
114
|
+
pbar.update()
|
|
115
|
+
|
|
116
|
+
def _relative_file_path(self, file_path: str) -> str:
|
|
117
|
+
"""
|
|
118
|
+
Path to the file relative to the main directory path.
|
|
119
|
+
:param file_path: Path to the loaded file
|
|
120
|
+
:return: Relative path, the original path when it is not located
|
|
121
|
+
inside the main directory
|
|
122
|
+
"""
|
|
123
|
+
try:
|
|
124
|
+
return os.path.relpath(file_path, self.main_dir_path)
|
|
125
|
+
except ValueError:
|
|
126
|
+
return file_path
|
|
127
|
+
|
|
128
|
+
def _list_files(self) -> list:
|
|
129
|
+
"""
|
|
130
|
+
Depends on the self.deep_read parameter, reads all files in main_dir_path
|
|
131
|
+
and subdirectories when set to True, otherwise reads only from main directory.
|
|
132
|
+
:return: List of paths to files to read
|
|
133
|
+
"""
|
|
134
|
+
if self.deep_read:
|
|
135
|
+
all_files_to_read = pathlib.Path(self.main_dir_path)
|
|
136
|
+
all_files_to_read = [str(d) for d in all_files_to_read.rglob("*")]
|
|
137
|
+
else:
|
|
138
|
+
all_files_to_read = [
|
|
139
|
+
os.path.join(self.main_dir_path, file_name)
|
|
140
|
+
for file_name in os.listdir(self.main_dir_path)
|
|
141
|
+
]
|
|
142
|
+
filtered_files = []
|
|
143
|
+
for file_path in all_files_to_read:
|
|
144
|
+
if os.path.isdir(file_path):
|
|
145
|
+
continue
|
|
146
|
+
ext = file_path.split(".")[-1].lower().strip()
|
|
147
|
+
if ext not in self.accept_extensions:
|
|
148
|
+
logging.warning("Skipped file {}".format(file_path))
|
|
149
|
+
continue
|
|
150
|
+
filtered_files.append(file_path)
|
|
151
|
+
return filtered_files
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def _load_document_mt(document: Document):
|
|
155
|
+
"""
|
|
156
|
+
Document to load into thread, object of Document may be returned
|
|
157
|
+
by this method because may be pickled!
|
|
158
|
+
:param document: Document to load
|
|
159
|
+
:return: Loaded document
|
|
160
|
+
"""
|
|
161
|
+
document.load()
|
|
162
|
+
return document
|
|
163
|
+
|
|
164
|
+
def _load_mt(
|
|
165
|
+
self,
|
|
166
|
+
prepare_proper_pages: bool = False,
|
|
167
|
+
merge_document_pages: bool = False,
|
|
168
|
+
clear_texts: bool = False,
|
|
169
|
+
use_text_denoiser: bool = False,
|
|
170
|
+
max_tokens_in_chunk: int = None,
|
|
171
|
+
number_of_overlap_tokens: int = None,
|
|
172
|
+
check_text_language: bool = False,
|
|
173
|
+
) -> None:
|
|
174
|
+
pool = Pool(self._processes_count)
|
|
175
|
+
async_loading = []
|
|
176
|
+
for file_path in self._files_in_directory:
|
|
177
|
+
doc = Document(
|
|
178
|
+
file_path=file_path,
|
|
179
|
+
relative_file_path=self._relative_file_path(file_path),
|
|
180
|
+
prepare_proper_pages=prepare_proper_pages,
|
|
181
|
+
merge_document_pages=merge_document_pages,
|
|
182
|
+
clear_texts=clear_texts,
|
|
183
|
+
use_text_denoiser=use_text_denoiser,
|
|
184
|
+
max_tokens_in_chunk=max_tokens_in_chunk,
|
|
185
|
+
number_of_overlap_tokens=number_of_overlap_tokens,
|
|
186
|
+
check_text_language=check_text_language,
|
|
187
|
+
)
|
|
188
|
+
async_loading.append(pool.apply_async(self._load_document_mt, args=(doc,)))
|
|
189
|
+
with tqdm.tqdm(total=len(async_loading), desc="Loading files") as pbar:
|
|
190
|
+
for worker in async_loading:
|
|
191
|
+
try:
|
|
192
|
+
wd = worker.get()
|
|
193
|
+
self._all_documents_in_directory.append(wd)
|
|
194
|
+
except Exception as error:
|
|
195
|
+
logging.warning(f"Problem with loading file: {error}")
|
|
196
|
+
pbar.update()
|
|
197
|
+
pool.close()
|
|
198
|
+
pool.join()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from radlab_data.text.processors.mappers import RawSentencesToTextMapper
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TextUtils:
|
|
5
|
+
"""
|
|
6
|
+
TextUtils like sentences to text position mapping.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
@staticmethod
|
|
10
|
+
def map_sentences_to_text(
|
|
11
|
+
sentences,
|
|
12
|
+
text,
|
|
13
|
+
remove_dot_when_mapping: bool = False,
|
|
14
|
+
exact_sentence_match: bool = True,
|
|
15
|
+
):
|
|
16
|
+
"""
|
|
17
|
+
Map the sentences to the text positions. Each sentence is described
|
|
18
|
+
as begin and end position. As the sentence mapper is used
|
|
19
|
+
`RawSentencesToTextMapper` from `radlab_data.text.processors.mappers`
|
|
20
|
+
:param sentences: List of the raw sentences to map
|
|
21
|
+
:param text: Raw text on which the sentences will be mapped
|
|
22
|
+
:param remove_dot_when_mapping: When sentence mapping is not found then
|
|
23
|
+
additionally will be done mapping without dot at the sentence end
|
|
24
|
+
:param exact_sentence_match: Exact sentence on text mapping
|
|
25
|
+
:return:
|
|
26
|
+
"""
|
|
27
|
+
return RawSentencesToTextMapper(
|
|
28
|
+
remove_dot_when_mapping=remove_dot_when_mapping,
|
|
29
|
+
exact_sentence_match=exact_sentence_match,
|
|
30
|
+
).apply(sentences, text)
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def text_language(text_str: str, probs: bool = False):
|
|
34
|
+
from ftlangdetect import detect
|
|
35
|
+
|
|
36
|
+
text_str = text_str.replace("\n", " ")
|
|
37
|
+
result = detect(text=text_str, low_memory=False)
|
|
38
|
+
return result if probs else result["lang"]
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def translate_text_deepl(text_str: str, target_lang: str, auth_key: str) -> str:
|
|
42
|
+
"""
|
|
43
|
+
Translates text into the target language.
|
|
44
|
+
Target must be an ISO 639-1 language code.
|
|
45
|
+
See https://g.co/cloud/translate/v2/translate-reference#supported_languages
|
|
46
|
+
"""
|
|
47
|
+
import deepl
|
|
48
|
+
|
|
49
|
+
translator = deepl.Translator(auth_key)
|
|
50
|
+
result = translator.translate_text(text_str, target_lang=target_lang.upper())
|
|
51
|
+
return result.text
|
|
File without changes
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
from urllib.parse import urljoin
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class APIHandler:
|
|
9
|
+
def __init__(
|
|
10
|
+
self,
|
|
11
|
+
base_api_url,
|
|
12
|
+
ping_check=False,
|
|
13
|
+
debug: bool = False,
|
|
14
|
+
raise_errors: bool = False,
|
|
15
|
+
) -> None:
|
|
16
|
+
"""
|
|
17
|
+
:param base_api_url: Base url of the API to call
|
|
18
|
+
:param ping_check: If set, the API is pinged when the handler is created
|
|
19
|
+
:param debug: If set, calls and responses are logged
|
|
20
|
+
:param raise_errors: If set, a failed call raises an exception instead
|
|
21
|
+
of returning None
|
|
22
|
+
"""
|
|
23
|
+
self._debug = debug
|
|
24
|
+
self._raise_errors = raise_errors
|
|
25
|
+
self.base_api_url = base_api_url
|
|
26
|
+
if ping_check:
|
|
27
|
+
self.call(endpoint="/ping")
|
|
28
|
+
|
|
29
|
+
def call(
|
|
30
|
+
self,
|
|
31
|
+
endpoint: str,
|
|
32
|
+
headers: dict | None = None,
|
|
33
|
+
params: dict | None = None,
|
|
34
|
+
method: str = "GET",
|
|
35
|
+
data: dict | None = None,
|
|
36
|
+
):
|
|
37
|
+
"""
|
|
38
|
+
Function to call an API with the given URL, headers, parameters, and method.
|
|
39
|
+
|
|
40
|
+
Parameters:
|
|
41
|
+
endpoint (str): The specific endpoint to call.
|
|
42
|
+
headers (dict, optional): Headers to be included in the request.
|
|
43
|
+
params (dict, optional): Parameters to be passed in the request.
|
|
44
|
+
method (str, optional): HTTP method to be used ('GET', 'POST', 'PUT', 'DELETE', etc.).
|
|
45
|
+
data (dict, optional): Data to be sent in the request body (for methods like POST or PUT).
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
dict: JSON response from the API if successful, otherwise returns None.
|
|
49
|
+
"""
|
|
50
|
+
ep_url = urljoin(self.base_api_url, endpoint)
|
|
51
|
+
if self._debug:
|
|
52
|
+
logging.debug(f"Calling {ep_url}")
|
|
53
|
+
try:
|
|
54
|
+
method = method.upper()
|
|
55
|
+
# Make the request based on the specified method
|
|
56
|
+
if method == "GET":
|
|
57
|
+
response = requests.get(ep_url, headers=headers, params=params)
|
|
58
|
+
elif method == "POST":
|
|
59
|
+
response = requests.post(
|
|
60
|
+
ep_url, headers=headers, params=params, json=data
|
|
61
|
+
)
|
|
62
|
+
elif method == "PUT":
|
|
63
|
+
response = requests.put(
|
|
64
|
+
ep_url, headers=headers, params=params, json=data
|
|
65
|
+
)
|
|
66
|
+
elif method == "DELETE":
|
|
67
|
+
response = requests.delete(ep_url, headers=headers, params=params)
|
|
68
|
+
else:
|
|
69
|
+
raise ValueError("Unsupported HTTP method:", method)
|
|
70
|
+
|
|
71
|
+
# Check if the request was successful
|
|
72
|
+
response.raise_for_status()
|
|
73
|
+
|
|
74
|
+
# Return the JSON response
|
|
75
|
+
if self._debug:
|
|
76
|
+
logging.debug(f"Got response from {ep_url}")
|
|
77
|
+
return response.json()
|
|
78
|
+
|
|
79
|
+
except requests.RequestException as error:
|
|
80
|
+
logging.warning(f"Error calling API {ep_url}: {error}")
|
|
81
|
+
if self._raise_errors:
|
|
82
|
+
raise
|
|
83
|
+
return None
|