radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,167 @@
1
+ import abc
2
+
3
+
4
+ class BaseMapper(abc.ABC):
5
+ def __init__(self):
6
+ self._obj1 = None
7
+ self._obj2 = None
8
+ self._mapping = None
9
+
10
+ @abc.abstractmethod
11
+ def apply(self, obj1, obj2) -> list:
12
+ """
13
+ Abstract method to apply mapping obj1 on obj2
14
+ :param obj1: Object to map
15
+ :param obj2: On this object obj1 will be mapped
16
+ :return: List (self._mapping) of mapping
17
+ """
18
+ pass
19
+
20
+
21
+ class RawSentencesToTextMapper(BaseMapper):
22
+ ESTIMATED_POSITION_LEN_CHARS_BIAS = 4
23
+
24
+ def __init__(
25
+ self,
26
+ simple: bool = True,
27
+ lower: bool = True,
28
+ remove_dot_when_mapping: bool = False,
29
+ exact_sentence_match: bool = True,
30
+ ):
31
+ """
32
+ Actually only simple mapping is available. Simple means, that only first
33
+ and the last sentence words are used to determine sentence position into
34
+ the whole text, text and the sentences are also lowered.
35
+ Mapping objects:
36
+ * self._obj1 have to be set as the list of raw sentences
37
+ * self._obj2 have to be set as the whole raw text
38
+ :param simple - if set to True, then simple mapping will be used
39
+ :param lower - if set to True, then compared texts will be converted to lower
40
+ :param remove_dot_when_mapping - In case when mapped sentence is not found
41
+ then mapping without dot at sentence end will be done
42
+ :param exact_sentence_match - If set to True, then exact sentence
43
+ will be match instead of heuristic
44
+ :exception - Raise Exception when not simple method will be used.
45
+ """
46
+ if not simple:
47
+ raise Exception("Only simple mapping is supporting now!")
48
+ super().__init__()
49
+ self._lower = lower
50
+ self._simple_map = simple
51
+ self._exact_sentence_match = exact_sentence_match
52
+ self._remove_dot_when_mapping = remove_dot_when_mapping
53
+
54
+ def apply(self, sentences: list, text: str):
55
+ self._obj1 = sentences
56
+ self._obj2 = text
57
+ self._mapping = []
58
+
59
+ end_position = 0
60
+ for s in sentences:
61
+ begin_position, end_position = self.map_sentence_to_text(
62
+ s, text, end_position
63
+ )
64
+ self._mapping.append([begin_position, end_position])
65
+ return self._mapping
66
+
67
+ def map_sentence_to_text(
68
+ self, sentence: str, text: str, begin_position: int
69
+ ) -> (int, int):
70
+ """
71
+ Map the single sentence on the given text, mapping procedure starts
72
+ from given begin_position. If begin_position is smaller than 0,
73
+ then mapping will not be done (assert).
74
+
75
+ :param sentence:
76
+ :param text:
77
+ :param begin_position:
78
+ :return:
79
+ """
80
+ assert begin_position >= 0, "Begin position cannot be smaller than 0!"
81
+ if self._exact_sentence_match:
82
+ return self.__exact_map_sentence_to_text(
83
+ sentence=sentence, text=text, begin_position=begin_position
84
+ )
85
+
86
+ _epos = (
87
+ begin_position + len(sentence) + self.ESTIMATED_POSITION_LEN_CHARS_BIAS
88
+ )
89
+ if _epos > len(text):
90
+ _epos = len(text)
91
+
92
+ begin_position -= 1
93
+ if begin_position < 0:
94
+ begin_position = 0
95
+
96
+ context = text[begin_position:_epos]
97
+ if self._lower:
98
+ context = context.lower()
99
+ sentence = sentence.lower()
100
+
101
+ spl_sent = sentence.split()
102
+ w_first, w_last = spl_sent[0], spl_sent[-1]
103
+ b_pos = self.__find_word_position(w_first, context, 0, None)
104
+ e_pos = self.__find_word_position(
105
+ word=w_last,
106
+ context=context,
107
+ start_position=b_pos,
108
+ estimated_len=len(sentence),
109
+ )
110
+
111
+ if e_pos == -1 and self._remove_dot_when_mapping:
112
+ if w_last[-1] == ".":
113
+ w_last = w_last[:-1]
114
+ e_pos = self.__find_word_position(
115
+ word=w_last,
116
+ context=context,
117
+ start_position=b_pos,
118
+ estimated_len=len(sentence),
119
+ )
120
+ return begin_position + b_pos, e_pos + begin_position
121
+
122
+ def __exact_map_sentence_to_text(
123
+ self, sentence: str, text: str, begin_position: int
124
+ ) -> (int, int):
125
+ """
126
+ Map the single sentence on the given text, mapping procedure starts
127
+ from given begin_position. If begin_position is smaller than 0,
128
+ then mapping will not be done (assert).
129
+
130
+ :param sentence:
131
+ :param text:
132
+ :param begin_position:
133
+ :return:
134
+ """
135
+ b_position = begin_position - 5
136
+ if b_position < 0:
137
+ b_position = 0
138
+ find_sentence = sentence
139
+ context = text[b_position:]
140
+ if self._lower:
141
+ context = context.lower()
142
+ find_sentence = find_sentence.lower()
143
+ find_position = context.lower().find(find_sentence.lower())
144
+ if find_position < 0:
145
+ return 0, len(sentence)
146
+ start_position = b_position + find_position
147
+ end_position = start_position + len(sentence)
148
+ return start_position, end_position
149
+
150
+ def __find_word_position(
151
+ self,
152
+ word: str,
153
+ context: str,
154
+ start_position: int,
155
+ estimated_len: int = None,
156
+ ):
157
+ if estimated_len is None:
158
+ return context[start_position:].find(word)
159
+
160
+ w_pos = context.rfind(word)
161
+ w_sent_len = (w_pos - start_position) + len(word)
162
+ if w_sent_len < 0:
163
+ return -1
164
+
165
+ if abs(estimated_len - w_sent_len) <= self.ESTIMATED_POSITION_LEN_CHARS_BIAS:
166
+ return w_pos + len(word)
167
+ return -1
@@ -0,0 +1,58 @@
1
+ T_REPL = "to_replace"
2
+ REPL_W = "replace_with"
3
+
4
+ # Some general regext to apply to get first stage of clear text
5
+ GENERAL_CLEANERS_REGEX = {
6
+ "repl_space_quot_space_to_comma": {
7
+ T_REPL: r" \? ",
8
+ REPL_W: ", ",
9
+ },
10
+ "repl_colon_comma_to_colon": {T_REPL: r":,", REPL_W: ":"},
11
+ "repl_colon_quote_to_colon": {T_REPL: r": \?", REPL_W: ":"},
12
+ "repl_quote_dot_to_dot": {T_REPL: r"\? \.", REPL_W: "."},
13
+ "repl_quote_dot_to_dot2": {T_REPL: r"\?\.", REPL_W: "."},
14
+ "repl_comma_dash_to_comma": {T_REPL: r", \-", REPL_W: ","},
15
+ "repl_dd_comma_to_comma": {T_REPL: r", ,", REPL_W: ","},
16
+ "repl_d_comma_to_comma": {T_REPL: r",,", REPL_W: ","},
17
+ }
18
+
19
+ # Abbreviations replacement
20
+ ABBREV_REPLACE_MAP = [
21
+ ("np.", "np_"),
22
+ ("ul.", "ul_"),
23
+ ("tj.", "tj_"),
24
+ ("inż.", "inż_"),
25
+ ("S.A.", "S_A_"),
26
+ ("s.a.", "s_a_"),
27
+ ]
28
+
29
+ # www/url
30
+ REMOVE_URL = [
31
+ (
32
+ "((http|https)\:\/\/)?[a-zA-Z0-9\.\/\?\:@\-_=#]"
33
+ "+\.([a-zA-Z]){2,6}([a-zA-Z0-9\.\&\/\?\:@\-_=#])*",
34
+ "WWW_URL",
35
+ ),
36
+ ]
37
+
38
+ # Add new line in some cases
39
+ ADD_NEW_LINE_IN_CASE = {
40
+ "repl_beg_chat_num": {T_REPL: r"^\(\d+\)", REPL_W: ""},
41
+ "repl_in_chat_num": {T_REPL: r"\(\d+\)", REPL_W: "\n"},
42
+ "repl_dot_newline1": {
43
+ T_REPL: r'(\s[A-zęąŃńćĆóÓźŹżŻśŚłŁ"()]+[0-9]{0,20}\.)',
44
+ REPL_W: r"\1\n\n",
45
+ },
46
+ "repl_dot_newline2": {T_REPL: r"(\s[0-9]+\.)", REPL_W: r"\1\n\n"},
47
+ "repl_dot_newline3": {T_REPL: r"(\s[A-Z]+\.)", REPL_W: r"\1\n\n"},
48
+ "repl_char_eum_in": {T_REPL: r"\s[a-z]\)", REPL_W: "\n\n"},
49
+ }
50
+
51
+ # These characters may occur on the sentence end
52
+ SENTENCE_END_CHARS = "!@#$%^&*()_-+={[}];:'\"<,>./?"
53
+
54
+ # Enumerations
55
+ ENUMERATION_REGEX = {
56
+ "enum_dot_standard": {T_REPL: r"(\s)[0-9]\.\s", REPL_W: r"\1"},
57
+ "enum_round_brackets": {T_REPL: r"(\s)[0-9]\)\s", REPL_W: r"\1"},
58
+ }
@@ -0,0 +1,60 @@
1
+ import spacy
2
+
3
+ # The spaCy model is loaded lazily on the first use, because importing this
4
+ # module (for example to use only the pair reader) should not require the
5
+ # Polish model to be installed and should not take several seconds.
6
+ MODEL_NAME = "pl_core_news_lg"
7
+ _nlp = None
8
+
9
+
10
+ def get_nlp():
11
+ """
12
+ Return the spaCy pipeline used for sentence splitting. The model is
13
+ loaded on the first call and cached for the next ones.
14
+ :return: Loaded spaCy pipeline
15
+ """
16
+ global _nlp
17
+ if _nlp is None:
18
+ _nlp = spacy.load(MODEL_NAME)
19
+ return _nlp
20
+
21
+
22
+ class SentenceSplitter:
23
+ min_sentence_len = 1
24
+
25
+ def split(
26
+ self,
27
+ text: str,
28
+ return_stats_dict: bool = False,
29
+ split_words_with_ws: bool = True,
30
+ ):
31
+ """
32
+ Split text to sentences and optionally return statistics
33
+
34
+ :param text: Text given as string to split to sentences
35
+ :param return_stats_dict: If set, then dictionary with stats will be returned
36
+ :param split_words_with_ws: If set, then number of words will be calculated
37
+ using standard str.split() method
38
+ :return: List of sentences (+ optionally statistics dict)
39
+ """
40
+ doc = get_nlp()(text)
41
+
42
+ sentences = []
43
+ text_stats = {
44
+ "number_of_sentences": 0,
45
+ "number_of_words": 0,
46
+ "number_of_characters": len(text),
47
+ }
48
+
49
+ for s in doc.sents:
50
+ if return_stats_dict:
51
+ text_stats["number_of_sentences"] += 1
52
+ if split_words_with_ws:
53
+ text_stats["number_of_words"] += len(s.text.strip().split())
54
+ else:
55
+ text_stats["number_of_words"] += len(s)
56
+
57
+ sentence = s.text.strip()
58
+ if len(sentence.split()) >= self.min_sentence_len:
59
+ sentences.append(sentence)
60
+ return (sentences, text_stats) if return_stats_dict else sentences
@@ -0,0 +1,198 @@
1
+ import os
2
+ from multiprocessing import Pool
3
+
4
+ import tqdm
5
+ import pathlib
6
+ import logging
7
+
8
+ from radlab_data.text.document import Document
9
+ from radlab_data.text.loaders.config import AVAILABLE_FILE_EXTENSIONS
10
+
11
+
12
+ class DirectoryFileReader:
13
+ def __init__(
14
+ self,
15
+ main_dir_path: str,
16
+ read_sub_dirs: bool = True,
17
+ accept_extensions: list = None,
18
+ processes_count: int = None,
19
+ ):
20
+ """
21
+ :param main_dir_path: Path to main directory to load files
22
+ :param read_sub_dirs: If set to True, then will be done listing
23
+ all files in main_dir_path with subdirectories
24
+ :param accept_extensions: List of accepted extension, defaults to
25
+ `AVAILABLE_FILE_EXTENSIONS`
26
+ :param processes_count: If number of processes is greater than 1,
27
+ will use given number of processes to load listed files.
28
+ """
29
+ self.main_dir_path = main_dir_path
30
+ self.deep_read = read_sub_dirs
31
+ self.accept_extensions = (
32
+ list(AVAILABLE_FILE_EXTENSIONS)
33
+ if accept_extensions is None
34
+ else accept_extensions
35
+ )
36
+
37
+ self._processes_count = processes_count
38
+ self._use_multiprocessing = processes_count is not None and processes_count > 1
39
+ self._files_in_directory = self._list_files()
40
+ self._all_documents_in_directory = []
41
+
42
+ @property
43
+ def documents(self) -> list:
44
+ return self._all_documents_in_directory
45
+
46
+ def load(
47
+ self,
48
+ prepare_proper_pages: bool = False,
49
+ merge_document_pages: bool = False,
50
+ clear_texts: bool = False,
51
+ use_text_denoiser: bool = False,
52
+ max_tokens_in_chunk: int = None,
53
+ number_of_overlap_tokens: int = None,
54
+ check_text_language: bool = False,
55
+ ) -> None:
56
+ """
57
+ Loads all documents listed in `self._files_in_directory`
58
+ :param prepare_proper_pages: IF set to True, then all texts/sentences
59
+ from the same page will be merged to single page.
60
+ :param merge_document_pages: If set to True then all pages from document
61
+ will be merged to single page, then document will contain single page.
62
+ :param clear_texts: If set to True, then each text will be clear
63
+ :param use_text_denoiser: If set to True, then each text will be denoised
64
+ :param max_tokens_in_chunk: If given then each text will be split
65
+ to chunks with given max number of tokens in single chunk
66
+ :param number_of_overlap_tokens: If given then each generated chunk
67
+ will contain number_of_overlap_tokens from the previous chunk.
68
+ :param check_text_language: If options is set to True, then language
69
+ of each text will be checked and stored to metadata
70
+ :return: None
71
+ """
72
+ logging.info(f"processes_count={self._processes_count}")
73
+ logging.info(f"prepare_proper_pages={prepare_proper_pages}")
74
+ logging.info(f"merge_document_pages={merge_document_pages}")
75
+ logging.info(f"clear_texts={clear_texts}")
76
+ logging.info(f"use_text_denoiser={use_text_denoiser}")
77
+ logging.info(f"check_text_language={check_text_language}")
78
+ logging.info(f"max_tokens_in_chunk={max_tokens_in_chunk}")
79
+ logging.info(f"number_of_overlap_tokens={number_of_overlap_tokens}")
80
+
81
+ if self._use_multiprocessing:
82
+ return self._load_mt(
83
+ prepare_proper_pages=prepare_proper_pages,
84
+ merge_document_pages=merge_document_pages,
85
+ clear_texts=clear_texts,
86
+ use_text_denoiser=use_text_denoiser,
87
+ max_tokens_in_chunk=max_tokens_in_chunk,
88
+ number_of_overlap_tokens=number_of_overlap_tokens,
89
+ check_text_language=check_text_language,
90
+ )
91
+
92
+ with tqdm.tqdm(
93
+ total=len(self._files_in_directory), desc="Loading files"
94
+ ) as pbar:
95
+ for file_path in self._files_in_directory:
96
+ try:
97
+ doc = Document(
98
+ file_path=file_path,
99
+ relative_file_path=self._relative_file_path(file_path),
100
+ prepare_proper_pages=prepare_proper_pages,
101
+ merge_document_pages=merge_document_pages,
102
+ clear_texts=clear_texts,
103
+ use_text_denoiser=use_text_denoiser,
104
+ max_tokens_in_chunk=max_tokens_in_chunk,
105
+ number_of_overlap_tokens=number_of_overlap_tokens,
106
+ check_text_language=check_text_language,
107
+ )
108
+ doc.load()
109
+ self._all_documents_in_directory.append(doc)
110
+ except Exception:
111
+ logging.warning(
112
+ f"Problem with loading file {file_path}", exc_info=True
113
+ )
114
+ pbar.update()
115
+
116
+ def _relative_file_path(self, file_path: str) -> str:
117
+ """
118
+ Path to the file relative to the main directory path.
119
+ :param file_path: Path to the loaded file
120
+ :return: Relative path, the original path when it is not located
121
+ inside the main directory
122
+ """
123
+ try:
124
+ return os.path.relpath(file_path, self.main_dir_path)
125
+ except ValueError:
126
+ return file_path
127
+
128
+ def _list_files(self) -> list:
129
+ """
130
+ Depends on the self.deep_read parameter, reads all files in main_dir_path
131
+ and subdirectories when set to True, otherwise reads only from main directory.
132
+ :return: List of paths to files to read
133
+ """
134
+ if self.deep_read:
135
+ all_files_to_read = pathlib.Path(self.main_dir_path)
136
+ all_files_to_read = [str(d) for d in all_files_to_read.rglob("*")]
137
+ else:
138
+ all_files_to_read = [
139
+ os.path.join(self.main_dir_path, file_name)
140
+ for file_name in os.listdir(self.main_dir_path)
141
+ ]
142
+ filtered_files = []
143
+ for file_path in all_files_to_read:
144
+ if os.path.isdir(file_path):
145
+ continue
146
+ ext = file_path.split(".")[-1].lower().strip()
147
+ if ext not in self.accept_extensions:
148
+ logging.warning("Skipped file {}".format(file_path))
149
+ continue
150
+ filtered_files.append(file_path)
151
+ return filtered_files
152
+
153
+ @staticmethod
154
+ def _load_document_mt(document: Document):
155
+ """
156
+ Document to load into thread, object of Document may be returned
157
+ by this method because may be pickled!
158
+ :param document: Document to load
159
+ :return: Loaded document
160
+ """
161
+ document.load()
162
+ return document
163
+
164
+ def _load_mt(
165
+ self,
166
+ prepare_proper_pages: bool = False,
167
+ merge_document_pages: bool = False,
168
+ clear_texts: bool = False,
169
+ use_text_denoiser: bool = False,
170
+ max_tokens_in_chunk: int = None,
171
+ number_of_overlap_tokens: int = None,
172
+ check_text_language: bool = False,
173
+ ) -> None:
174
+ pool = Pool(self._processes_count)
175
+ async_loading = []
176
+ for file_path in self._files_in_directory:
177
+ doc = Document(
178
+ file_path=file_path,
179
+ relative_file_path=self._relative_file_path(file_path),
180
+ prepare_proper_pages=prepare_proper_pages,
181
+ merge_document_pages=merge_document_pages,
182
+ clear_texts=clear_texts,
183
+ use_text_denoiser=use_text_denoiser,
184
+ max_tokens_in_chunk=max_tokens_in_chunk,
185
+ number_of_overlap_tokens=number_of_overlap_tokens,
186
+ check_text_language=check_text_language,
187
+ )
188
+ async_loading.append(pool.apply_async(self._load_document_mt, args=(doc,)))
189
+ with tqdm.tqdm(total=len(async_loading), desc="Loading files") as pbar:
190
+ for worker in async_loading:
191
+ try:
192
+ wd = worker.get()
193
+ self._all_documents_in_directory.append(wd)
194
+ except Exception as error:
195
+ logging.warning(f"Problem with loading file: {error}")
196
+ pbar.update()
197
+ pool.close()
198
+ pool.join()
@@ -0,0 +1,51 @@
1
+ from radlab_data.text.processors.mappers import RawSentencesToTextMapper
2
+
3
+
4
+ class TextUtils:
5
+ """
6
+ TextUtils like sentences to text position mapping.
7
+ """
8
+
9
+ @staticmethod
10
+ def map_sentences_to_text(
11
+ sentences,
12
+ text,
13
+ remove_dot_when_mapping: bool = False,
14
+ exact_sentence_match: bool = True,
15
+ ):
16
+ """
17
+ Map the sentences to the text positions. Each sentence is described
18
+ as begin and end position. As the sentence mapper is used
19
+ `RawSentencesToTextMapper` from `radlab_data.text.processors.mappers`
20
+ :param sentences: List of the raw sentences to map
21
+ :param text: Raw text on which the sentences will be mapped
22
+ :param remove_dot_when_mapping: When sentence mapping is not found then
23
+ additionally will be done mapping without dot at the sentence end
24
+ :param exact_sentence_match: Exact sentence on text mapping
25
+ :return:
26
+ """
27
+ return RawSentencesToTextMapper(
28
+ remove_dot_when_mapping=remove_dot_when_mapping,
29
+ exact_sentence_match=exact_sentence_match,
30
+ ).apply(sentences, text)
31
+
32
+ @staticmethod
33
+ def text_language(text_str: str, probs: bool = False):
34
+ from ftlangdetect import detect
35
+
36
+ text_str = text_str.replace("\n", " ")
37
+ result = detect(text=text_str, low_memory=False)
38
+ return result if probs else result["lang"]
39
+
40
+ @staticmethod
41
+ def translate_text_deepl(text_str: str, target_lang: str, auth_key: str) -> str:
42
+ """
43
+ Translates text into the target language.
44
+ Target must be an ISO 639-1 language code.
45
+ See https://g.co/cloud/translate/v2/translate-reference#supported_languages
46
+ """
47
+ import deepl
48
+
49
+ translator = deepl.Translator(auth_key)
50
+ result = translator.translate_text(text_str, target_lang=target_lang.upper())
51
+ return result.text
File without changes
@@ -0,0 +1,83 @@
1
+ import logging
2
+
3
+ import requests
4
+
5
+ from urllib.parse import urljoin
6
+
7
+
8
+ class APIHandler:
9
+ def __init__(
10
+ self,
11
+ base_api_url,
12
+ ping_check=False,
13
+ debug: bool = False,
14
+ raise_errors: bool = False,
15
+ ) -> None:
16
+ """
17
+ :param base_api_url: Base url of the API to call
18
+ :param ping_check: If set, the API is pinged when the handler is created
19
+ :param debug: If set, calls and responses are logged
20
+ :param raise_errors: If set, a failed call raises an exception instead
21
+ of returning None
22
+ """
23
+ self._debug = debug
24
+ self._raise_errors = raise_errors
25
+ self.base_api_url = base_api_url
26
+ if ping_check:
27
+ self.call(endpoint="/ping")
28
+
29
+ def call(
30
+ self,
31
+ endpoint: str,
32
+ headers: dict | None = None,
33
+ params: dict | None = None,
34
+ method: str = "GET",
35
+ data: dict | None = None,
36
+ ):
37
+ """
38
+ Function to call an API with the given URL, headers, parameters, and method.
39
+
40
+ Parameters:
41
+ endpoint (str): The specific endpoint to call.
42
+ headers (dict, optional): Headers to be included in the request.
43
+ params (dict, optional): Parameters to be passed in the request.
44
+ method (str, optional): HTTP method to be used ('GET', 'POST', 'PUT', 'DELETE', etc.).
45
+ data (dict, optional): Data to be sent in the request body (for methods like POST or PUT).
46
+
47
+ Returns:
48
+ dict: JSON response from the API if successful, otherwise returns None.
49
+ """
50
+ ep_url = urljoin(self.base_api_url, endpoint)
51
+ if self._debug:
52
+ logging.debug(f"Calling {ep_url}")
53
+ try:
54
+ method = method.upper()
55
+ # Make the request based on the specified method
56
+ if method == "GET":
57
+ response = requests.get(ep_url, headers=headers, params=params)
58
+ elif method == "POST":
59
+ response = requests.post(
60
+ ep_url, headers=headers, params=params, json=data
61
+ )
62
+ elif method == "PUT":
63
+ response = requests.put(
64
+ ep_url, headers=headers, params=params, json=data
65
+ )
66
+ elif method == "DELETE":
67
+ response = requests.delete(ep_url, headers=headers, params=params)
68
+ else:
69
+ raise ValueError("Unsupported HTTP method:", method)
70
+
71
+ # Check if the request was successful
72
+ response.raise_for_status()
73
+
74
+ # Return the JSON response
75
+ if self._debug:
76
+ logging.debug(f"Got response from {ep_url}")
77
+ return response.json()
78
+
79
+ except requests.RequestException as error:
80
+ logging.warning(f"Error calling API {ep_url}: {error}")
81
+ if self._raise_errors:
82
+ raise
83
+ return None