radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,111 @@
1
+ import re
2
+
3
+ from bs4 import BeautifulSoup
4
+
5
+ from radlab_data.text.loaders.base import DocumentPage, read_text_file
6
+ from radlab_data.text.loaders.loader_i import LoaderI
7
+
8
+
9
+ class HTMLLoader(LoaderI):
10
+ """
11
+ Loader for HTML files. All markup which does not contribute to the
12
+ readable content (scripts, styles, metadata, ...) is removed and the
13
+ cleaned text is exposed as one or many document pages.
14
+
15
+ Recognised options:
16
+ * `split_by_paragraph`: when set to True, each paragraph (a fragment
17
+ separated by a blank line) becomes a separate document page. Without
18
+ this option the whole file is loaded as a single page.
19
+ """
20
+
21
+ TAGS_TO_REMOVE = [
22
+ "script",
23
+ "style",
24
+ "noscript",
25
+ "svg",
26
+ "meta",
27
+ "link",
28
+ "head",
29
+ "footer",
30
+ "header",
31
+ ]
32
+
33
+ MULTIPLE_EMPTY_LINES_RE = re.compile(r"\n\s*\n+")
34
+ MULTIPLE_SPACES_RE = re.compile(r"[ \t]+")
35
+
36
+ def __init__(
37
+ self,
38
+ filepath: str,
39
+ options: dict = None,
40
+ encoding: str = "utf-8",
41
+ autodetect_encoding: bool = True,
42
+ ) -> None:
43
+ """
44
+ :param filepath: Path to html file
45
+ :param options: Options shared with other loaders (see `LoaderI`),
46
+ see the class docstring for the loader specific options
47
+ :param encoding: Encoding used to read the file
48
+ :param autodetect_encoding: If the file cannot be decoded with
49
+ `encoding`, encodings from `FALLBACK_ENCODINGS` are tried
50
+ """
51
+ LoaderI.__init__(self, filepath=filepath, options=options)
52
+ self.encoding = encoding
53
+ self.autodetect_encoding = autodetect_encoding
54
+
55
+ def load(self) -> list:
56
+ """
57
+ Load the html file and remove all its markup.
58
+ :return: List of loaded document pages
59
+ """
60
+ if self.doc_pages:
61
+ return self.doc_pages
62
+
63
+ clean_text = self._clean_html(
64
+ read_text_file(
65
+ self.filepath,
66
+ encoding=self.encoding,
67
+ autodetect_encoding=self.autodetect_encoding,
68
+ )
69
+ )
70
+
71
+ if not self.options.get("split_by_paragraph", False):
72
+ self.doc_pages = [
73
+ DocumentPage(
74
+ page_content=clean_text,
75
+ metadata={"source": str(self.filepath)},
76
+ )
77
+ ]
78
+ return self.doc_pages
79
+
80
+ paragraph_number = 0
81
+ for paragraph in clean_text.split("\n\n"):
82
+ if not paragraph.strip():
83
+ continue
84
+ paragraph_number += 1
85
+ self.doc_pages.append(
86
+ DocumentPage(
87
+ page_content=paragraph.strip(),
88
+ metadata={
89
+ "source": str(self.filepath),
90
+ "paragraph": paragraph_number,
91
+ },
92
+ )
93
+ )
94
+ return self.doc_pages
95
+
96
+ def _clean_html(self, raw_html: str) -> str:
97
+ """
98
+ Remove tags which are not a content of the document and normalize
99
+ whitespace. Paragraph breaks are preserved, multiple spaces and
100
+ tabs are reduced to a single space.
101
+ :param raw_html: Raw html content
102
+ :return: Cleaned, readable text
103
+ """
104
+ soup = BeautifulSoup(raw_html, "html.parser")
105
+ for tag in soup(self.TAGS_TO_REMOVE):
106
+ tag.decompose()
107
+
108
+ text = soup.get_text(separator="\n")
109
+ text = self.MULTIPLE_EMPTY_LINES_RE.sub("\n\n", text)
110
+ text = self.MULTIPLE_SPACES_RE.sub(" ", text)
111
+ return text.strip()
@@ -0,0 +1,30 @@
1
+ from radlab_data.text.loaders.base import DocumentPage
2
+
3
+ from radlab_data.text.loaders.loader_i import LoaderI
4
+
5
+
6
+ class InputTextLoader(LoaderI):
7
+ """
8
+ Loads given text string as Document
9
+ """
10
+
11
+ DOCUMENT_CONTENT_VARIABLE = "document_content"
12
+
13
+ def __init__(self, filepath: str, options: dict = None) -> None:
14
+ """
15
+ :param filepath: Filename extension have to be: `input_text`
16
+ """
17
+ LoaderI.__init__(self, filepath=filepath, options=options)
18
+
19
+ if self.DOCUMENT_CONTENT_VARIABLE not in options:
20
+ raise Exception(
21
+ f"No {self.DOCUMENT_CONTENT_VARIABLE} in "
22
+ f"options while loading input_text!"
23
+ )
24
+ self._content = options[self.DOCUMENT_CONTENT_VARIABLE]
25
+
26
+ def load(self):
27
+ self.doc_pages = [
28
+ DocumentPage(page_content=self._content, metadata={"page": 1})
29
+ ]
30
+ return self.doc_pages
@@ -0,0 +1,267 @@
1
+ import sys
2
+ import abc
3
+ import copy
4
+ from functools import lru_cache
5
+
6
+ from radlab_data.text.utils import TextUtils
7
+ from radlab_data.text.loaders.config import TOKENIZER_PATH
8
+ from radlab_data.text.processors.cleaner import FullChainProcessor
9
+
10
+
11
+ @lru_cache(maxsize=4)
12
+ def get_tokenizer(tokenizer_path: str = None):
13
+ """
14
+ Return the tokenizer for the given path. Tokenizers are cached, because
15
+ loading the same tokenizer over and over again (once per loaded document)
16
+ is a significant and completely avoidable overhead.
17
+
18
+ The ``transformers`` import happens here, not at module import time, so the
19
+ core document-loading parts of ``radlab_data`` can be imported on hosts that
20
+ do not carry the model stack.
21
+ :param tokenizer_path: Path or name of the tokenizer, defaults to
22
+ `config.TOKENIZER_PATH`
23
+ :return: Loaded tokenizer with the model max length limitation removed
24
+ """
25
+ if tokenizer_path is None:
26
+ tokenizer_path = TOKENIZER_PATH
27
+ from transformers import AutoTokenizer
28
+
29
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
30
+ tokenizer.model_max_length = sys.maxsize
31
+ return tokenizer
32
+
33
+
34
+ class LoaderI(abc.ABC):
35
+ def __init__(self, filepath: str, options: dict = None) -> None:
36
+ """
37
+ :param filepath: Path to the loaded file
38
+ :param options: Options shared with the concrete document loader
39
+ """
40
+ self.filepath = filepath
41
+ self.options = {} if not options else options
42
+ self.doc_pages = []
43
+ self.text_chunk_type = None
44
+ self._text_cleaner = FullChainProcessor(phrases_to_remove=[])
45
+ self._tokenizer_instance = None
46
+
47
+ self._check_language = self.options.get("check_text_language", False)
48
+ if self._check_language is None:
49
+ self._check_language = False
50
+
51
+ self._resolve_text_chunk_type()
52
+
53
+ @property
54
+ def _tokenizer(self):
55
+ """
56
+ Lazily load the tokenizer on first access. Loaders only need it when
57
+ chunking pages by tokens, so merely constructing (or importing) a
58
+ loader no longer requires ``transformers`` to be installed.
59
+ :return: Tokenizer used for token-level chunking
60
+ """
61
+ if self._tokenizer_instance is None:
62
+ self._tokenizer_instance = get_tokenizer()
63
+ return self._tokenizer_instance
64
+
65
+ def _resolve_text_chunk_type(self):
66
+ sep = ""
67
+ self.text_chunk_type = "raw_text"
68
+ if "clear_texts" in self.options:
69
+ clr = self.options.get("clear_texts", False)
70
+ if clr is not None and clr:
71
+ self.text_chunk_type = "clear_texts"
72
+ if "use_text_denoiser" in self.options:
73
+ utd = self.options.get("use_text_denoiser", False)
74
+ if utd is not None and utd:
75
+ sep = "_"
76
+ self.text_chunk_type += f"{sep}denoised"
77
+ if "merge_to_proper_pages" in self.options:
78
+ mp = self.options.get("merge_to_proper_pages", False)
79
+ if mp is not None and mp:
80
+ sep = "_"
81
+ self.text_chunk_type += f"{sep}proper_page"
82
+ if "merge_document_pages" in self.options:
83
+ md = self.options.get("merge_document_pages", False)
84
+ if md is not None and md:
85
+ sep = "_"
86
+ self.text_chunk_type += f"{sep}merged_document"
87
+ if "max_tokens_in_chunk" in self.options:
88
+ mt = self.options.get("max_tokens_in_chunk", 0)
89
+ if mt is not None and mt > 1:
90
+ sep = "_"
91
+ self.text_chunk_type += f"{sep}chunk_max_tokens_{mt}"
92
+
93
+ def as_dict(self) -> dict:
94
+ """
95
+ Return document as dictionary (json)
96
+ :return: Document converted to dictionary
97
+ """
98
+ relative_file_path = self.options.get("relative_file_path", self.filepath)
99
+ category = self.options.get("document_category", "")
100
+
101
+ doc_dict = {
102
+ "filepath": self.filepath,
103
+ "relative_filepath": relative_file_path,
104
+ "category": category,
105
+ "options": self.options,
106
+ "pages": [],
107
+ }
108
+ for doc_page in self.doc_pages:
109
+ if "text_chunk_type" not in doc_page.metadata:
110
+ doc_page.metadata["text_chunk_type"] = self.text_chunk_type
111
+ doc_dict["pages"].append(
112
+ {
113
+ "page_number": doc_page.metadata.get("page", 0),
114
+ "table_number": doc_page.metadata.get("table", 0),
115
+ "row_number": doc_page.metadata.get("row", 0),
116
+ "column_number": doc_page.metadata.get("column", 0),
117
+ "page_content": doc_page.page_content,
118
+ "metadata": doc_page.metadata,
119
+ }
120
+ )
121
+ return doc_dict
122
+
123
+ def merge_all_pages(self) -> None:
124
+ """
125
+ Of option to merging is set as true, all pages are
126
+ concatenated as single. Actual self.doc_pages is replaced
127
+ with list with the single page (whole concatenated document)
128
+ :return: None
129
+ """
130
+ if not self.options.get("merge_document_pages", False) or not len(
131
+ self.doc_pages
132
+ ):
133
+ return None
134
+
135
+ merged_pages = ""
136
+ for doc_page in self.doc_pages:
137
+ merged_pages += doc_page.page_content + "\n"
138
+ document_page = self.doc_pages[0]
139
+ document_page.metadata["page"] = 1
140
+ document_page.page_content = merged_pages
141
+ self.doc_pages = [document_page]
142
+
143
+ def clear_doc_pages(self) -> None:
144
+ """
145
+ If clear_texts options is given, then all pages will be cleared
146
+ using FullChainProcess cleaner. Cleaning is done in place.
147
+ :return: None
148
+ """
149
+ if not self.options.get("clear_texts", False):
150
+ return None
151
+
152
+ for doc_page in self.doc_pages:
153
+ doc_page.page_content = self._text_cleaner.apply(doc_page.page_content)
154
+
155
+ def split_to_chunks(self) -> None:
156
+ """
157
+ Each page will be split into chunks of given size. Additionally,
158
+ generated chunks will be overlapped if option is set.
159
+ :return: None
160
+ """
161
+ chunk_size = self.options.get("max_tokens_in_chunk", 0)
162
+ overlap_size = self.options.get("number_of_overlap_tokens", 0)
163
+ if overlap_size is not None and overlap_size > 0:
164
+ self.text_chunk_type += f"_overlap_{overlap_size}"
165
+
166
+ if chunk_size is None or chunk_size < 2:
167
+ return None
168
+
169
+ new_pages = []
170
+ for doc_page in self.doc_pages:
171
+ new_pages.extend(
172
+ self._split_page_to_chunks_max_tokens(
173
+ doc_page=doc_page,
174
+ chunk_size=chunk_size,
175
+ overlap_size=overlap_size,
176
+ )
177
+ )
178
+ self.doc_pages = new_pages
179
+
180
+ def check_language(self) -> None:
181
+ if not self._check_language:
182
+ return None
183
+
184
+ for doc_page in self.doc_pages:
185
+ doc_page.metadata["language"] = TextUtils.text_language(
186
+ text_str=doc_page.page_content, probs=False
187
+ )
188
+
189
+ @staticmethod
190
+ def chunks(tokenized_page, chunk_size, overlap_size):
191
+ step_size = chunk_size
192
+ if overlap_size is not None and overlap_size > 1:
193
+ step_size = chunk_size - overlap_size
194
+ for index in range(0, len(tokenized_page), step_size):
195
+ yield tokenized_page[index : index + chunk_size]
196
+
197
+ def _split_page_to_chunks_max_tokens(
198
+ self, doc_page, chunk_size, overlap_size
199
+ ) -> list:
200
+ """
201
+ Split page content to chunks of given max_tokens_in_chunk size
202
+ :param doc_page: Document to split
203
+ :param chunk_size: Chunk size (max number of tokens)
204
+ :param overlap_size: Number of tokens overlapping adjacent chunks
205
+ :return: List of objects of Document (deepcopy of doc_page
206
+ with tokenized and chunked document page content)
207
+ """
208
+ if chunk_size < 1:
209
+ return [doc_page]
210
+
211
+ # NOTE:
212
+ # This is not needed because tokenizer max model len is set as sys.maxsize
213
+ # If this option will not be available, teb uncomment:
214
+ # chunked_page_content_str = self._split_page_content_to_max_model_len(
215
+ # doc_page.page_content
216
+ # )
217
+ # ...and comment the line below
218
+ chunked_page_content_str = [doc_page.page_content]
219
+
220
+ all_chunks = []
221
+ chunk_number = 1
222
+ for chunk_str in chunked_page_content_str:
223
+ tokenized_page = self._tokenizer(chunk_str).input_ids
224
+ for chunk in self.chunks(tokenized_page, chunk_size, overlap_size):
225
+ chunk_doc = copy.deepcopy(doc_page)
226
+ chunk_doc.page_content = self._tokenizer.decode(chunk)
227
+ chunk_doc.metadata["chunk"] = chunk_number
228
+ all_chunks.append(chunk_doc)
229
+ chunk_number += 1
230
+ return all_chunks
231
+
232
+ def _split_page_content_to_max_model_len(self, page_content_str: str) -> list:
233
+ """
234
+ When the page content is longer than the maximum model sequence length,
235
+ then the text will be split into chunks of that maximum model length.
236
+
237
+ In case when chunk of text ends on the non-whitespace character then
238
+ this chunk will be cut to the first whitespace character before
239
+ the end-position (max model length) of processed page text.
240
+
241
+
242
+ :param page_content_str: Page content as string
243
+ :return: List of chunks of page content with maximum model length
244
+ """
245
+ max_text_len = self._tokenizer.model_max_length
246
+ page_content_len = len(page_content_str)
247
+ if page_content_len <= max_text_len:
248
+ return [page_content_str]
249
+
250
+ content_chunks = []
251
+
252
+ begin_position = 0
253
+ end_position = max_text_len
254
+
255
+ # last 5% of data will be used as slicing context
256
+ slice_left_size = int(max_text_len * 0.05)
257
+
258
+ while True:
259
+ if (begin_position + slice_left_size) >= page_content_len:
260
+ break
261
+ page_chunk = page_content_str[begin_position:end_position]
262
+ content_chunks.append(page_chunk)
263
+ begin_position = end_position - slice_left_size
264
+ end_position = begin_position + max_text_len
265
+ if begin_position + max_text_len >= page_content_len:
266
+ end_position = page_content_len
267
+ return content_chunks
@@ -0,0 +1,129 @@
1
+ import re
2
+
3
+ from radlab_data.text.loaders.txt import TXTLoader
4
+
5
+
6
+ class MarkdownLoader(TXTLoader):
7
+ """
8
+ Loader for markdown (`md`) files.
9
+
10
+ Markdown file is a plain text file, so reading is fully delegated to
11
+ the TXTLoader. Additionally, each page is marked with the
12
+ `document_format` metadata and, if the `strip_markdown` option is set
13
+ to True, markdown markup (headings, emphasis, links, code fences,
14
+ lists, tables, ...) is removed from the loaded text.
15
+ """
16
+
17
+ STRIP_MARKDOWN_OPTION = "strip_markdown"
18
+ DOCUMENT_FORMAT = "markdown"
19
+
20
+ _CODE_PLACEHOLDER = "\x00markdown_code_{index}\x00"
21
+
22
+ _FRONT_MATTER_RE = re.compile(r"\A---[ \t]*\n.*?\n---[ \t]*\n", re.DOTALL)
23
+ _FENCED_CODE_RE = re.compile(r"(?:```|~~~)[^\n]*\n(.*?)\n(?:```|~~~)", re.DOTALL)
24
+ _INLINE_CODE_RE = re.compile(r"`+([^`\n]+?)`+")
25
+ _ESCAPED_CHAR_RE = re.compile(r"\\([\\`*_{}\[\]()#+\-.!>~|])")
26
+ _IMAGE_RE = re.compile(r"!\[([^\]]*)\](?:\([^)]*\)|\[[^\]]*\])")
27
+ _LINK_RE = re.compile(r"\[([^\]]*)\](?:\([^)]*\)|\[[^\]]*\])")
28
+ _AUTOLINK_RE = re.compile(r"<((?:https?|mailto):[^<>\s]+)>")
29
+ _HTML_TAG_RE = re.compile(r"</?[a-zA-Z][^>]*>")
30
+ _HEADING_RE = re.compile(r"^[ \t]{0,3}#{1,6}[ \t]+", re.MULTILINE)
31
+ _BLOCKQUOTE_RE = re.compile(r"^[ \t]{0,3}>[ \t]?", re.MULTILINE)
32
+ _TASK_LIST_MARKER_RE = re.compile(
33
+ r"^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+\[[ xX]\][ \t]+", re.MULTILINE
34
+ )
35
+ _LIST_MARKER_RE = re.compile(r"^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+", re.MULTILINE)
36
+ _EMPTY_LINE_MARKER_RE = re.compile(
37
+ r"^[ \t]{0,3}(?:=+|-{2,}|\*{3,}|_{3,})[ \t]*$", re.MULTILINE
38
+ )
39
+ _TABLE_DIVIDER_RE = re.compile(r"^[ \t]*\|[ \t:|-]*\|[ \t]*$", re.MULTILINE)
40
+ _TABLE_PIPE_RE = re.compile(r"[ \t]*\|[ \t]*")
41
+ _STRIKETHROUGH_RE = re.compile(r"~~(?P<text>[^\n]+?)~~")
42
+ _ASTERISK_EMPHASIS_RE = re.compile(
43
+ r"(?P<marker>\*{1,3})(?P<text>[^\n]*?\S)(?P=marker)"
44
+ )
45
+ _UNDERSCORE_EMPHASIS_RE = re.compile(
46
+ r"(?<![\w~_])(?P<marker>_{1,3})(?=\S)(?P<text>[^\n]*?\S)(?P=marker)(?![\w~_])"
47
+ )
48
+ _LEADING_SPACES_RE = re.compile(r"^[ \t]+", re.MULTILINE)
49
+ _TRAILING_SPACES_RE = re.compile(r"[ \t]+\n")
50
+ _EMPTY_LINES_RE = re.compile(r"\n{3,}")
51
+
52
+ def __init__(self, filepath: str, options: dict = None) -> None:
53
+ """
54
+ :param filepath: Path to markdown file
55
+ :param options: Reader options. If `strip_markdown` is set to True,
56
+ then markdown markup is removed from the loaded text.
57
+ """
58
+ TXTLoader.__init__(self, filepath=filepath, options=options)
59
+ self._strip_markdown = bool(self.options.get(self.STRIP_MARKDOWN_OPTION, False))
60
+
61
+ def load(self) -> list:
62
+ """
63
+ Load markdown file. Content is read the same way as in TXTLoader,
64
+ then (optionally) markdown markup is removed and every page is
65
+ marked with the markdown document format.
66
+ :return: List of loaded document pages
67
+ """
68
+ self.doc_pages = TXTLoader.load(self)
69
+
70
+ for doc_page in self.doc_pages:
71
+ if self._strip_markdown:
72
+ doc_page.page_content = self.strip_markdown(doc_page.page_content)
73
+ doc_page.metadata["document_format"] = self.DOCUMENT_FORMAT
74
+ return self.doc_pages
75
+
76
+ def strip_markdown(self, text: str) -> str:
77
+ """
78
+ Remove markdown markup from the given text. Text of headings, links,
79
+ list items and code is kept, only the markup itself is removed.
80
+ :param text: Text with markdown markup
81
+ :return: Text without markdown markup
82
+ """
83
+ if not text:
84
+ return text
85
+
86
+ text = self._FRONT_MATTER_RE.sub("", text)
87
+
88
+ placeholders = []
89
+
90
+ def _keep_code(match: re.Match) -> str:
91
+ placeholders.append(match.group(1))
92
+ return self._CODE_PLACEHOLDER.format(index=len(placeholders) - 1)
93
+
94
+ def _keep_escaped_char(match: re.Match) -> str:
95
+ placeholders.append(match.group(1))
96
+ return self._CODE_PLACEHOLDER.format(index=len(placeholders) - 1)
97
+
98
+ text = self._FENCED_CODE_RE.sub(_keep_code, text)
99
+ text = self._ESCAPED_CHAR_RE.sub(_keep_escaped_char, text)
100
+ text = self._INLINE_CODE_RE.sub(_keep_code, text)
101
+
102
+ text = self._IMAGE_RE.sub(lambda match: match.group(1), text)
103
+ text = self._LINK_RE.sub(lambda match: match.group(1), text)
104
+ text = self._AUTOLINK_RE.sub(lambda match: match.group(1), text)
105
+ text = self._HTML_TAG_RE.sub("", text)
106
+
107
+ text = self._HEADING_RE.sub("", text)
108
+ text = self._BLOCKQUOTE_RE.sub("", text)
109
+ text = self._TASK_LIST_MARKER_RE.sub("", text)
110
+ text = self._LIST_MARKER_RE.sub("", text)
111
+ text = self._EMPTY_LINE_MARKER_RE.sub("", text)
112
+
113
+ text = self._STRIKETHROUGH_RE.sub(lambda match: match.group("text"), text)
114
+ text = self._ASTERISK_EMPHASIS_RE.sub(lambda match: match.group("text"), text)
115
+ text = self._UNDERSCORE_EMPHASIS_RE.sub(lambda match: match.group("text"), text)
116
+
117
+ text = self._TABLE_DIVIDER_RE.sub("", text)
118
+ text = self._TABLE_PIPE_RE.sub(" ", text)
119
+
120
+ text = self._LEADING_SPACES_RE.sub("", text)
121
+ text = self._TRAILING_SPACES_RE.sub("\n", text)
122
+ text = self._EMPTY_LINES_RE.sub("\n\n", text)
123
+ text = text.strip()
124
+
125
+ for index, placeholder_value in reversed(list(enumerate(placeholders))):
126
+ text = text.replace(
127
+ self._CODE_PLACEHOLDER.format(index=index), placeholder_value
128
+ )
129
+ return text
@@ -0,0 +1,157 @@
1
+ import logging
2
+ from datetime import datetime
3
+
4
+ import pypdf
5
+
6
+ from radlab_data.text.loaders.base import DocumentPage
7
+ from radlab_data.text.loaders.loader_i import LoaderI
8
+
9
+
10
+ class PDFLoader(LoaderI):
11
+ """
12
+ Loader for PDF files based on the `pypdf` library. Each PDF page is
13
+ loaded as a single document page.
14
+ """
15
+
16
+ STANDARD_METADATA_KEYS = {
17
+ "source",
18
+ "total_pages",
19
+ "creationdate",
20
+ "creator",
21
+ "producer",
22
+ }
23
+
24
+ def __init__(
25
+ self,
26
+ filepath: str,
27
+ options: dict = None,
28
+ password: str = None,
29
+ extraction_mode: str = "plain",
30
+ extraction_kwargs: dict = None,
31
+ ) -> None:
32
+ """
33
+ :param filepath: Path to pdf file
34
+ :param options: Options shared with other loaders (see `LoaderI`)
35
+ :param password: Password for an encrypted pdf file
36
+ :param extraction_mode: `plain` or `layout` text extraction mode
37
+ :param extraction_kwargs: Additional arguments passed to
38
+ `pypdf.PageObject.extract_text`
39
+ """
40
+ LoaderI.__init__(self, filepath=filepath, options=options)
41
+ self.password = password
42
+ self.extraction_mode = extraction_mode
43
+ self.extraction_kwargs = {} if extraction_kwargs is None else extraction_kwargs
44
+
45
+ def load(self) -> list:
46
+ """
47
+ Load all pages of the pdf file.
48
+ :return: List of loaded document pages
49
+ """
50
+ self.doc_pages = []
51
+ with open(self.filepath, "rb") as pdf_file:
52
+ pdf_reader = pypdf.PdfReader(pdf_file, password=self.password)
53
+ document_metadata = self._document_metadata(pdf_reader)
54
+ page_labels = self._page_labels(pdf_reader)
55
+
56
+ for page_number, pdf_page in enumerate(pdf_reader.pages):
57
+ metadata = dict(document_metadata)
58
+ metadata["page"] = page_number
59
+ if page_labels is not None:
60
+ metadata["page_label"] = page_labels[page_number]
61
+ self.doc_pages.append(
62
+ DocumentPage(
63
+ page_content=self._extract_text(pdf_page).strip(),
64
+ metadata=self._validate_metadata(metadata),
65
+ )
66
+ )
67
+ return self.doc_pages
68
+
69
+ def _extract_text(self, pdf_page) -> str:
70
+ """
71
+ Extract text from the single pdf page
72
+ :param pdf_page: `pypdf` page object to extract text from
73
+ :return: Extracted page text
74
+ """
75
+ if self.extraction_mode == "plain" and not self.extraction_kwargs:
76
+ text = pdf_page.extract_text()
77
+ else:
78
+ text = pdf_page.extract_text(
79
+ extraction_mode=self.extraction_mode, **self.extraction_kwargs
80
+ )
81
+ return text if text is not None else ""
82
+
83
+ def _document_metadata(self, pdf_reader) -> dict:
84
+ """
85
+ Prepare metadata shared by all pages of the loaded document
86
+ :param pdf_reader: `pypdf` reader of the loaded document
87
+ :return: Document metadata
88
+ """
89
+ metadata = {"producer": "PyPDF", "creator": "PyPDF", "creationdate": ""}
90
+ metadata.update(dict(pdf_reader.metadata or {}))
91
+ metadata["source"] = str(self.filepath)
92
+ metadata["total_pages"] = len(pdf_reader.pages)
93
+ return self._purge_metadata(metadata)
94
+
95
+ @staticmethod
96
+ def _page_labels(pdf_reader):
97
+ """
98
+ Read labels of all pages. Malformed documents may not provide them.
99
+ :param pdf_reader: `pypdf` reader of the loaded document
100
+ :return: List of page labels or None when labels are not available
101
+ """
102
+ try:
103
+ return list(pdf_reader.page_labels)
104
+ except Exception as error:
105
+ logging.debug(f"Page labels are not available: {error}")
106
+ return None
107
+
108
+ @staticmethod
109
+ def _purge_metadata(metadata: dict) -> dict:
110
+ """
111
+ Clean metadata keys and values of the loaded document. Keys are
112
+ lowercased and stripped with the leading `/` (as used by pdf files),
113
+ dates are converted to the ISO format and only string and integer
114
+ values are kept.
115
+ :param metadata: Metadata read from the pdf document
116
+ :return: Cleaned metadata
117
+ """
118
+ normalized_keys = {"page_count": "total_pages", "file_path": "source"}
119
+ purged_metadata = {}
120
+ for key, value in metadata.items():
121
+ if not isinstance(value, (str, int)):
122
+ value = str(value)
123
+ if key.startswith("/"):
124
+ key = key[1:]
125
+ key = key.lower()
126
+
127
+ if key in ("creationdate", "moddate"):
128
+ try:
129
+ purged_metadata[key] = datetime.strptime(
130
+ value.replace("'", ""), "D:%Y%m%d%H%M%S%z"
131
+ ).isoformat("T")
132
+ except ValueError:
133
+ purged_metadata[key] = value
134
+ elif key in normalized_keys:
135
+ purged_metadata[normalized_keys[key]] = value
136
+ purged_metadata[key] = value
137
+ elif isinstance(value, str):
138
+ purged_metadata[key] = value.strip()
139
+ elif isinstance(value, int):
140
+ purged_metadata[key] = value
141
+ return purged_metadata
142
+
143
+ def _validate_metadata(self, metadata: dict) -> dict:
144
+ """
145
+ Check if metadata of the page contains all standard keys and
146
+ the page number is an integer
147
+ :param metadata: Metadata of the loaded page
148
+ :return: Validated metadata
149
+ """
150
+ if not self.STANDARD_METADATA_KEYS.issubset(metadata.keys()):
151
+ raise ValueError(
152
+ "Metadata of the loaded pdf document have to contain "
153
+ f"the following keys: {sorted(self.STANDARD_METADATA_KEYS)}"
154
+ )
155
+ if not isinstance(metadata.get("page", 0), int):
156
+ raise ValueError("The PDF metadata page must be an integer.")
157
+ return metadata