radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
|
|
5
|
+
from radlab_data.text.loaders.base import DocumentPage, read_text_file
|
|
6
|
+
from radlab_data.text.loaders.loader_i import LoaderI
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class HTMLLoader(LoaderI):
|
|
10
|
+
"""
|
|
11
|
+
Loader for HTML files. All markup which does not contribute to the
|
|
12
|
+
readable content (scripts, styles, metadata, ...) is removed and the
|
|
13
|
+
cleaned text is exposed as one or many document pages.
|
|
14
|
+
|
|
15
|
+
Recognised options:
|
|
16
|
+
* `split_by_paragraph`: when set to True, each paragraph (a fragment
|
|
17
|
+
separated by a blank line) becomes a separate document page. Without
|
|
18
|
+
this option the whole file is loaded as a single page.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
TAGS_TO_REMOVE = [
|
|
22
|
+
"script",
|
|
23
|
+
"style",
|
|
24
|
+
"noscript",
|
|
25
|
+
"svg",
|
|
26
|
+
"meta",
|
|
27
|
+
"link",
|
|
28
|
+
"head",
|
|
29
|
+
"footer",
|
|
30
|
+
"header",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
MULTIPLE_EMPTY_LINES_RE = re.compile(r"\n\s*\n+")
|
|
34
|
+
MULTIPLE_SPACES_RE = re.compile(r"[ \t]+")
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
filepath: str,
|
|
39
|
+
options: dict = None,
|
|
40
|
+
encoding: str = "utf-8",
|
|
41
|
+
autodetect_encoding: bool = True,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""
|
|
44
|
+
:param filepath: Path to html file
|
|
45
|
+
:param options: Options shared with other loaders (see `LoaderI`),
|
|
46
|
+
see the class docstring for the loader specific options
|
|
47
|
+
:param encoding: Encoding used to read the file
|
|
48
|
+
:param autodetect_encoding: If the file cannot be decoded with
|
|
49
|
+
`encoding`, encodings from `FALLBACK_ENCODINGS` are tried
|
|
50
|
+
"""
|
|
51
|
+
LoaderI.__init__(self, filepath=filepath, options=options)
|
|
52
|
+
self.encoding = encoding
|
|
53
|
+
self.autodetect_encoding = autodetect_encoding
|
|
54
|
+
|
|
55
|
+
def load(self) -> list:
|
|
56
|
+
"""
|
|
57
|
+
Load the html file and remove all its markup.
|
|
58
|
+
:return: List of loaded document pages
|
|
59
|
+
"""
|
|
60
|
+
if self.doc_pages:
|
|
61
|
+
return self.doc_pages
|
|
62
|
+
|
|
63
|
+
clean_text = self._clean_html(
|
|
64
|
+
read_text_file(
|
|
65
|
+
self.filepath,
|
|
66
|
+
encoding=self.encoding,
|
|
67
|
+
autodetect_encoding=self.autodetect_encoding,
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if not self.options.get("split_by_paragraph", False):
|
|
72
|
+
self.doc_pages = [
|
|
73
|
+
DocumentPage(
|
|
74
|
+
page_content=clean_text,
|
|
75
|
+
metadata={"source": str(self.filepath)},
|
|
76
|
+
)
|
|
77
|
+
]
|
|
78
|
+
return self.doc_pages
|
|
79
|
+
|
|
80
|
+
paragraph_number = 0
|
|
81
|
+
for paragraph in clean_text.split("\n\n"):
|
|
82
|
+
if not paragraph.strip():
|
|
83
|
+
continue
|
|
84
|
+
paragraph_number += 1
|
|
85
|
+
self.doc_pages.append(
|
|
86
|
+
DocumentPage(
|
|
87
|
+
page_content=paragraph.strip(),
|
|
88
|
+
metadata={
|
|
89
|
+
"source": str(self.filepath),
|
|
90
|
+
"paragraph": paragraph_number,
|
|
91
|
+
},
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
return self.doc_pages
|
|
95
|
+
|
|
96
|
+
def _clean_html(self, raw_html: str) -> str:
|
|
97
|
+
"""
|
|
98
|
+
Remove tags which are not a content of the document and normalize
|
|
99
|
+
whitespace. Paragraph breaks are preserved, multiple spaces and
|
|
100
|
+
tabs are reduced to a single space.
|
|
101
|
+
:param raw_html: Raw html content
|
|
102
|
+
:return: Cleaned, readable text
|
|
103
|
+
"""
|
|
104
|
+
soup = BeautifulSoup(raw_html, "html.parser")
|
|
105
|
+
for tag in soup(self.TAGS_TO_REMOVE):
|
|
106
|
+
tag.decompose()
|
|
107
|
+
|
|
108
|
+
text = soup.get_text(separator="\n")
|
|
109
|
+
text = self.MULTIPLE_EMPTY_LINES_RE.sub("\n\n", text)
|
|
110
|
+
text = self.MULTIPLE_SPACES_RE.sub(" ", text)
|
|
111
|
+
return text.strip()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from radlab_data.text.loaders.base import DocumentPage
|
|
2
|
+
|
|
3
|
+
from radlab_data.text.loaders.loader_i import LoaderI
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class InputTextLoader(LoaderI):
|
|
7
|
+
"""
|
|
8
|
+
Loads given text string as Document
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
DOCUMENT_CONTENT_VARIABLE = "document_content"
|
|
12
|
+
|
|
13
|
+
def __init__(self, filepath: str, options: dict = None) -> None:
|
|
14
|
+
"""
|
|
15
|
+
:param filepath: Filename extension have to be: `input_text`
|
|
16
|
+
"""
|
|
17
|
+
LoaderI.__init__(self, filepath=filepath, options=options)
|
|
18
|
+
|
|
19
|
+
if self.DOCUMENT_CONTENT_VARIABLE not in options:
|
|
20
|
+
raise Exception(
|
|
21
|
+
f"No {self.DOCUMENT_CONTENT_VARIABLE} in "
|
|
22
|
+
f"options while loading input_text!"
|
|
23
|
+
)
|
|
24
|
+
self._content = options[self.DOCUMENT_CONTENT_VARIABLE]
|
|
25
|
+
|
|
26
|
+
def load(self):
|
|
27
|
+
self.doc_pages = [
|
|
28
|
+
DocumentPage(page_content=self._content, metadata={"page": 1})
|
|
29
|
+
]
|
|
30
|
+
return self.doc_pages
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import abc
|
|
3
|
+
import copy
|
|
4
|
+
from functools import lru_cache
|
|
5
|
+
|
|
6
|
+
from radlab_data.text.utils import TextUtils
|
|
7
|
+
from radlab_data.text.loaders.config import TOKENIZER_PATH
|
|
8
|
+
from radlab_data.text.processors.cleaner import FullChainProcessor
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@lru_cache(maxsize=4)
|
|
12
|
+
def get_tokenizer(tokenizer_path: str = None):
|
|
13
|
+
"""
|
|
14
|
+
Return the tokenizer for the given path. Tokenizers are cached, because
|
|
15
|
+
loading the same tokenizer over and over again (once per loaded document)
|
|
16
|
+
is a significant and completely avoidable overhead.
|
|
17
|
+
|
|
18
|
+
The ``transformers`` import happens here, not at module import time, so the
|
|
19
|
+
core document-loading parts of ``radlab_data`` can be imported on hosts that
|
|
20
|
+
do not carry the model stack.
|
|
21
|
+
:param tokenizer_path: Path or name of the tokenizer, defaults to
|
|
22
|
+
`config.TOKENIZER_PATH`
|
|
23
|
+
:return: Loaded tokenizer with the model max length limitation removed
|
|
24
|
+
"""
|
|
25
|
+
if tokenizer_path is None:
|
|
26
|
+
tokenizer_path = TOKENIZER_PATH
|
|
27
|
+
from transformers import AutoTokenizer
|
|
28
|
+
|
|
29
|
+
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
|
|
30
|
+
tokenizer.model_max_length = sys.maxsize
|
|
31
|
+
return tokenizer
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LoaderI(abc.ABC):
|
|
35
|
+
def __init__(self, filepath: str, options: dict = None) -> None:
|
|
36
|
+
"""
|
|
37
|
+
:param filepath: Path to the loaded file
|
|
38
|
+
:param options: Options shared with the concrete document loader
|
|
39
|
+
"""
|
|
40
|
+
self.filepath = filepath
|
|
41
|
+
self.options = {} if not options else options
|
|
42
|
+
self.doc_pages = []
|
|
43
|
+
self.text_chunk_type = None
|
|
44
|
+
self._text_cleaner = FullChainProcessor(phrases_to_remove=[])
|
|
45
|
+
self._tokenizer_instance = None
|
|
46
|
+
|
|
47
|
+
self._check_language = self.options.get("check_text_language", False)
|
|
48
|
+
if self._check_language is None:
|
|
49
|
+
self._check_language = False
|
|
50
|
+
|
|
51
|
+
self._resolve_text_chunk_type()
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def _tokenizer(self):
|
|
55
|
+
"""
|
|
56
|
+
Lazily load the tokenizer on first access. Loaders only need it when
|
|
57
|
+
chunking pages by tokens, so merely constructing (or importing) a
|
|
58
|
+
loader no longer requires ``transformers`` to be installed.
|
|
59
|
+
:return: Tokenizer used for token-level chunking
|
|
60
|
+
"""
|
|
61
|
+
if self._tokenizer_instance is None:
|
|
62
|
+
self._tokenizer_instance = get_tokenizer()
|
|
63
|
+
return self._tokenizer_instance
|
|
64
|
+
|
|
65
|
+
def _resolve_text_chunk_type(self):
|
|
66
|
+
sep = ""
|
|
67
|
+
self.text_chunk_type = "raw_text"
|
|
68
|
+
if "clear_texts" in self.options:
|
|
69
|
+
clr = self.options.get("clear_texts", False)
|
|
70
|
+
if clr is not None and clr:
|
|
71
|
+
self.text_chunk_type = "clear_texts"
|
|
72
|
+
if "use_text_denoiser" in self.options:
|
|
73
|
+
utd = self.options.get("use_text_denoiser", False)
|
|
74
|
+
if utd is not None and utd:
|
|
75
|
+
sep = "_"
|
|
76
|
+
self.text_chunk_type += f"{sep}denoised"
|
|
77
|
+
if "merge_to_proper_pages" in self.options:
|
|
78
|
+
mp = self.options.get("merge_to_proper_pages", False)
|
|
79
|
+
if mp is not None and mp:
|
|
80
|
+
sep = "_"
|
|
81
|
+
self.text_chunk_type += f"{sep}proper_page"
|
|
82
|
+
if "merge_document_pages" in self.options:
|
|
83
|
+
md = self.options.get("merge_document_pages", False)
|
|
84
|
+
if md is not None and md:
|
|
85
|
+
sep = "_"
|
|
86
|
+
self.text_chunk_type += f"{sep}merged_document"
|
|
87
|
+
if "max_tokens_in_chunk" in self.options:
|
|
88
|
+
mt = self.options.get("max_tokens_in_chunk", 0)
|
|
89
|
+
if mt is not None and mt > 1:
|
|
90
|
+
sep = "_"
|
|
91
|
+
self.text_chunk_type += f"{sep}chunk_max_tokens_{mt}"
|
|
92
|
+
|
|
93
|
+
def as_dict(self) -> dict:
|
|
94
|
+
"""
|
|
95
|
+
Return document as dictionary (json)
|
|
96
|
+
:return: Document converted to dictionary
|
|
97
|
+
"""
|
|
98
|
+
relative_file_path = self.options.get("relative_file_path", self.filepath)
|
|
99
|
+
category = self.options.get("document_category", "")
|
|
100
|
+
|
|
101
|
+
doc_dict = {
|
|
102
|
+
"filepath": self.filepath,
|
|
103
|
+
"relative_filepath": relative_file_path,
|
|
104
|
+
"category": category,
|
|
105
|
+
"options": self.options,
|
|
106
|
+
"pages": [],
|
|
107
|
+
}
|
|
108
|
+
for doc_page in self.doc_pages:
|
|
109
|
+
if "text_chunk_type" not in doc_page.metadata:
|
|
110
|
+
doc_page.metadata["text_chunk_type"] = self.text_chunk_type
|
|
111
|
+
doc_dict["pages"].append(
|
|
112
|
+
{
|
|
113
|
+
"page_number": doc_page.metadata.get("page", 0),
|
|
114
|
+
"table_number": doc_page.metadata.get("table", 0),
|
|
115
|
+
"row_number": doc_page.metadata.get("row", 0),
|
|
116
|
+
"column_number": doc_page.metadata.get("column", 0),
|
|
117
|
+
"page_content": doc_page.page_content,
|
|
118
|
+
"metadata": doc_page.metadata,
|
|
119
|
+
}
|
|
120
|
+
)
|
|
121
|
+
return doc_dict
|
|
122
|
+
|
|
123
|
+
def merge_all_pages(self) -> None:
|
|
124
|
+
"""
|
|
125
|
+
Of option to merging is set as true, all pages are
|
|
126
|
+
concatenated as single. Actual self.doc_pages is replaced
|
|
127
|
+
with list with the single page (whole concatenated document)
|
|
128
|
+
:return: None
|
|
129
|
+
"""
|
|
130
|
+
if not self.options.get("merge_document_pages", False) or not len(
|
|
131
|
+
self.doc_pages
|
|
132
|
+
):
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
merged_pages = ""
|
|
136
|
+
for doc_page in self.doc_pages:
|
|
137
|
+
merged_pages += doc_page.page_content + "\n"
|
|
138
|
+
document_page = self.doc_pages[0]
|
|
139
|
+
document_page.metadata["page"] = 1
|
|
140
|
+
document_page.page_content = merged_pages
|
|
141
|
+
self.doc_pages = [document_page]
|
|
142
|
+
|
|
143
|
+
def clear_doc_pages(self) -> None:
|
|
144
|
+
"""
|
|
145
|
+
If clear_texts options is given, then all pages will be cleared
|
|
146
|
+
using FullChainProcess cleaner. Cleaning is done in place.
|
|
147
|
+
:return: None
|
|
148
|
+
"""
|
|
149
|
+
if not self.options.get("clear_texts", False):
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
for doc_page in self.doc_pages:
|
|
153
|
+
doc_page.page_content = self._text_cleaner.apply(doc_page.page_content)
|
|
154
|
+
|
|
155
|
+
def split_to_chunks(self) -> None:
|
|
156
|
+
"""
|
|
157
|
+
Each page will be split into chunks of given size. Additionally,
|
|
158
|
+
generated chunks will be overlapped if option is set.
|
|
159
|
+
:return: None
|
|
160
|
+
"""
|
|
161
|
+
chunk_size = self.options.get("max_tokens_in_chunk", 0)
|
|
162
|
+
overlap_size = self.options.get("number_of_overlap_tokens", 0)
|
|
163
|
+
if overlap_size is not None and overlap_size > 0:
|
|
164
|
+
self.text_chunk_type += f"_overlap_{overlap_size}"
|
|
165
|
+
|
|
166
|
+
if chunk_size is None or chunk_size < 2:
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
new_pages = []
|
|
170
|
+
for doc_page in self.doc_pages:
|
|
171
|
+
new_pages.extend(
|
|
172
|
+
self._split_page_to_chunks_max_tokens(
|
|
173
|
+
doc_page=doc_page,
|
|
174
|
+
chunk_size=chunk_size,
|
|
175
|
+
overlap_size=overlap_size,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
self.doc_pages = new_pages
|
|
179
|
+
|
|
180
|
+
def check_language(self) -> None:
|
|
181
|
+
if not self._check_language:
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
for doc_page in self.doc_pages:
|
|
185
|
+
doc_page.metadata["language"] = TextUtils.text_language(
|
|
186
|
+
text_str=doc_page.page_content, probs=False
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def chunks(tokenized_page, chunk_size, overlap_size):
|
|
191
|
+
step_size = chunk_size
|
|
192
|
+
if overlap_size is not None and overlap_size > 1:
|
|
193
|
+
step_size = chunk_size - overlap_size
|
|
194
|
+
for index in range(0, len(tokenized_page), step_size):
|
|
195
|
+
yield tokenized_page[index : index + chunk_size]
|
|
196
|
+
|
|
197
|
+
def _split_page_to_chunks_max_tokens(
|
|
198
|
+
self, doc_page, chunk_size, overlap_size
|
|
199
|
+
) -> list:
|
|
200
|
+
"""
|
|
201
|
+
Split page content to chunks of given max_tokens_in_chunk size
|
|
202
|
+
:param doc_page: Document to split
|
|
203
|
+
:param chunk_size: Chunk size (max number of tokens)
|
|
204
|
+
:param overlap_size: Number of tokens overlapping adjacent chunks
|
|
205
|
+
:return: List of objects of Document (deepcopy of doc_page
|
|
206
|
+
with tokenized and chunked document page content)
|
|
207
|
+
"""
|
|
208
|
+
if chunk_size < 1:
|
|
209
|
+
return [doc_page]
|
|
210
|
+
|
|
211
|
+
# NOTE:
|
|
212
|
+
# This is not needed because tokenizer max model len is set as sys.maxsize
|
|
213
|
+
# If this option will not be available, teb uncomment:
|
|
214
|
+
# chunked_page_content_str = self._split_page_content_to_max_model_len(
|
|
215
|
+
# doc_page.page_content
|
|
216
|
+
# )
|
|
217
|
+
# ...and comment the line below
|
|
218
|
+
chunked_page_content_str = [doc_page.page_content]
|
|
219
|
+
|
|
220
|
+
all_chunks = []
|
|
221
|
+
chunk_number = 1
|
|
222
|
+
for chunk_str in chunked_page_content_str:
|
|
223
|
+
tokenized_page = self._tokenizer(chunk_str).input_ids
|
|
224
|
+
for chunk in self.chunks(tokenized_page, chunk_size, overlap_size):
|
|
225
|
+
chunk_doc = copy.deepcopy(doc_page)
|
|
226
|
+
chunk_doc.page_content = self._tokenizer.decode(chunk)
|
|
227
|
+
chunk_doc.metadata["chunk"] = chunk_number
|
|
228
|
+
all_chunks.append(chunk_doc)
|
|
229
|
+
chunk_number += 1
|
|
230
|
+
return all_chunks
|
|
231
|
+
|
|
232
|
+
def _split_page_content_to_max_model_len(self, page_content_str: str) -> list:
|
|
233
|
+
"""
|
|
234
|
+
When the page content is longer than the maximum model sequence length,
|
|
235
|
+
then the text will be split into chunks of that maximum model length.
|
|
236
|
+
|
|
237
|
+
In case when chunk of text ends on the non-whitespace character then
|
|
238
|
+
this chunk will be cut to the first whitespace character before
|
|
239
|
+
the end-position (max model length) of processed page text.
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
:param page_content_str: Page content as string
|
|
243
|
+
:return: List of chunks of page content with maximum model length
|
|
244
|
+
"""
|
|
245
|
+
max_text_len = self._tokenizer.model_max_length
|
|
246
|
+
page_content_len = len(page_content_str)
|
|
247
|
+
if page_content_len <= max_text_len:
|
|
248
|
+
return [page_content_str]
|
|
249
|
+
|
|
250
|
+
content_chunks = []
|
|
251
|
+
|
|
252
|
+
begin_position = 0
|
|
253
|
+
end_position = max_text_len
|
|
254
|
+
|
|
255
|
+
# last 5% of data will be used as slicing context
|
|
256
|
+
slice_left_size = int(max_text_len * 0.05)
|
|
257
|
+
|
|
258
|
+
while True:
|
|
259
|
+
if (begin_position + slice_left_size) >= page_content_len:
|
|
260
|
+
break
|
|
261
|
+
page_chunk = page_content_str[begin_position:end_position]
|
|
262
|
+
content_chunks.append(page_chunk)
|
|
263
|
+
begin_position = end_position - slice_left_size
|
|
264
|
+
end_position = begin_position + max_text_len
|
|
265
|
+
if begin_position + max_text_len >= page_content_len:
|
|
266
|
+
end_position = page_content_len
|
|
267
|
+
return content_chunks
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from radlab_data.text.loaders.txt import TXTLoader
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MarkdownLoader(TXTLoader):
|
|
7
|
+
"""
|
|
8
|
+
Loader for markdown (`md`) files.
|
|
9
|
+
|
|
10
|
+
Markdown file is a plain text file, so reading is fully delegated to
|
|
11
|
+
the TXTLoader. Additionally, each page is marked with the
|
|
12
|
+
`document_format` metadata and, if the `strip_markdown` option is set
|
|
13
|
+
to True, markdown markup (headings, emphasis, links, code fences,
|
|
14
|
+
lists, tables, ...) is removed from the loaded text.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
STRIP_MARKDOWN_OPTION = "strip_markdown"
|
|
18
|
+
DOCUMENT_FORMAT = "markdown"
|
|
19
|
+
|
|
20
|
+
_CODE_PLACEHOLDER = "\x00markdown_code_{index}\x00"
|
|
21
|
+
|
|
22
|
+
_FRONT_MATTER_RE = re.compile(r"\A---[ \t]*\n.*?\n---[ \t]*\n", re.DOTALL)
|
|
23
|
+
_FENCED_CODE_RE = re.compile(r"(?:```|~~~)[^\n]*\n(.*?)\n(?:```|~~~)", re.DOTALL)
|
|
24
|
+
_INLINE_CODE_RE = re.compile(r"`+([^`\n]+?)`+")
|
|
25
|
+
_ESCAPED_CHAR_RE = re.compile(r"\\([\\`*_{}\[\]()#+\-.!>~|])")
|
|
26
|
+
_IMAGE_RE = re.compile(r"!\[([^\]]*)\](?:\([^)]*\)|\[[^\]]*\])")
|
|
27
|
+
_LINK_RE = re.compile(r"\[([^\]]*)\](?:\([^)]*\)|\[[^\]]*\])")
|
|
28
|
+
_AUTOLINK_RE = re.compile(r"<((?:https?|mailto):[^<>\s]+)>")
|
|
29
|
+
_HTML_TAG_RE = re.compile(r"</?[a-zA-Z][^>]*>")
|
|
30
|
+
_HEADING_RE = re.compile(r"^[ \t]{0,3}#{1,6}[ \t]+", re.MULTILINE)
|
|
31
|
+
_BLOCKQUOTE_RE = re.compile(r"^[ \t]{0,3}>[ \t]?", re.MULTILINE)
|
|
32
|
+
_TASK_LIST_MARKER_RE = re.compile(
|
|
33
|
+
r"^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+\[[ xX]\][ \t]+", re.MULTILINE
|
|
34
|
+
)
|
|
35
|
+
_LIST_MARKER_RE = re.compile(r"^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+", re.MULTILINE)
|
|
36
|
+
_EMPTY_LINE_MARKER_RE = re.compile(
|
|
37
|
+
r"^[ \t]{0,3}(?:=+|-{2,}|\*{3,}|_{3,})[ \t]*$", re.MULTILINE
|
|
38
|
+
)
|
|
39
|
+
_TABLE_DIVIDER_RE = re.compile(r"^[ \t]*\|[ \t:|-]*\|[ \t]*$", re.MULTILINE)
|
|
40
|
+
_TABLE_PIPE_RE = re.compile(r"[ \t]*\|[ \t]*")
|
|
41
|
+
_STRIKETHROUGH_RE = re.compile(r"~~(?P<text>[^\n]+?)~~")
|
|
42
|
+
_ASTERISK_EMPHASIS_RE = re.compile(
|
|
43
|
+
r"(?P<marker>\*{1,3})(?P<text>[^\n]*?\S)(?P=marker)"
|
|
44
|
+
)
|
|
45
|
+
_UNDERSCORE_EMPHASIS_RE = re.compile(
|
|
46
|
+
r"(?<![\w~_])(?P<marker>_{1,3})(?=\S)(?P<text>[^\n]*?\S)(?P=marker)(?![\w~_])"
|
|
47
|
+
)
|
|
48
|
+
_LEADING_SPACES_RE = re.compile(r"^[ \t]+", re.MULTILINE)
|
|
49
|
+
_TRAILING_SPACES_RE = re.compile(r"[ \t]+\n")
|
|
50
|
+
_EMPTY_LINES_RE = re.compile(r"\n{3,}")
|
|
51
|
+
|
|
52
|
+
def __init__(self, filepath: str, options: dict = None) -> None:
|
|
53
|
+
"""
|
|
54
|
+
:param filepath: Path to markdown file
|
|
55
|
+
:param options: Reader options. If `strip_markdown` is set to True,
|
|
56
|
+
then markdown markup is removed from the loaded text.
|
|
57
|
+
"""
|
|
58
|
+
TXTLoader.__init__(self, filepath=filepath, options=options)
|
|
59
|
+
self._strip_markdown = bool(self.options.get(self.STRIP_MARKDOWN_OPTION, False))
|
|
60
|
+
|
|
61
|
+
def load(self) -> list:
|
|
62
|
+
"""
|
|
63
|
+
Load markdown file. Content is read the same way as in TXTLoader,
|
|
64
|
+
then (optionally) markdown markup is removed and every page is
|
|
65
|
+
marked with the markdown document format.
|
|
66
|
+
:return: List of loaded document pages
|
|
67
|
+
"""
|
|
68
|
+
self.doc_pages = TXTLoader.load(self)
|
|
69
|
+
|
|
70
|
+
for doc_page in self.doc_pages:
|
|
71
|
+
if self._strip_markdown:
|
|
72
|
+
doc_page.page_content = self.strip_markdown(doc_page.page_content)
|
|
73
|
+
doc_page.metadata["document_format"] = self.DOCUMENT_FORMAT
|
|
74
|
+
return self.doc_pages
|
|
75
|
+
|
|
76
|
+
def strip_markdown(self, text: str) -> str:
|
|
77
|
+
"""
|
|
78
|
+
Remove markdown markup from the given text. Text of headings, links,
|
|
79
|
+
list items and code is kept, only the markup itself is removed.
|
|
80
|
+
:param text: Text with markdown markup
|
|
81
|
+
:return: Text without markdown markup
|
|
82
|
+
"""
|
|
83
|
+
if not text:
|
|
84
|
+
return text
|
|
85
|
+
|
|
86
|
+
text = self._FRONT_MATTER_RE.sub("", text)
|
|
87
|
+
|
|
88
|
+
placeholders = []
|
|
89
|
+
|
|
90
|
+
def _keep_code(match: re.Match) -> str:
|
|
91
|
+
placeholders.append(match.group(1))
|
|
92
|
+
return self._CODE_PLACEHOLDER.format(index=len(placeholders) - 1)
|
|
93
|
+
|
|
94
|
+
def _keep_escaped_char(match: re.Match) -> str:
|
|
95
|
+
placeholders.append(match.group(1))
|
|
96
|
+
return self._CODE_PLACEHOLDER.format(index=len(placeholders) - 1)
|
|
97
|
+
|
|
98
|
+
text = self._FENCED_CODE_RE.sub(_keep_code, text)
|
|
99
|
+
text = self._ESCAPED_CHAR_RE.sub(_keep_escaped_char, text)
|
|
100
|
+
text = self._INLINE_CODE_RE.sub(_keep_code, text)
|
|
101
|
+
|
|
102
|
+
text = self._IMAGE_RE.sub(lambda match: match.group(1), text)
|
|
103
|
+
text = self._LINK_RE.sub(lambda match: match.group(1), text)
|
|
104
|
+
text = self._AUTOLINK_RE.sub(lambda match: match.group(1), text)
|
|
105
|
+
text = self._HTML_TAG_RE.sub("", text)
|
|
106
|
+
|
|
107
|
+
text = self._HEADING_RE.sub("", text)
|
|
108
|
+
text = self._BLOCKQUOTE_RE.sub("", text)
|
|
109
|
+
text = self._TASK_LIST_MARKER_RE.sub("", text)
|
|
110
|
+
text = self._LIST_MARKER_RE.sub("", text)
|
|
111
|
+
text = self._EMPTY_LINE_MARKER_RE.sub("", text)
|
|
112
|
+
|
|
113
|
+
text = self._STRIKETHROUGH_RE.sub(lambda match: match.group("text"), text)
|
|
114
|
+
text = self._ASTERISK_EMPHASIS_RE.sub(lambda match: match.group("text"), text)
|
|
115
|
+
text = self._UNDERSCORE_EMPHASIS_RE.sub(lambda match: match.group("text"), text)
|
|
116
|
+
|
|
117
|
+
text = self._TABLE_DIVIDER_RE.sub("", text)
|
|
118
|
+
text = self._TABLE_PIPE_RE.sub(" ", text)
|
|
119
|
+
|
|
120
|
+
text = self._LEADING_SPACES_RE.sub("", text)
|
|
121
|
+
text = self._TRAILING_SPACES_RE.sub("\n", text)
|
|
122
|
+
text = self._EMPTY_LINES_RE.sub("\n\n", text)
|
|
123
|
+
text = text.strip()
|
|
124
|
+
|
|
125
|
+
for index, placeholder_value in reversed(list(enumerate(placeholders))):
|
|
126
|
+
text = text.replace(
|
|
127
|
+
self._CODE_PLACEHOLDER.format(index=index), placeholder_value
|
|
128
|
+
)
|
|
129
|
+
return text
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
|
|
4
|
+
import pypdf
|
|
5
|
+
|
|
6
|
+
from radlab_data.text.loaders.base import DocumentPage
|
|
7
|
+
from radlab_data.text.loaders.loader_i import LoaderI
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PDFLoader(LoaderI):
|
|
11
|
+
"""
|
|
12
|
+
Loader for PDF files based on the `pypdf` library. Each PDF page is
|
|
13
|
+
loaded as a single document page.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
STANDARD_METADATA_KEYS = {
|
|
17
|
+
"source",
|
|
18
|
+
"total_pages",
|
|
19
|
+
"creationdate",
|
|
20
|
+
"creator",
|
|
21
|
+
"producer",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
filepath: str,
|
|
27
|
+
options: dict = None,
|
|
28
|
+
password: str = None,
|
|
29
|
+
extraction_mode: str = "plain",
|
|
30
|
+
extraction_kwargs: dict = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
"""
|
|
33
|
+
:param filepath: Path to pdf file
|
|
34
|
+
:param options: Options shared with other loaders (see `LoaderI`)
|
|
35
|
+
:param password: Password for an encrypted pdf file
|
|
36
|
+
:param extraction_mode: `plain` or `layout` text extraction mode
|
|
37
|
+
:param extraction_kwargs: Additional arguments passed to
|
|
38
|
+
`pypdf.PageObject.extract_text`
|
|
39
|
+
"""
|
|
40
|
+
LoaderI.__init__(self, filepath=filepath, options=options)
|
|
41
|
+
self.password = password
|
|
42
|
+
self.extraction_mode = extraction_mode
|
|
43
|
+
self.extraction_kwargs = {} if extraction_kwargs is None else extraction_kwargs
|
|
44
|
+
|
|
45
|
+
def load(self) -> list:
|
|
46
|
+
"""
|
|
47
|
+
Load all pages of the pdf file.
|
|
48
|
+
:return: List of loaded document pages
|
|
49
|
+
"""
|
|
50
|
+
self.doc_pages = []
|
|
51
|
+
with open(self.filepath, "rb") as pdf_file:
|
|
52
|
+
pdf_reader = pypdf.PdfReader(pdf_file, password=self.password)
|
|
53
|
+
document_metadata = self._document_metadata(pdf_reader)
|
|
54
|
+
page_labels = self._page_labels(pdf_reader)
|
|
55
|
+
|
|
56
|
+
for page_number, pdf_page in enumerate(pdf_reader.pages):
|
|
57
|
+
metadata = dict(document_metadata)
|
|
58
|
+
metadata["page"] = page_number
|
|
59
|
+
if page_labels is not None:
|
|
60
|
+
metadata["page_label"] = page_labels[page_number]
|
|
61
|
+
self.doc_pages.append(
|
|
62
|
+
DocumentPage(
|
|
63
|
+
page_content=self._extract_text(pdf_page).strip(),
|
|
64
|
+
metadata=self._validate_metadata(metadata),
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
return self.doc_pages
|
|
68
|
+
|
|
69
|
+
def _extract_text(self, pdf_page) -> str:
|
|
70
|
+
"""
|
|
71
|
+
Extract text from the single pdf page
|
|
72
|
+
:param pdf_page: `pypdf` page object to extract text from
|
|
73
|
+
:return: Extracted page text
|
|
74
|
+
"""
|
|
75
|
+
if self.extraction_mode == "plain" and not self.extraction_kwargs:
|
|
76
|
+
text = pdf_page.extract_text()
|
|
77
|
+
else:
|
|
78
|
+
text = pdf_page.extract_text(
|
|
79
|
+
extraction_mode=self.extraction_mode, **self.extraction_kwargs
|
|
80
|
+
)
|
|
81
|
+
return text if text is not None else ""
|
|
82
|
+
|
|
83
|
+
def _document_metadata(self, pdf_reader) -> dict:
|
|
84
|
+
"""
|
|
85
|
+
Prepare metadata shared by all pages of the loaded document
|
|
86
|
+
:param pdf_reader: `pypdf` reader of the loaded document
|
|
87
|
+
:return: Document metadata
|
|
88
|
+
"""
|
|
89
|
+
metadata = {"producer": "PyPDF", "creator": "PyPDF", "creationdate": ""}
|
|
90
|
+
metadata.update(dict(pdf_reader.metadata or {}))
|
|
91
|
+
metadata["source"] = str(self.filepath)
|
|
92
|
+
metadata["total_pages"] = len(pdf_reader.pages)
|
|
93
|
+
return self._purge_metadata(metadata)
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _page_labels(pdf_reader):
|
|
97
|
+
"""
|
|
98
|
+
Read labels of all pages. Malformed documents may not provide them.
|
|
99
|
+
:param pdf_reader: `pypdf` reader of the loaded document
|
|
100
|
+
:return: List of page labels or None when labels are not available
|
|
101
|
+
"""
|
|
102
|
+
try:
|
|
103
|
+
return list(pdf_reader.page_labels)
|
|
104
|
+
except Exception as error:
|
|
105
|
+
logging.debug(f"Page labels are not available: {error}")
|
|
106
|
+
return None
|
|
107
|
+
|
|
108
|
+
@staticmethod
|
|
109
|
+
def _purge_metadata(metadata: dict) -> dict:
|
|
110
|
+
"""
|
|
111
|
+
Clean metadata keys and values of the loaded document. Keys are
|
|
112
|
+
lowercased and stripped with the leading `/` (as used by pdf files),
|
|
113
|
+
dates are converted to the ISO format and only string and integer
|
|
114
|
+
values are kept.
|
|
115
|
+
:param metadata: Metadata read from the pdf document
|
|
116
|
+
:return: Cleaned metadata
|
|
117
|
+
"""
|
|
118
|
+
normalized_keys = {"page_count": "total_pages", "file_path": "source"}
|
|
119
|
+
purged_metadata = {}
|
|
120
|
+
for key, value in metadata.items():
|
|
121
|
+
if not isinstance(value, (str, int)):
|
|
122
|
+
value = str(value)
|
|
123
|
+
if key.startswith("/"):
|
|
124
|
+
key = key[1:]
|
|
125
|
+
key = key.lower()
|
|
126
|
+
|
|
127
|
+
if key in ("creationdate", "moddate"):
|
|
128
|
+
try:
|
|
129
|
+
purged_metadata[key] = datetime.strptime(
|
|
130
|
+
value.replace("'", ""), "D:%Y%m%d%H%M%S%z"
|
|
131
|
+
).isoformat("T")
|
|
132
|
+
except ValueError:
|
|
133
|
+
purged_metadata[key] = value
|
|
134
|
+
elif key in normalized_keys:
|
|
135
|
+
purged_metadata[normalized_keys[key]] = value
|
|
136
|
+
purged_metadata[key] = value
|
|
137
|
+
elif isinstance(value, str):
|
|
138
|
+
purged_metadata[key] = value.strip()
|
|
139
|
+
elif isinstance(value, int):
|
|
140
|
+
purged_metadata[key] = value
|
|
141
|
+
return purged_metadata
|
|
142
|
+
|
|
143
|
+
def _validate_metadata(self, metadata: dict) -> dict:
|
|
144
|
+
"""
|
|
145
|
+
Check if metadata of the page contains all standard keys and
|
|
146
|
+
the page number is an integer
|
|
147
|
+
:param metadata: Metadata of the loaded page
|
|
148
|
+
:return: Validated metadata
|
|
149
|
+
"""
|
|
150
|
+
if not self.STANDARD_METADATA_KEYS.issubset(metadata.keys()):
|
|
151
|
+
raise ValueError(
|
|
152
|
+
"Metadata of the loaded pdf document have to contain "
|
|
153
|
+
f"the following keys: {sorted(self.STANDARD_METADATA_KEYS)}"
|
|
154
|
+
)
|
|
155
|
+
if not isinstance(metadata.get("page", 0), int):
|
|
156
|
+
raise ValueError("The PDF metadata page must be an integer.")
|
|
157
|
+
return metadata
|