langchain-pymupdf-layout 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_pymupdf_layout-0.1.0/LICENSE +131 -0
- langchain_pymupdf_layout-0.1.0/PKG-INFO +219 -0
- langchain_pymupdf_layout-0.1.0/README.md +73 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout/__init__.py +12 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout/pymupdf_layout_loader.py +152 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout/pymupdf_layout_parser.py +264 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout.egg-info/PKG-INFO +219 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout.egg-info/SOURCES.txt +11 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout.egg-info/dependency_links.txt +1 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout.egg-info/requires.txt +4 -0
- langchain_pymupdf_layout-0.1.0/langchain_pymupdf_layout.egg-info/top_level.txt +1 -0
- langchain_pymupdf_layout-0.1.0/pyproject.toml +19 -0
- langchain_pymupdf_layout-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# PolyForm Noncommercial License 1.0.0
|
|
2
|
+
|
|
3
|
+
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
|
4
|
+
|
|
5
|
+
## Acceptance
|
|
6
|
+
|
|
7
|
+
In order to get any license under these terms, you must agree
|
|
8
|
+
to them as both strict obligations and conditions to all
|
|
9
|
+
your licenses.
|
|
10
|
+
|
|
11
|
+
## Copyright License
|
|
12
|
+
|
|
13
|
+
The licensor grants you a copyright license for the
|
|
14
|
+
software to do everything you might do with the software
|
|
15
|
+
that would otherwise infringe the licensor's copyright
|
|
16
|
+
in it for any permitted purpose. However, you may
|
|
17
|
+
only distribute the software according to [Distribution
|
|
18
|
+
License](#distribution-license) and make changes or new works
|
|
19
|
+
based on the software according to [Changes and New Works
|
|
20
|
+
License](#changes-and-new-works-license).
|
|
21
|
+
|
|
22
|
+
## Distribution License
|
|
23
|
+
|
|
24
|
+
The licensor grants you an additional copyright license
|
|
25
|
+
to distribute copies of the software. Your license
|
|
26
|
+
to distribute covers distributing the software with
|
|
27
|
+
changes and new works permitted by [Changes and New Works
|
|
28
|
+
License](#changes-and-new-works-license).
|
|
29
|
+
|
|
30
|
+
## Notices
|
|
31
|
+
|
|
32
|
+
You must ensure that anyone who gets a copy of any part of
|
|
33
|
+
the software from you also gets a copy of these terms or the
|
|
34
|
+
URL for them above, as well as copies of any plain-text lines
|
|
35
|
+
beginning with `Required Notice:` that the licensor provided
|
|
36
|
+
with the software. For example:
|
|
37
|
+
|
|
38
|
+
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
|
|
39
|
+
|
|
40
|
+
## Changes and New Works License
|
|
41
|
+
|
|
42
|
+
The licensor grants you an additional copyright license to
|
|
43
|
+
make changes and new works based on the software for any
|
|
44
|
+
permitted purpose.
|
|
45
|
+
|
|
46
|
+
## Patent License
|
|
47
|
+
|
|
48
|
+
The licensor grants you a patent license for the software that
|
|
49
|
+
covers patent claims the licensor can license, or becomes able
|
|
50
|
+
to license, that you would infringe by using the software.
|
|
51
|
+
|
|
52
|
+
## Noncommercial Purposes
|
|
53
|
+
|
|
54
|
+
Any noncommercial purpose is a permitted purpose.
|
|
55
|
+
|
|
56
|
+
## Personal Uses
|
|
57
|
+
|
|
58
|
+
Personal use for research, experiment, and testing for
|
|
59
|
+
the benefit of public knowledge, personal study, private
|
|
60
|
+
entertainment, hobby projects, amateur pursuits, or religious
|
|
61
|
+
observance, without any anticipated commercial application,
|
|
62
|
+
is use for a permitted purpose.
|
|
63
|
+
|
|
64
|
+
## Noncommercial Organizations
|
|
65
|
+
|
|
66
|
+
Use by any charitable organization, educational institution,
|
|
67
|
+
public research organization, public safety or health
|
|
68
|
+
organization, environmental protection organization,
|
|
69
|
+
or government institution is use for a permitted purpose
|
|
70
|
+
regardless of the source of funding or obligations resulting
|
|
71
|
+
from the funding.
|
|
72
|
+
|
|
73
|
+
## Fair Use
|
|
74
|
+
|
|
75
|
+
You may have "fair use" rights for the software under the
|
|
76
|
+
law. These terms do not limit them.
|
|
77
|
+
|
|
78
|
+
## No Other Rights
|
|
79
|
+
|
|
80
|
+
These terms do not allow you to sublicense or transfer any of
|
|
81
|
+
your licenses to anyone else, or prevent the licensor from
|
|
82
|
+
granting licenses to anyone else. These terms do not imply
|
|
83
|
+
any other licenses.
|
|
84
|
+
|
|
85
|
+
## Patent Defense
|
|
86
|
+
|
|
87
|
+
If you make any written claim that the software infringes or
|
|
88
|
+
contributes to infringement of any patent, your patent license
|
|
89
|
+
for the software granted under these terms ends immediately. If
|
|
90
|
+
your company makes such a claim, your patent license ends
|
|
91
|
+
immediately for work on behalf of your company.
|
|
92
|
+
|
|
93
|
+
## Violations
|
|
94
|
+
|
|
95
|
+
The first time you are notified in writing that you have
|
|
96
|
+
violated any of these terms, or done anything with the software
|
|
97
|
+
not covered by your licenses, your licenses can nonetheless
|
|
98
|
+
continue if you come into full compliance with these terms,
|
|
99
|
+
and take practical steps to correct past violations, within
|
|
100
|
+
32 days of receiving notice. Otherwise, all your licenses
|
|
101
|
+
end immediately.
|
|
102
|
+
|
|
103
|
+
## No Liability
|
|
104
|
+
|
|
105
|
+
***As far as the law allows, the software comes as is, without
|
|
106
|
+
any warranty or condition, and the licensor will not be liable
|
|
107
|
+
to you for any damages arising out of these terms or the use
|
|
108
|
+
or nature of the software, under any kind of legal claim.***
|
|
109
|
+
|
|
110
|
+
## Definitions
|
|
111
|
+
|
|
112
|
+
The **licensor** is the individual or entity offering these
|
|
113
|
+
terms, and the **software** is the software the licensor makes
|
|
114
|
+
available under these terms.
|
|
115
|
+
|
|
116
|
+
**You** refers to the individual or entity agreeing to these
|
|
117
|
+
terms.
|
|
118
|
+
|
|
119
|
+
**Your company** is any legal entity, sole proprietorship,
|
|
120
|
+
or other kind of organization that you work for, plus all
|
|
121
|
+
organizations that have control over, are under the control of,
|
|
122
|
+
or are under common control with that organization. **Control**
|
|
123
|
+
means ownership of substantially all the assets of an entity,
|
|
124
|
+
or the power to direct its management and policies by vote,
|
|
125
|
+
contract, or otherwise. Control can be direct or indirect.
|
|
126
|
+
|
|
127
|
+
**Your licenses** are all the licenses granted to you for the
|
|
128
|
+
software under these terms.
|
|
129
|
+
|
|
130
|
+
**Use** means anything you do with the software requiring one
|
|
131
|
+
of your licenses.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-pymupdf-layout
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis.
|
|
5
|
+
Author-email: Jamie Lemon <jamie.lemon@artifex.com>
|
|
6
|
+
License: # PolyForm Noncommercial License 1.0.0
|
|
7
|
+
|
|
8
|
+
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
|
9
|
+
|
|
10
|
+
## Acceptance
|
|
11
|
+
|
|
12
|
+
In order to get any license under these terms, you must agree
|
|
13
|
+
to them as both strict obligations and conditions to all
|
|
14
|
+
your licenses.
|
|
15
|
+
|
|
16
|
+
## Copyright License
|
|
17
|
+
|
|
18
|
+
The licensor grants you a copyright license for the
|
|
19
|
+
software to do everything you might do with the software
|
|
20
|
+
that would otherwise infringe the licensor's copyright
|
|
21
|
+
in it for any permitted purpose. However, you may
|
|
22
|
+
only distribute the software according to [Distribution
|
|
23
|
+
License](#distribution-license) and make changes or new works
|
|
24
|
+
based on the software according to [Changes and New Works
|
|
25
|
+
License](#changes-and-new-works-license).
|
|
26
|
+
|
|
27
|
+
## Distribution License
|
|
28
|
+
|
|
29
|
+
The licensor grants you an additional copyright license
|
|
30
|
+
to distribute copies of the software. Your license
|
|
31
|
+
to distribute covers distributing the software with
|
|
32
|
+
changes and new works permitted by [Changes and New Works
|
|
33
|
+
License](#changes-and-new-works-license).
|
|
34
|
+
|
|
35
|
+
## Notices
|
|
36
|
+
|
|
37
|
+
You must ensure that anyone who gets a copy of any part of
|
|
38
|
+
the software from you also gets a copy of these terms or the
|
|
39
|
+
URL for them above, as well as copies of any plain-text lines
|
|
40
|
+
beginning with `Required Notice:` that the licensor provided
|
|
41
|
+
with the software. For example:
|
|
42
|
+
|
|
43
|
+
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
|
|
44
|
+
|
|
45
|
+
## Changes and New Works License
|
|
46
|
+
|
|
47
|
+
The licensor grants you an additional copyright license to
|
|
48
|
+
make changes and new works based on the software for any
|
|
49
|
+
permitted purpose.
|
|
50
|
+
|
|
51
|
+
## Patent License
|
|
52
|
+
|
|
53
|
+
The licensor grants you a patent license for the software that
|
|
54
|
+
covers patent claims the licensor can license, or becomes able
|
|
55
|
+
to license, that you would infringe by using the software.
|
|
56
|
+
|
|
57
|
+
## Noncommercial Purposes
|
|
58
|
+
|
|
59
|
+
Any noncommercial purpose is a permitted purpose.
|
|
60
|
+
|
|
61
|
+
## Personal Uses
|
|
62
|
+
|
|
63
|
+
Personal use for research, experiment, and testing for
|
|
64
|
+
the benefit of public knowledge, personal study, private
|
|
65
|
+
entertainment, hobby projects, amateur pursuits, or religious
|
|
66
|
+
observance, without any anticipated commercial application,
|
|
67
|
+
is use for a permitted purpose.
|
|
68
|
+
|
|
69
|
+
## Noncommercial Organizations
|
|
70
|
+
|
|
71
|
+
Use by any charitable organization, educational institution,
|
|
72
|
+
public research organization, public safety or health
|
|
73
|
+
organization, environmental protection organization,
|
|
74
|
+
or government institution is use for a permitted purpose
|
|
75
|
+
regardless of the source of funding or obligations resulting
|
|
76
|
+
from the funding.
|
|
77
|
+
|
|
78
|
+
## Fair Use
|
|
79
|
+
|
|
80
|
+
You may have "fair use" rights for the software under the
|
|
81
|
+
law. These terms do not limit them.
|
|
82
|
+
|
|
83
|
+
## No Other Rights
|
|
84
|
+
|
|
85
|
+
These terms do not allow you to sublicense or transfer any of
|
|
86
|
+
your licenses to anyone else, or prevent the licensor from
|
|
87
|
+
granting licenses to anyone else. These terms do not imply
|
|
88
|
+
any other licenses.
|
|
89
|
+
|
|
90
|
+
## Patent Defense
|
|
91
|
+
|
|
92
|
+
If you make any written claim that the software infringes or
|
|
93
|
+
contributes to infringement of any patent, your patent license
|
|
94
|
+
for the software granted under these terms ends immediately. If
|
|
95
|
+
your company makes such a claim, your patent license ends
|
|
96
|
+
immediately for work on behalf of your company.
|
|
97
|
+
|
|
98
|
+
## Violations
|
|
99
|
+
|
|
100
|
+
The first time you are notified in writing that you have
|
|
101
|
+
violated any of these terms, or done anything with the software
|
|
102
|
+
not covered by your licenses, your licenses can nonetheless
|
|
103
|
+
continue if you come into full compliance with these terms,
|
|
104
|
+
and take practical steps to correct past violations, within
|
|
105
|
+
32 days of receiving notice. Otherwise, all your licenses
|
|
106
|
+
end immediately.
|
|
107
|
+
|
|
108
|
+
## No Liability
|
|
109
|
+
|
|
110
|
+
***As far as the law allows, the software comes as is, without
|
|
111
|
+
any warranty or condition, and the licensor will not be liable
|
|
112
|
+
to you for any damages arising out of these terms or the use
|
|
113
|
+
or nature of the software, under any kind of legal claim.***
|
|
114
|
+
|
|
115
|
+
## Definitions
|
|
116
|
+
|
|
117
|
+
The **licensor** is the individual or entity offering these
|
|
118
|
+
terms, and the **software** is the software the licensor makes
|
|
119
|
+
available under these terms.
|
|
120
|
+
|
|
121
|
+
**You** refers to the individual or entity agreeing to these
|
|
122
|
+
terms.
|
|
123
|
+
|
|
124
|
+
**Your company** is any legal entity, sole proprietorship,
|
|
125
|
+
or other kind of organization that you work for, plus all
|
|
126
|
+
organizations that have control over, are under the control of,
|
|
127
|
+
or are under common control with that organization. **Control**
|
|
128
|
+
means ownership of substantially all the assets of an entity,
|
|
129
|
+
or the power to direct its management and policies by vote,
|
|
130
|
+
contract, or otherwise. Control can be direct or indirect.
|
|
131
|
+
|
|
132
|
+
**Your licenses** are all the licenses granted to you for the
|
|
133
|
+
software under these terms.
|
|
134
|
+
|
|
135
|
+
**Use** means anything you do with the software requiring one
|
|
136
|
+
of your licenses.
|
|
137
|
+
|
|
138
|
+
Requires-Python: >=3.7
|
|
139
|
+
Description-Content-Type: text/markdown
|
|
140
|
+
License-File: LICENSE
|
|
141
|
+
Requires-Dist: langchain-core<2.0.0,>=1.0.0
|
|
142
|
+
Requires-Dist: pymupdf>=0.26.6
|
|
143
|
+
Requires-Dist: pymupdf-layout>=0.26.6
|
|
144
|
+
Requires-Dist: pymupdf4llm>=0.2.0
|
|
145
|
+
Dynamic: license-file
|
|
146
|
+
|
|
147
|
+
# langchain-pymupdf-layout
|
|
148
|
+
|
|
149
|
+
An integration package connecting **PyMuPDF Layout** to **LangChain**.
|
|
150
|
+
|
|
151
|
+
Load PDF content to Markdown using AI-based, CPU only, layout analysis.
|
|
152
|
+
|
|
153
|
+
[](https://github.com/langchain-ai/langchain)
|
|
154
|
+
[](https://pypi.org/project/pymupdf-layout/)
|
|
155
|
+
[](https://polyformproject.org/licenses/noncommercial/1.0.0/)
|
|
156
|
+
|
|
157
|
+
## Features
|
|
158
|
+
|
|
159
|
+
- 📚 Structured data extraction from your documents
|
|
160
|
+
- 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
|
|
161
|
+
- 🔍 Detect and isolate header and footer patterns on each page
|
|
162
|
+
|
|
163
|
+
For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
|
|
164
|
+
|
|
165
|
+
## Requirements
|
|
166
|
+
|
|
167
|
+
- Python 3.11 or higher
|
|
168
|
+
- LangChain Core v1.0.0 or higher
|
|
169
|
+
- PyMuPDF v0.26.6 or higher
|
|
170
|
+
- PyMuPDF4LLM v0.2.0 or higher
|
|
171
|
+
- PyMuPDF Layout c0.26.6 or higher
|
|
172
|
+
|
|
173
|
+
## Installation
|
|
174
|
+
|
|
175
|
+
Install the package using pip to start using the Document Loader:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
pip install -U langchain-pymupdf-layout
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## Usage
|
|
183
|
+
|
|
184
|
+
You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
|
|
185
|
+
|
|
186
|
+
Below is an example of how to set up and utilize this loader:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from langchain_pymupdf_layout import version
|
|
190
|
+
|
|
191
|
+
print(version()) # Output: version number
|
|
192
|
+
|
|
193
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
194
|
+
|
|
195
|
+
from langchain_pymupdf_layout import PyMuPDFLayoutLoader
|
|
196
|
+
|
|
197
|
+
loader = PyMuPDFLayoutLoader(
|
|
198
|
+
file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
|
|
199
|
+
show_progress=False,
|
|
200
|
+
# See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
documents = loader.load()
|
|
204
|
+
|
|
205
|
+
# Chunk
|
|
206
|
+
text_splitter = RecursiveCharacterTextSplitter(
|
|
207
|
+
chunk_size=1000,
|
|
208
|
+
chunk_overlap=200
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
chunks = text_splitter.split_documents(documents)
|
|
212
|
+
|
|
213
|
+
print(f"Loaded {len(documents)} document(s)")
|
|
214
|
+
print(f"Created {len(chunks)} chunk(s)")
|
|
215
|
+
|
|
216
|
+
content = chunks[0].page_content
|
|
217
|
+
print(f"\ncontent:\n{content}")
|
|
218
|
+
|
|
219
|
+
```
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# langchain-pymupdf-layout
|
|
2
|
+
|
|
3
|
+
An integration package connecting **PyMuPDF Layout** to **LangChain**.
|
|
4
|
+
|
|
5
|
+
Load PDF content to Markdown using AI-based, CPU only, layout analysis.
|
|
6
|
+
|
|
7
|
+
[](https://github.com/langchain-ai/langchain)
|
|
8
|
+
[](https://pypi.org/project/pymupdf-layout/)
|
|
9
|
+
[](https://polyformproject.org/licenses/noncommercial/1.0.0/)
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
- 📚 Structured data extraction from your documents
|
|
14
|
+
- 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
|
|
15
|
+
- 🔍 Detect and isolate header and footer patterns on each page
|
|
16
|
+
|
|
17
|
+
For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
|
|
18
|
+
|
|
19
|
+
## Requirements
|
|
20
|
+
|
|
21
|
+
- Python 3.11 or higher
|
|
22
|
+
- LangChain Core v1.0.0 or higher
|
|
23
|
+
- PyMuPDF v0.26.6 or higher
|
|
24
|
+
- PyMuPDF4LLM v0.2.0 or higher
|
|
25
|
+
- PyMuPDF Layout c0.26.6 or higher
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
Install the package using pip to start using the Document Loader:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install -U langchain-pymupdf-layout
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
|
|
39
|
+
|
|
40
|
+
Below is an example of how to set up and utilize this loader:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from langchain_pymupdf_layout import version
|
|
44
|
+
|
|
45
|
+
print(version()) # Output: version number
|
|
46
|
+
|
|
47
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
48
|
+
|
|
49
|
+
from langchain_pymupdf_layout import PyMuPDFLayoutLoader
|
|
50
|
+
|
|
51
|
+
loader = PyMuPDFLayoutLoader(
|
|
52
|
+
file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
|
|
53
|
+
show_progress=False,
|
|
54
|
+
# See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
documents = loader.load()
|
|
58
|
+
|
|
59
|
+
# Chunk
|
|
60
|
+
text_splitter = RecursiveCharacterTextSplitter(
|
|
61
|
+
chunk_size=1000,
|
|
62
|
+
chunk_overlap=200
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
chunks = text_splitter.split_documents(documents)
|
|
66
|
+
|
|
67
|
+
print(f"Loaded {len(documents)} document(s)")
|
|
68
|
+
print(f"Created {len(chunks)} chunk(s)")
|
|
69
|
+
|
|
70
|
+
content = chunks[0].page_content
|
|
71
|
+
print(f"\ncontent:\n{content}")
|
|
72
|
+
|
|
73
|
+
```
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Langchain PyMuPDF Layout - Load PDF content to Markdown using AI-based, CPU only, layout analysis
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
|
|
7
|
+
def version():
|
|
8
|
+
return __version__
|
|
9
|
+
|
|
10
|
+
from .pymupdf_layout_loader import PyMuPDFLayoutLoader
|
|
11
|
+
from .pymupdf_layout_parser import PyMuPDFLayoutParser
|
|
12
|
+
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import tempfile
|
|
5
|
+
from abc import ABC
|
|
6
|
+
from pathlib import PurePath
|
|
7
|
+
from typing import (
|
|
8
|
+
TYPE_CHECKING,
|
|
9
|
+
Any,
|
|
10
|
+
Iterator,
|
|
11
|
+
Optional,
|
|
12
|
+
Union,
|
|
13
|
+
)
|
|
14
|
+
from urllib.parse import urlparse
|
|
15
|
+
|
|
16
|
+
import requests
|
|
17
|
+
from langchain_core.documents import Document
|
|
18
|
+
from langchain_core.document_loaders import (
|
|
19
|
+
Blob,
|
|
20
|
+
BaseLoader,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
from langchain_pymupdf_layout.pymupdf_layout_parser import PyMuPDFLayoutParser
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__file__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Directly taken from:-
|
|
29
|
+
# github.com/langchain-ai/langchain/
|
|
30
|
+
# libs/community/langchain_community/document_loaders/pdf.py
|
|
31
|
+
class BasePDFLoader(BaseLoader, ABC):
|
|
32
|
+
"""Base Loader class for `PDF` files.
|
|
33
|
+
|
|
34
|
+
If the file is a web path, it will download it to a temporary file, use it, then
|
|
35
|
+
clean up the temporary file after completion.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self, file_path: Union[str, PurePath], *, headers: Optional[dict] = None
|
|
40
|
+
):
|
|
41
|
+
"""Initialize with a file path.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
file_path: Either a local, S3 or web path to a PDF file.
|
|
45
|
+
headers: Headers to use for GET request to download a file from a web path.
|
|
46
|
+
"""
|
|
47
|
+
self.file_path = str(file_path)
|
|
48
|
+
self.web_path = None
|
|
49
|
+
self.headers = headers
|
|
50
|
+
if "~" in self.file_path:
|
|
51
|
+
self.file_path = os.path.expanduser(self.file_path)
|
|
52
|
+
|
|
53
|
+
# If the file is a web path or S3, download it to a temporary file,
|
|
54
|
+
# and use that. It's better to use a BlobLoader.
|
|
55
|
+
if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
|
|
56
|
+
self.temp_dir = tempfile.TemporaryDirectory()
|
|
57
|
+
_, suffix = os.path.splitext(self.file_path)
|
|
58
|
+
if self._is_s3_presigned_url(self.file_path):
|
|
59
|
+
suffix = urlparse(self.file_path).path.split("/")[-1]
|
|
60
|
+
temp_pdf = os.path.join(self.temp_dir.name, f"tmp{suffix}")
|
|
61
|
+
self.web_path = self.file_path
|
|
62
|
+
if not self._is_s3_url(self.file_path):
|
|
63
|
+
r = requests.get(self.file_path, headers=self.headers)
|
|
64
|
+
if r.status_code != 200:
|
|
65
|
+
raise ValueError(
|
|
66
|
+
"Check the url of your file; returned status code %s"
|
|
67
|
+
% r.status_code
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
with open(temp_pdf, mode="wb") as f:
|
|
71
|
+
f.write(r.content)
|
|
72
|
+
self.file_path = str(temp_pdf)
|
|
73
|
+
elif not os.path.isfile(self.file_path):
|
|
74
|
+
raise ValueError("File path %s is not a valid file or url" % self.file_path)
|
|
75
|
+
|
|
76
|
+
def __del__(self) -> None:
|
|
77
|
+
if hasattr(self, "temp_dir"):
|
|
78
|
+
self.temp_dir.cleanup()
|
|
79
|
+
|
|
80
|
+
@staticmethod
|
|
81
|
+
def _is_valid_url(url: str) -> bool:
|
|
82
|
+
"""Check if the url is valid."""
|
|
83
|
+
parsed = urlparse(url)
|
|
84
|
+
return bool(parsed.netloc) and bool(parsed.scheme)
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def _is_s3_url(url: str) -> bool:
|
|
88
|
+
"""check if the url is S3"""
|
|
89
|
+
try:
|
|
90
|
+
result = urlparse(url)
|
|
91
|
+
if result.scheme == "s3" and result.netloc:
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
except ValueError:
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def _is_s3_presigned_url(url: str) -> bool:
|
|
99
|
+
"""Check if the url is a presigned S3 url."""
|
|
100
|
+
try:
|
|
101
|
+
result = urlparse(url)
|
|
102
|
+
return bool(re.search(r"\.s3\.amazonaws\.com$", result.netloc))
|
|
103
|
+
except ValueError:
|
|
104
|
+
return False
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def source(self) -> str:
|
|
108
|
+
return self.web_path if self.web_path is not None else self.file_path
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class PyMuPDFLayoutLoader(BasePDFLoader):
|
|
112
|
+
|
|
113
|
+
def __init__(
|
|
114
|
+
self,
|
|
115
|
+
file_path: Union[str, PurePath],
|
|
116
|
+
*,
|
|
117
|
+
headers: Optional[dict] = None,
|
|
118
|
+
password: Optional[str] = None,
|
|
119
|
+
**pymupdf_layout_kwargs,
|
|
120
|
+
) -> None:
|
|
121
|
+
|
|
122
|
+
super().__init__(file_path, headers=headers)
|
|
123
|
+
self.parser = PyMuPDFLayoutParser(
|
|
124
|
+
password=password,
|
|
125
|
+
**pymupdf_layout_kwargs,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _lazy_load(self, **kwargs: Any) -> Iterator[Document]:
|
|
130
|
+
"""Lazily parse a blob from a PDF document.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
blob: The blob from a PDF document to parse.
|
|
134
|
+
|
|
135
|
+
Yield:
|
|
136
|
+
An iterator over the parsed documents with PDF content.
|
|
137
|
+
"""
|
|
138
|
+
if kwargs:
|
|
139
|
+
logger.warning(
|
|
140
|
+
f"Received runtime arguments {kwargs}. Passed runtime args to `load`"
|
|
141
|
+
" are completely ignored."
|
|
142
|
+
" Please pass arguments during initialization instead."
|
|
143
|
+
)
|
|
144
|
+
if self.web_path:
|
|
145
|
+
blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
|
|
146
|
+
else:
|
|
147
|
+
blob = Blob.from_path(self.file_path)
|
|
148
|
+
yield from self.parser.lazy_parse(blob)
|
|
149
|
+
|
|
150
|
+
def load(self, **kwargs: Any) -> list[Document]:
|
|
151
|
+
return list(self._lazy_load(**kwargs))
|
|
152
|
+
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Contains PyMuPDF4LLM parser class to parse blobs from PDFs."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import pymupdf.layout
|
|
8
|
+
except ImportError:
|
|
9
|
+
raise ImportError(
|
|
10
|
+
"pymupdf.layout package not found, please install it "
|
|
11
|
+
"with `pip install pymupdf-layout`"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
|
|
16
|
+
from typing import (
|
|
17
|
+
Any,
|
|
18
|
+
Iterator,
|
|
19
|
+
Optional,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
from langchain_core.documents import Document
|
|
23
|
+
from langchain_core.document_loaders import (
|
|
24
|
+
Blob,
|
|
25
|
+
BaseBlobParser
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_STD_METADATA_KEYS = {"source", "total_pages", "creationdate", "creator", "producer"}
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
def _validate_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
|
33
|
+
"""Validate that the metadata has all the standard keys and the page is an integer.
|
|
34
|
+
|
|
35
|
+
The standard keys are:
|
|
36
|
+
- source
|
|
37
|
+
- total_page
|
|
38
|
+
- creationdate
|
|
39
|
+
- creator
|
|
40
|
+
- producer
|
|
41
|
+
|
|
42
|
+
Validate that page is an integer if it is present.
|
|
43
|
+
"""
|
|
44
|
+
if not _STD_METADATA_KEYS.issubset(metadata.keys()):
|
|
45
|
+
raise ValueError("The PDF parser must valorize the standard metadata.")
|
|
46
|
+
if not isinstance(metadata.get("page", 0), int):
|
|
47
|
+
raise ValueError("The PDF metadata page must be a integer.")
|
|
48
|
+
return metadata
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _purge_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
|
52
|
+
"""Purge metadata from unwanted keys and normalize key names.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
metadata: The original metadata dictionary.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
The cleaned and normalized the key format of metadata dictionary.
|
|
59
|
+
"""
|
|
60
|
+
new_metadata: dict[str, Any] = {}
|
|
61
|
+
map_key = {
|
|
62
|
+
"page_count": "total_pages",
|
|
63
|
+
"file_path": "source",
|
|
64
|
+
}
|
|
65
|
+
for k, v in metadata.items():
|
|
66
|
+
if type(v) not in [str, int]:
|
|
67
|
+
v = str(v)
|
|
68
|
+
if k.startswith("/"):
|
|
69
|
+
k = k[1:]
|
|
70
|
+
k = k.lower()
|
|
71
|
+
if k in ["creationdate", "moddate"]:
|
|
72
|
+
try:
|
|
73
|
+
new_metadata[k] = datetime.strptime(
|
|
74
|
+
v.replace("'", ""), "D:%Y%m%d%H%M%S%z"
|
|
75
|
+
).isoformat("T")
|
|
76
|
+
except ValueError:
|
|
77
|
+
new_metadata[k] = v
|
|
78
|
+
elif k in map_key:
|
|
79
|
+
# Normalize key with others PDF parser
|
|
80
|
+
new_metadata[map_key[k]] = v
|
|
81
|
+
new_metadata[k] = v
|
|
82
|
+
elif isinstance(v, str):
|
|
83
|
+
new_metadata[k] = v.strip()
|
|
84
|
+
elif isinstance(v, int):
|
|
85
|
+
new_metadata[k] = v
|
|
86
|
+
return new_metadata
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class PyMuPDFLayoutParser(BaseBlobParser):
|
|
90
|
+
"""Parse a blob from a PDF using `PyMuPDF4LLM` library with `PyMuPDF Layout` imported.
|
|
91
|
+
|
|
92
|
+
Examples:
|
|
93
|
+
Setup:
|
|
94
|
+
|
|
95
|
+
.. code-block:: bash
|
|
96
|
+
|
|
97
|
+
pip install -U langchain-pymupdf-layout
|
|
98
|
+
|
|
99
|
+
Load a blob from a PDF file:
|
|
100
|
+
|
|
101
|
+
.. code-block:: python
|
|
102
|
+
|
|
103
|
+
from langchain_core.documents.base import Blob
|
|
104
|
+
|
|
105
|
+
blob = Blob.from_path("./test.pdf")
|
|
106
|
+
|
|
107
|
+
Instantiate the parser:
|
|
108
|
+
|
|
109
|
+
.. code-block:: python
|
|
110
|
+
|
|
111
|
+
from langchain_pymupdf_layout import PyMuPDFLayoutParser
|
|
112
|
+
|
|
113
|
+
parser = PyMuPDFLayoutParser()
|
|
114
|
+
|
|
115
|
+
Lazily parse the blob:
|
|
116
|
+
|
|
117
|
+
.. code-block:: python
|
|
118
|
+
|
|
119
|
+
docs = []
|
|
120
|
+
docs_lazy = parser.lazy_parse(blob)
|
|
121
|
+
|
|
122
|
+
for doc in docs_lazy:
|
|
123
|
+
docs.append(doc)
|
|
124
|
+
print(docs[0].page_content[:100])
|
|
125
|
+
print(docs[0].metadata)
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
# PyMuPDF is not thread safe.
|
|
129
|
+
# See https://pymupdf.readthedocs.io/en/latest/recipes-multiprocessing.html
|
|
130
|
+
_lock = threading.Lock()
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self,
|
|
134
|
+
*,
|
|
135
|
+
password: Optional[str] = None,
|
|
136
|
+
**pymupdf_layout_kwargs,
|
|
137
|
+
) -> None:
|
|
138
|
+
"""Initialize a parser to extract PDF content in markdown using PyMuPDF4LLM.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
password: Optional password for opening encrypted PDFs.
|
|
142
|
+
**pymupdf_layout_kwargs: Additional keyword arguments to pass directly to the
|
|
143
|
+
`pymupdf4llm.to_markdown` function.
|
|
144
|
+
See: https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/api.html#to_markdown
|
|
145
|
+
And: https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
This method does not directly return data. Use the `parse` or `lazy_parse`
|
|
149
|
+
methods to retrieve parsed documents with content and metadata.
|
|
150
|
+
|
|
151
|
+
Raises:
|
|
152
|
+
ValueError: If unsupported `pymupdf_layout_kwargs` are provided (e.g.,
|
|
153
|
+
`table_strategy`, `detect_bg_color`, `ignore_images`).
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
if "table_strategy" in pymupdf_layout_kwargs:
|
|
157
|
+
raise ValueError("PyMuPDF Layout argument: table_strategy should not be set")
|
|
158
|
+
|
|
159
|
+
if "ignore_images" in pymupdf_layout_kwargs:
|
|
160
|
+
raise ValueError("PyMuPDF Layout argument: ignore_images should not be set")
|
|
161
|
+
|
|
162
|
+
if "graphics_limit" in pymupdf_layout_kwargs:
|
|
163
|
+
raise ValueError("PyMuPDF Layout argument: graphics_limit should not be set")
|
|
164
|
+
|
|
165
|
+
if "detect_bg_color" in pymupdf_layout_kwargs:
|
|
166
|
+
raise ValueError("PyMuPDF Layout argument: detect_bg_color should not be set")
|
|
167
|
+
|
|
168
|
+
super().__init__()
|
|
169
|
+
|
|
170
|
+
self.password = password
|
|
171
|
+
self.pymupdf_layout_kwargs = pymupdf_layout_kwargs
|
|
172
|
+
|
|
173
|
+
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
|
174
|
+
"""Lazily parse a blob from a PDF document.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
blob: The blob from a PDF document to parse.
|
|
178
|
+
|
|
179
|
+
Raises:
|
|
180
|
+
ImportError: If the `pymupdf4llm` package is not found.
|
|
181
|
+
|
|
182
|
+
Yield:
|
|
183
|
+
An iterator over the parsed documents with PDF content.
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
with PyMuPDFLayoutParser._lock:
|
|
187
|
+
with blob.as_bytes_io() as file_path:
|
|
188
|
+
if blob.data is None:
|
|
189
|
+
doc = pymupdf.open(file_path)
|
|
190
|
+
else:
|
|
191
|
+
doc = pymupdf.open(stream=file_path, filetype="pdf")
|
|
192
|
+
if doc.is_encrypted:
|
|
193
|
+
doc.authenticate(self.password)
|
|
194
|
+
doc_metadata = self._extract_metadata(doc, blob)
|
|
195
|
+
|
|
196
|
+
yield Document(
|
|
197
|
+
page_content=self._get_md(doc),
|
|
198
|
+
metadata=_validate_metadata(doc_metadata),
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def _get_md(
|
|
202
|
+
self,
|
|
203
|
+
doc: pymupdf.Document,
|
|
204
|
+
#pages: list | range | None = None,
|
|
205
|
+
) -> str:
|
|
206
|
+
"""Get the content of the page in markdown using PyMuPDF4LLM
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
doc: The PyMuPDF document object.
|
|
210
|
+
pages: The pages. If omitted (None) all pages are processed.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
str: The content of the page in markdown.
|
|
214
|
+
"""
|
|
215
|
+
try:
|
|
216
|
+
import pymupdf4llm
|
|
217
|
+
except ImportError:
|
|
218
|
+
raise ImportError(
|
|
219
|
+
"pymupdf4llm package not found, please install it "
|
|
220
|
+
"with `pip install pymupdf4llm`"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
pymupdf4llm_params: dict[str, Any] = {
|
|
224
|
+
**self.pymupdf_layout_kwargs,
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
md = pymupdf4llm.to_markdown(
|
|
228
|
+
doc,
|
|
229
|
+
**pymupdf4llm_params,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
return md
|
|
233
|
+
|
|
234
|
+
def _extract_metadata(self, doc: pymupdf.Document, blob: Blob) -> dict:
|
|
235
|
+
"""Extract metadata from the PDF document.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
doc: The PyMuPDF document object.
|
|
239
|
+
blob: The blob being parsed.
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
dict: The extracted metadata from the PDF.
|
|
243
|
+
"""
|
|
244
|
+
metadata = _purge_metadata(
|
|
245
|
+
{
|
|
246
|
+
**{
|
|
247
|
+
"producer": "PyMuPDF Layout",
|
|
248
|
+
"creator": "PyMuPDF Layout",
|
|
249
|
+
"creationdate": "",
|
|
250
|
+
"source": blob.source,
|
|
251
|
+
"file_path": blob.source,
|
|
252
|
+
"total_pages": len(doc),
|
|
253
|
+
},
|
|
254
|
+
**{
|
|
255
|
+
k: doc.metadata[k]
|
|
256
|
+
for k in doc.metadata
|
|
257
|
+
if isinstance(doc.metadata[k], (str, int))
|
|
258
|
+
},
|
|
259
|
+
}
|
|
260
|
+
)
|
|
261
|
+
for k in ("modDate", "creationDate"):
|
|
262
|
+
if k in doc.metadata:
|
|
263
|
+
metadata[k] = doc.metadata[k]
|
|
264
|
+
return metadata
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-pymupdf-layout
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis.
|
|
5
|
+
Author-email: Jamie Lemon <jamie.lemon@artifex.com>
|
|
6
|
+
License: # PolyForm Noncommercial License 1.0.0
|
|
7
|
+
|
|
8
|
+
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
|
9
|
+
|
|
10
|
+
## Acceptance
|
|
11
|
+
|
|
12
|
+
In order to get any license under these terms, you must agree
|
|
13
|
+
to them as both strict obligations and conditions to all
|
|
14
|
+
your licenses.
|
|
15
|
+
|
|
16
|
+
## Copyright License
|
|
17
|
+
|
|
18
|
+
The licensor grants you a copyright license for the
|
|
19
|
+
software to do everything you might do with the software
|
|
20
|
+
that would otherwise infringe the licensor's copyright
|
|
21
|
+
in it for any permitted purpose. However, you may
|
|
22
|
+
only distribute the software according to [Distribution
|
|
23
|
+
License](#distribution-license) and make changes or new works
|
|
24
|
+
based on the software according to [Changes and New Works
|
|
25
|
+
License](#changes-and-new-works-license).
|
|
26
|
+
|
|
27
|
+
## Distribution License
|
|
28
|
+
|
|
29
|
+
The licensor grants you an additional copyright license
|
|
30
|
+
to distribute copies of the software. Your license
|
|
31
|
+
to distribute covers distributing the software with
|
|
32
|
+
changes and new works permitted by [Changes and New Works
|
|
33
|
+
License](#changes-and-new-works-license).
|
|
34
|
+
|
|
35
|
+
## Notices
|
|
36
|
+
|
|
37
|
+
You must ensure that anyone who gets a copy of any part of
|
|
38
|
+
the software from you also gets a copy of these terms or the
|
|
39
|
+
URL for them above, as well as copies of any plain-text lines
|
|
40
|
+
beginning with `Required Notice:` that the licensor provided
|
|
41
|
+
with the software. For example:
|
|
42
|
+
|
|
43
|
+
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
|
|
44
|
+
|
|
45
|
+
## Changes and New Works License
|
|
46
|
+
|
|
47
|
+
The licensor grants you an additional copyright license to
|
|
48
|
+
make changes and new works based on the software for any
|
|
49
|
+
permitted purpose.
|
|
50
|
+
|
|
51
|
+
## Patent License
|
|
52
|
+
|
|
53
|
+
The licensor grants you a patent license for the software that
|
|
54
|
+
covers patent claims the licensor can license, or becomes able
|
|
55
|
+
to license, that you would infringe by using the software.
|
|
56
|
+
|
|
57
|
+
## Noncommercial Purposes
|
|
58
|
+
|
|
59
|
+
Any noncommercial purpose is a permitted purpose.
|
|
60
|
+
|
|
61
|
+
## Personal Uses
|
|
62
|
+
|
|
63
|
+
Personal use for research, experiment, and testing for
|
|
64
|
+
the benefit of public knowledge, personal study, private
|
|
65
|
+
entertainment, hobby projects, amateur pursuits, or religious
|
|
66
|
+
observance, without any anticipated commercial application,
|
|
67
|
+
is use for a permitted purpose.
|
|
68
|
+
|
|
69
|
+
## Noncommercial Organizations
|
|
70
|
+
|
|
71
|
+
Use by any charitable organization, educational institution,
|
|
72
|
+
public research organization, public safety or health
|
|
73
|
+
organization, environmental protection organization,
|
|
74
|
+
or government institution is use for a permitted purpose
|
|
75
|
+
regardless of the source of funding or obligations resulting
|
|
76
|
+
from the funding.
|
|
77
|
+
|
|
78
|
+
## Fair Use
|
|
79
|
+
|
|
80
|
+
You may have "fair use" rights for the software under the
|
|
81
|
+
law. These terms do not limit them.
|
|
82
|
+
|
|
83
|
+
## No Other Rights
|
|
84
|
+
|
|
85
|
+
These terms do not allow you to sublicense or transfer any of
|
|
86
|
+
your licenses to anyone else, or prevent the licensor from
|
|
87
|
+
granting licenses to anyone else. These terms do not imply
|
|
88
|
+
any other licenses.
|
|
89
|
+
|
|
90
|
+
## Patent Defense
|
|
91
|
+
|
|
92
|
+
If you make any written claim that the software infringes or
|
|
93
|
+
contributes to infringement of any patent, your patent license
|
|
94
|
+
for the software granted under these terms ends immediately. If
|
|
95
|
+
your company makes such a claim, your patent license ends
|
|
96
|
+
immediately for work on behalf of your company.
|
|
97
|
+
|
|
98
|
+
## Violations
|
|
99
|
+
|
|
100
|
+
The first time you are notified in writing that you have
|
|
101
|
+
violated any of these terms, or done anything with the software
|
|
102
|
+
not covered by your licenses, your licenses can nonetheless
|
|
103
|
+
continue if you come into full compliance with these terms,
|
|
104
|
+
and take practical steps to correct past violations, within
|
|
105
|
+
32 days of receiving notice. Otherwise, all your licenses
|
|
106
|
+
end immediately.
|
|
107
|
+
|
|
108
|
+
## No Liability
|
|
109
|
+
|
|
110
|
+
***As far as the law allows, the software comes as is, without
|
|
111
|
+
any warranty or condition, and the licensor will not be liable
|
|
112
|
+
to you for any damages arising out of these terms or the use
|
|
113
|
+
or nature of the software, under any kind of legal claim.***
|
|
114
|
+
|
|
115
|
+
## Definitions
|
|
116
|
+
|
|
117
|
+
The **licensor** is the individual or entity offering these
|
|
118
|
+
terms, and the **software** is the software the licensor makes
|
|
119
|
+
available under these terms.
|
|
120
|
+
|
|
121
|
+
**You** refers to the individual or entity agreeing to these
|
|
122
|
+
terms.
|
|
123
|
+
|
|
124
|
+
**Your company** is any legal entity, sole proprietorship,
|
|
125
|
+
or other kind of organization that you work for, plus all
|
|
126
|
+
organizations that have control over, are under the control of,
|
|
127
|
+
or are under common control with that organization. **Control**
|
|
128
|
+
means ownership of substantially all the assets of an entity,
|
|
129
|
+
or the power to direct its management and policies by vote,
|
|
130
|
+
contract, or otherwise. Control can be direct or indirect.
|
|
131
|
+
|
|
132
|
+
**Your licenses** are all the licenses granted to you for the
|
|
133
|
+
software under these terms.
|
|
134
|
+
|
|
135
|
+
**Use** means anything you do with the software requiring one
|
|
136
|
+
of your licenses.
|
|
137
|
+
|
|
138
|
+
Requires-Python: >=3.7
|
|
139
|
+
Description-Content-Type: text/markdown
|
|
140
|
+
License-File: LICENSE
|
|
141
|
+
Requires-Dist: langchain-core<2.0.0,>=1.0.0
|
|
142
|
+
Requires-Dist: pymupdf>=0.26.6
|
|
143
|
+
Requires-Dist: pymupdf-layout>=0.26.6
|
|
144
|
+
Requires-Dist: pymupdf4llm>=0.2.0
|
|
145
|
+
Dynamic: license-file
|
|
146
|
+
|
|
147
|
+
# langchain-pymupdf-layout
|
|
148
|
+
|
|
149
|
+
An integration package connecting **PyMuPDF Layout** to **LangChain**.
|
|
150
|
+
|
|
151
|
+
Load PDF content to Markdown using AI-based, CPU only, layout analysis.
|
|
152
|
+
|
|
153
|
+
[](https://github.com/langchain-ai/langchain)
|
|
154
|
+
[](https://pypi.org/project/pymupdf-layout/)
|
|
155
|
+
[](https://polyformproject.org/licenses/noncommercial/1.0.0/)
|
|
156
|
+
|
|
157
|
+
## Features
|
|
158
|
+
|
|
159
|
+
- 📚 Structured data extraction from your documents
|
|
160
|
+
- 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
|
|
161
|
+
- 🔍 Detect and isolate header and footer patterns on each page
|
|
162
|
+
|
|
163
|
+
For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
|
|
164
|
+
|
|
165
|
+
## Requirements
|
|
166
|
+
|
|
167
|
+
- Python 3.11 or higher
|
|
168
|
+
- LangChain Core v1.0.0 or higher
|
|
169
|
+
- PyMuPDF v0.26.6 or higher
|
|
170
|
+
- PyMuPDF4LLM v0.2.0 or higher
|
|
171
|
+
- PyMuPDF Layout c0.26.6 or higher
|
|
172
|
+
|
|
173
|
+
## Installation
|
|
174
|
+
|
|
175
|
+
Install the package using pip to start using the Document Loader:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
pip install -U langchain-pymupdf-layout
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## Usage
|
|
183
|
+
|
|
184
|
+
You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
|
|
185
|
+
|
|
186
|
+
Below is an example of how to set up and utilize this loader:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from langchain_pymupdf_layout import version
|
|
190
|
+
|
|
191
|
+
print(version()) # Output: version number
|
|
192
|
+
|
|
193
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
194
|
+
|
|
195
|
+
from langchain_pymupdf_layout import PyMuPDFLayoutLoader
|
|
196
|
+
|
|
197
|
+
loader = PyMuPDFLayoutLoader(
|
|
198
|
+
file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
|
|
199
|
+
show_progress=False,
|
|
200
|
+
# See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
documents = loader.load()
|
|
204
|
+
|
|
205
|
+
# Chunk
|
|
206
|
+
text_splitter = RecursiveCharacterTextSplitter(
|
|
207
|
+
chunk_size=1000,
|
|
208
|
+
chunk_overlap=200
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
chunks = text_splitter.split_documents(documents)
|
|
212
|
+
|
|
213
|
+
print(f"Loaded {len(documents)} document(s)")
|
|
214
|
+
print(f"Created {len(chunks)} chunk(s)")
|
|
215
|
+
|
|
216
|
+
content = chunks[0].page_content
|
|
217
|
+
print(f"\ncontent:\n{content}")
|
|
218
|
+
|
|
219
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
langchain_pymupdf_layout/__init__.py
|
|
5
|
+
langchain_pymupdf_layout/pymupdf_layout_loader.py
|
|
6
|
+
langchain_pymupdf_layout/pymupdf_layout_parser.py
|
|
7
|
+
langchain_pymupdf_layout.egg-info/PKG-INFO
|
|
8
|
+
langchain_pymupdf_layout.egg-info/SOURCES.txt
|
|
9
|
+
langchain_pymupdf_layout.egg-info/dependency_links.txt
|
|
10
|
+
langchain_pymupdf_layout.egg-info/requires.txt
|
|
11
|
+
langchain_pymupdf_layout.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
langchain_pymupdf_layout
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "langchain-pymupdf-layout"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [{name = "Jamie Lemon", email = "jamie.lemon@artifex.com"}]
|
|
11
|
+
license = {file = "LICENSE"}
|
|
12
|
+
requires-python = ">=3.7"
|
|
13
|
+
|
|
14
|
+
dependencies = [
|
|
15
|
+
"langchain-core>=1.0.0,<2.0.0",
|
|
16
|
+
"pymupdf>=0.26.6",
|
|
17
|
+
"pymupdf-layout>=0.26.6",
|
|
18
|
+
"pymupdf4llm>=0.2.0"
|
|
19
|
+
]
|