langchain-pymupdf-layout 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,131 @@
1
+ # PolyForm Noncommercial License 1.0.0
2
+
3
+ <https://polyformproject.org/licenses/noncommercial/1.0.0>
4
+
5
+ ## Acceptance
6
+
7
+ In order to get any license under these terms, you must agree
8
+ to them as both strict obligations and conditions to all
9
+ your licenses.
10
+
11
+ ## Copyright License
12
+
13
+ The licensor grants you a copyright license for the
14
+ software to do everything you might do with the software
15
+ that would otherwise infringe the licensor's copyright
16
+ in it for any permitted purpose. However, you may
17
+ only distribute the software according to [Distribution
18
+ License](#distribution-license) and make changes or new works
19
+ based on the software according to [Changes and New Works
20
+ License](#changes-and-new-works-license).
21
+
22
+ ## Distribution License
23
+
24
+ The licensor grants you an additional copyright license
25
+ to distribute copies of the software. Your license
26
+ to distribute covers distributing the software with
27
+ changes and new works permitted by [Changes and New Works
28
+ License](#changes-and-new-works-license).
29
+
30
+ ## Notices
31
+
32
+ You must ensure that anyone who gets a copy of any part of
33
+ the software from you also gets a copy of these terms or the
34
+ URL for them above, as well as copies of any plain-text lines
35
+ beginning with `Required Notice:` that the licensor provided
36
+ with the software. For example:
37
+
38
+ > Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
39
+
40
+ ## Changes and New Works License
41
+
42
+ The licensor grants you an additional copyright license to
43
+ make changes and new works based on the software for any
44
+ permitted purpose.
45
+
46
+ ## Patent License
47
+
48
+ The licensor grants you a patent license for the software that
49
+ covers patent claims the licensor can license, or becomes able
50
+ to license, that you would infringe by using the software.
51
+
52
+ ## Noncommercial Purposes
53
+
54
+ Any noncommercial purpose is a permitted purpose.
55
+
56
+ ## Personal Uses
57
+
58
+ Personal use for research, experiment, and testing for
59
+ the benefit of public knowledge, personal study, private
60
+ entertainment, hobby projects, amateur pursuits, or religious
61
+ observance, without any anticipated commercial application,
62
+ is use for a permitted purpose.
63
+
64
+ ## Noncommercial Organizations
65
+
66
+ Use by any charitable organization, educational institution,
67
+ public research organization, public safety or health
68
+ organization, environmental protection organization,
69
+ or government institution is use for a permitted purpose
70
+ regardless of the source of funding or obligations resulting
71
+ from the funding.
72
+
73
+ ## Fair Use
74
+
75
+ You may have "fair use" rights for the software under the
76
+ law. These terms do not limit them.
77
+
78
+ ## No Other Rights
79
+
80
+ These terms do not allow you to sublicense or transfer any of
81
+ your licenses to anyone else, or prevent the licensor from
82
+ granting licenses to anyone else. These terms do not imply
83
+ any other licenses.
84
+
85
+ ## Patent Defense
86
+
87
+ If you make any written claim that the software infringes or
88
+ contributes to infringement of any patent, your patent license
89
+ for the software granted under these terms ends immediately. If
90
+ your company makes such a claim, your patent license ends
91
+ immediately for work on behalf of your company.
92
+
93
+ ## Violations
94
+
95
+ The first time you are notified in writing that you have
96
+ violated any of these terms, or done anything with the software
97
+ not covered by your licenses, your licenses can nonetheless
98
+ continue if you come into full compliance with these terms,
99
+ and take practical steps to correct past violations, within
100
+ 32 days of receiving notice. Otherwise, all your licenses
101
+ end immediately.
102
+
103
+ ## No Liability
104
+
105
+ ***As far as the law allows, the software comes as is, without
106
+ any warranty or condition, and the licensor will not be liable
107
+ to you for any damages arising out of these terms or the use
108
+ or nature of the software, under any kind of legal claim.***
109
+
110
+ ## Definitions
111
+
112
+ The **licensor** is the individual or entity offering these
113
+ terms, and the **software** is the software the licensor makes
114
+ available under these terms.
115
+
116
+ **You** refers to the individual or entity agreeing to these
117
+ terms.
118
+
119
+ **Your company** is any legal entity, sole proprietorship,
120
+ or other kind of organization that you work for, plus all
121
+ organizations that have control over, are under the control of,
122
+ or are under common control with that organization. **Control**
123
+ means ownership of substantially all the assets of an entity,
124
+ or the power to direct its management and policies by vote,
125
+ contract, or otherwise. Control can be direct or indirect.
126
+
127
+ **Your licenses** are all the licenses granted to you for the
128
+ software under these terms.
129
+
130
+ **Use** means anything you do with the software requiring one
131
+ of your licenses.
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: langchain-pymupdf-layout
3
+ Version: 0.1.0
4
+ Summary: An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis.
5
+ Author-email: Jamie Lemon <jamie.lemon@artifex.com>
6
+ License: # PolyForm Noncommercial License 1.0.0
7
+
8
+ <https://polyformproject.org/licenses/noncommercial/1.0.0>
9
+
10
+ ## Acceptance
11
+
12
+ In order to get any license under these terms, you must agree
13
+ to them as both strict obligations and conditions to all
14
+ your licenses.
15
+
16
+ ## Copyright License
17
+
18
+ The licensor grants you a copyright license for the
19
+ software to do everything you might do with the software
20
+ that would otherwise infringe the licensor's copyright
21
+ in it for any permitted purpose. However, you may
22
+ only distribute the software according to [Distribution
23
+ License](#distribution-license) and make changes or new works
24
+ based on the software according to [Changes and New Works
25
+ License](#changes-and-new-works-license).
26
+
27
+ ## Distribution License
28
+
29
+ The licensor grants you an additional copyright license
30
+ to distribute copies of the software. Your license
31
+ to distribute covers distributing the software with
32
+ changes and new works permitted by [Changes and New Works
33
+ License](#changes-and-new-works-license).
34
+
35
+ ## Notices
36
+
37
+ You must ensure that anyone who gets a copy of any part of
38
+ the software from you also gets a copy of these terms or the
39
+ URL for them above, as well as copies of any plain-text lines
40
+ beginning with `Required Notice:` that the licensor provided
41
+ with the software. For example:
42
+
43
+ > Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
44
+
45
+ ## Changes and New Works License
46
+
47
+ The licensor grants you an additional copyright license to
48
+ make changes and new works based on the software for any
49
+ permitted purpose.
50
+
51
+ ## Patent License
52
+
53
+ The licensor grants you a patent license for the software that
54
+ covers patent claims the licensor can license, or becomes able
55
+ to license, that you would infringe by using the software.
56
+
57
+ ## Noncommercial Purposes
58
+
59
+ Any noncommercial purpose is a permitted purpose.
60
+
61
+ ## Personal Uses
62
+
63
+ Personal use for research, experiment, and testing for
64
+ the benefit of public knowledge, personal study, private
65
+ entertainment, hobby projects, amateur pursuits, or religious
66
+ observance, without any anticipated commercial application,
67
+ is use for a permitted purpose.
68
+
69
+ ## Noncommercial Organizations
70
+
71
+ Use by any charitable organization, educational institution,
72
+ public research organization, public safety or health
73
+ organization, environmental protection organization,
74
+ or government institution is use for a permitted purpose
75
+ regardless of the source of funding or obligations resulting
76
+ from the funding.
77
+
78
+ ## Fair Use
79
+
80
+ You may have "fair use" rights for the software under the
81
+ law. These terms do not limit them.
82
+
83
+ ## No Other Rights
84
+
85
+ These terms do not allow you to sublicense or transfer any of
86
+ your licenses to anyone else, or prevent the licensor from
87
+ granting licenses to anyone else. These terms do not imply
88
+ any other licenses.
89
+
90
+ ## Patent Defense
91
+
92
+ If you make any written claim that the software infringes or
93
+ contributes to infringement of any patent, your patent license
94
+ for the software granted under these terms ends immediately. If
95
+ your company makes such a claim, your patent license ends
96
+ immediately for work on behalf of your company.
97
+
98
+ ## Violations
99
+
100
+ The first time you are notified in writing that you have
101
+ violated any of these terms, or done anything with the software
102
+ not covered by your licenses, your licenses can nonetheless
103
+ continue if you come into full compliance with these terms,
104
+ and take practical steps to correct past violations, within
105
+ 32 days of receiving notice. Otherwise, all your licenses
106
+ end immediately.
107
+
108
+ ## No Liability
109
+
110
+ ***As far as the law allows, the software comes as is, without
111
+ any warranty or condition, and the licensor will not be liable
112
+ to you for any damages arising out of these terms or the use
113
+ or nature of the software, under any kind of legal claim.***
114
+
115
+ ## Definitions
116
+
117
+ The **licensor** is the individual or entity offering these
118
+ terms, and the **software** is the software the licensor makes
119
+ available under these terms.
120
+
121
+ **You** refers to the individual or entity agreeing to these
122
+ terms.
123
+
124
+ **Your company** is any legal entity, sole proprietorship,
125
+ or other kind of organization that you work for, plus all
126
+ organizations that have control over, are under the control of,
127
+ or are under common control with that organization. **Control**
128
+ means ownership of substantially all the assets of an entity,
129
+ or the power to direct its management and policies by vote,
130
+ contract, or otherwise. Control can be direct or indirect.
131
+
132
+ **Your licenses** are all the licenses granted to you for the
133
+ software under these terms.
134
+
135
+ **Use** means anything you do with the software requiring one
136
+ of your licenses.
137
+
138
+ Requires-Python: >=3.7
139
+ Description-Content-Type: text/markdown
140
+ License-File: LICENSE
141
+ Requires-Dist: langchain-core<2.0.0,>=1.0.0
142
+ Requires-Dist: pymupdf>=0.26.6
143
+ Requires-Dist: pymupdf-layout>=0.26.6
144
+ Requires-Dist: pymupdf4llm>=0.2.0
145
+ Dynamic: license-file
146
+
147
+ # langchain-pymupdf-layout
148
+
149
+ An integration package connecting **PyMuPDF Layout** to **LangChain**.
150
+
151
+ Load PDF content to Markdown using AI-based, CPU only, layout analysis.
152
+
153
+ [![LangChain](https://img.shields.io/badge/LangChain-v1.0+-blue)](https://github.com/langchain-ai/langchain)
154
+ [![Python version](https://img.shields.io/badge/python-3.11+-blue)](https://pypi.org/project/pymupdf-layout/)
155
+ [![License PolyForm Noncommercial](https://img.shields.io/badge/license-Polyform_Noncommercial-purple)](https://polyformproject.org/licenses/noncommercial/1.0.0/)
156
+
157
+ ## Features
158
+
159
+ - 📚 Structured data extraction from your documents
160
+ - 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
161
+ - 🔍 Detect and isolate header and footer patterns on each page
162
+
163
+ For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
164
+
165
+ ## Requirements
166
+
167
+ - Python 3.11 or higher
168
+ - LangChain Core v1.0.0 or higher
169
+ - PyMuPDF v0.26.6 or higher
170
+ - PyMuPDF4LLM v0.2.0 or higher
171
+ - PyMuPDF Layout c0.26.6 or higher
172
+
173
+ ## Installation
174
+
175
+ Install the package using pip to start using the Document Loader:
176
+
177
+ ```bash
178
+ pip install -U langchain-pymupdf-layout
179
+
180
+ ```
181
+
182
+ ## Usage
183
+
184
+ You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
185
+
186
+ Below is an example of how to set up and utilize this loader:
187
+
188
+ ```python
189
+ from langchain_pymupdf_layout import version
190
+
191
+ print(version()) # Output: version number
192
+
193
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
194
+
195
+ from langchain_pymupdf_layout import PyMuPDFLayoutLoader
196
+
197
+ loader = PyMuPDFLayoutLoader(
198
+ file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
199
+ show_progress=False,
200
+ # See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
201
+ )
202
+
203
+ documents = loader.load()
204
+
205
+ # Chunk
206
+ text_splitter = RecursiveCharacterTextSplitter(
207
+ chunk_size=1000,
208
+ chunk_overlap=200
209
+ )
210
+
211
+ chunks = text_splitter.split_documents(documents)
212
+
213
+ print(f"Loaded {len(documents)} document(s)")
214
+ print(f"Created {len(chunks)} chunk(s)")
215
+
216
+ content = chunks[0].page_content
217
+ print(f"\ncontent:\n{content}")
218
+
219
+ ```
@@ -0,0 +1,73 @@
1
+ # langchain-pymupdf-layout
2
+
3
+ An integration package connecting **PyMuPDF Layout** to **LangChain**.
4
+
5
+ Load PDF content to Markdown using AI-based, CPU only, layout analysis.
6
+
7
+ [![LangChain](https://img.shields.io/badge/LangChain-v1.0+-blue)](https://github.com/langchain-ai/langchain)
8
+ [![Python version](https://img.shields.io/badge/python-3.11+-blue)](https://pypi.org/project/pymupdf-layout/)
9
+ [![License PolyForm Noncommercial](https://img.shields.io/badge/license-Polyform_Noncommercial-purple)](https://polyformproject.org/licenses/noncommercial/1.0.0/)
10
+
11
+ ## Features
12
+
13
+ - 📚 Structured data extraction from your documents
14
+ - 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
15
+ - 🔍 Detect and isolate header and footer patterns on each page
16
+
17
+ For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
18
+
19
+ ## Requirements
20
+
21
+ - Python 3.11 or higher
22
+ - LangChain Core v1.0.0 or higher
23
+ - PyMuPDF v0.26.6 or higher
24
+ - PyMuPDF4LLM v0.2.0 or higher
25
+ - PyMuPDF Layout c0.26.6 or higher
26
+
27
+ ## Installation
28
+
29
+ Install the package using pip to start using the Document Loader:
30
+
31
+ ```bash
32
+ pip install -U langchain-pymupdf-layout
33
+
34
+ ```
35
+
36
+ ## Usage
37
+
38
+ You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
39
+
40
+ Below is an example of how to set up and utilize this loader:
41
+
42
+ ```python
43
+ from langchain_pymupdf_layout import version
44
+
45
+ print(version()) # Output: version number
46
+
47
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
48
+
49
+ from langchain_pymupdf_layout import PyMuPDFLayoutLoader
50
+
51
+ loader = PyMuPDFLayoutLoader(
52
+ file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
53
+ show_progress=False,
54
+ # See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
55
+ )
56
+
57
+ documents = loader.load()
58
+
59
+ # Chunk
60
+ text_splitter = RecursiveCharacterTextSplitter(
61
+ chunk_size=1000,
62
+ chunk_overlap=200
63
+ )
64
+
65
+ chunks = text_splitter.split_documents(documents)
66
+
67
+ print(f"Loaded {len(documents)} document(s)")
68
+ print(f"Created {len(chunks)} chunk(s)")
69
+
70
+ content = chunks[0].page_content
71
+ print(f"\ncontent:\n{content}")
72
+
73
+ ```
@@ -0,0 +1,12 @@
1
+ """
2
+ Langchain PyMuPDF Layout - Load PDF content to Markdown using AI-based, CPU only, layout analysis
3
+ """
4
+
5
+ __version__ = "0.1.0"
6
+
7
+ def version():
8
+ return __version__
9
+
10
+ from .pymupdf_layout_loader import PyMuPDFLayoutLoader
11
+ from .pymupdf_layout_parser import PyMuPDFLayoutParser
12
+
@@ -0,0 +1,152 @@
1
+ import logging
2
+ import os
3
+ import re
4
+ import tempfile
5
+ from abc import ABC
6
+ from pathlib import PurePath
7
+ from typing import (
8
+ TYPE_CHECKING,
9
+ Any,
10
+ Iterator,
11
+ Optional,
12
+ Union,
13
+ )
14
+ from urllib.parse import urlparse
15
+
16
+ import requests
17
+ from langchain_core.documents import Document
18
+ from langchain_core.document_loaders import (
19
+ Blob,
20
+ BaseLoader,
21
+ )
22
+
23
+ from langchain_pymupdf_layout.pymupdf_layout_parser import PyMuPDFLayoutParser
24
+
25
+ logger = logging.getLogger(__file__)
26
+
27
+
28
+ # Directly taken from:-
29
+ # github.com/langchain-ai/langchain/
30
+ # libs/community/langchain_community/document_loaders/pdf.py
31
+ class BasePDFLoader(BaseLoader, ABC):
32
+ """Base Loader class for `PDF` files.
33
+
34
+ If the file is a web path, it will download it to a temporary file, use it, then
35
+ clean up the temporary file after completion.
36
+ """
37
+
38
+ def __init__(
39
+ self, file_path: Union[str, PurePath], *, headers: Optional[dict] = None
40
+ ):
41
+ """Initialize with a file path.
42
+
43
+ Args:
44
+ file_path: Either a local, S3 or web path to a PDF file.
45
+ headers: Headers to use for GET request to download a file from a web path.
46
+ """
47
+ self.file_path = str(file_path)
48
+ self.web_path = None
49
+ self.headers = headers
50
+ if "~" in self.file_path:
51
+ self.file_path = os.path.expanduser(self.file_path)
52
+
53
+ # If the file is a web path or S3, download it to a temporary file,
54
+ # and use that. It's better to use a BlobLoader.
55
+ if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
56
+ self.temp_dir = tempfile.TemporaryDirectory()
57
+ _, suffix = os.path.splitext(self.file_path)
58
+ if self._is_s3_presigned_url(self.file_path):
59
+ suffix = urlparse(self.file_path).path.split("/")[-1]
60
+ temp_pdf = os.path.join(self.temp_dir.name, f"tmp{suffix}")
61
+ self.web_path = self.file_path
62
+ if not self._is_s3_url(self.file_path):
63
+ r = requests.get(self.file_path, headers=self.headers)
64
+ if r.status_code != 200:
65
+ raise ValueError(
66
+ "Check the url of your file; returned status code %s"
67
+ % r.status_code
68
+ )
69
+
70
+ with open(temp_pdf, mode="wb") as f:
71
+ f.write(r.content)
72
+ self.file_path = str(temp_pdf)
73
+ elif not os.path.isfile(self.file_path):
74
+ raise ValueError("File path %s is not a valid file or url" % self.file_path)
75
+
76
+ def __del__(self) -> None:
77
+ if hasattr(self, "temp_dir"):
78
+ self.temp_dir.cleanup()
79
+
80
+ @staticmethod
81
+ def _is_valid_url(url: str) -> bool:
82
+ """Check if the url is valid."""
83
+ parsed = urlparse(url)
84
+ return bool(parsed.netloc) and bool(parsed.scheme)
85
+
86
+ @staticmethod
87
+ def _is_s3_url(url: str) -> bool:
88
+ """check if the url is S3"""
89
+ try:
90
+ result = urlparse(url)
91
+ if result.scheme == "s3" and result.netloc:
92
+ return True
93
+ return False
94
+ except ValueError:
95
+ return False
96
+
97
+ @staticmethod
98
+ def _is_s3_presigned_url(url: str) -> bool:
99
+ """Check if the url is a presigned S3 url."""
100
+ try:
101
+ result = urlparse(url)
102
+ return bool(re.search(r"\.s3\.amazonaws\.com$", result.netloc))
103
+ except ValueError:
104
+ return False
105
+
106
+ @property
107
+ def source(self) -> str:
108
+ return self.web_path if self.web_path is not None else self.file_path
109
+
110
+
111
+ class PyMuPDFLayoutLoader(BasePDFLoader):
112
+
113
+ def __init__(
114
+ self,
115
+ file_path: Union[str, PurePath],
116
+ *,
117
+ headers: Optional[dict] = None,
118
+ password: Optional[str] = None,
119
+ **pymupdf_layout_kwargs,
120
+ ) -> None:
121
+
122
+ super().__init__(file_path, headers=headers)
123
+ self.parser = PyMuPDFLayoutParser(
124
+ password=password,
125
+ **pymupdf_layout_kwargs,
126
+ )
127
+
128
+
129
+ def _lazy_load(self, **kwargs: Any) -> Iterator[Document]:
130
+ """Lazily parse a blob from a PDF document.
131
+
132
+ Args:
133
+ blob: The blob from a PDF document to parse.
134
+
135
+ Yield:
136
+ An iterator over the parsed documents with PDF content.
137
+ """
138
+ if kwargs:
139
+ logger.warning(
140
+ f"Received runtime arguments {kwargs}. Passed runtime args to `load`"
141
+ " are completely ignored."
142
+ " Please pass arguments during initialization instead."
143
+ )
144
+ if self.web_path:
145
+ blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
146
+ else:
147
+ blob = Blob.from_path(self.file_path)
148
+ yield from self.parser.lazy_parse(blob)
149
+
150
+ def load(self, **kwargs: Any) -> list[Document]:
151
+ return list(self._lazy_load(**kwargs))
152
+
@@ -0,0 +1,264 @@
1
+ """Contains PyMuPDF4LLM parser class to parse blobs from PDFs."""
2
+
3
+ import logging
4
+ import threading
5
+
6
+ try:
7
+ import pymupdf.layout
8
+ except ImportError:
9
+ raise ImportError(
10
+ "pymupdf.layout package not found, please install it "
11
+ "with `pip install pymupdf-layout`"
12
+ )
13
+
14
+ from datetime import datetime
15
+
16
+ from typing import (
17
+ Any,
18
+ Iterator,
19
+ Optional,
20
+ )
21
+
22
+ from langchain_core.documents import Document
23
+ from langchain_core.document_loaders import (
24
+ Blob,
25
+ BaseBlobParser
26
+ )
27
+
28
+ _STD_METADATA_KEYS = {"source", "total_pages", "creationdate", "creator", "producer"}
29
+
30
+ logger = logging.getLogger(__name__)
31
+
32
+ def _validate_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
33
+ """Validate that the metadata has all the standard keys and the page is an integer.
34
+
35
+ The standard keys are:
36
+ - source
37
+ - total_page
38
+ - creationdate
39
+ - creator
40
+ - producer
41
+
42
+ Validate that page is an integer if it is present.
43
+ """
44
+ if not _STD_METADATA_KEYS.issubset(metadata.keys()):
45
+ raise ValueError("The PDF parser must valorize the standard metadata.")
46
+ if not isinstance(metadata.get("page", 0), int):
47
+ raise ValueError("The PDF metadata page must be a integer.")
48
+ return metadata
49
+
50
+
51
+ def _purge_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
52
+ """Purge metadata from unwanted keys and normalize key names.
53
+
54
+ Args:
55
+ metadata: The original metadata dictionary.
56
+
57
+ Returns:
58
+ The cleaned and normalized the key format of metadata dictionary.
59
+ """
60
+ new_metadata: dict[str, Any] = {}
61
+ map_key = {
62
+ "page_count": "total_pages",
63
+ "file_path": "source",
64
+ }
65
+ for k, v in metadata.items():
66
+ if type(v) not in [str, int]:
67
+ v = str(v)
68
+ if k.startswith("/"):
69
+ k = k[1:]
70
+ k = k.lower()
71
+ if k in ["creationdate", "moddate"]:
72
+ try:
73
+ new_metadata[k] = datetime.strptime(
74
+ v.replace("'", ""), "D:%Y%m%d%H%M%S%z"
75
+ ).isoformat("T")
76
+ except ValueError:
77
+ new_metadata[k] = v
78
+ elif k in map_key:
79
+ # Normalize key with others PDF parser
80
+ new_metadata[map_key[k]] = v
81
+ new_metadata[k] = v
82
+ elif isinstance(v, str):
83
+ new_metadata[k] = v.strip()
84
+ elif isinstance(v, int):
85
+ new_metadata[k] = v
86
+ return new_metadata
87
+
88
+
89
+ class PyMuPDFLayoutParser(BaseBlobParser):
90
+ """Parse a blob from a PDF using `PyMuPDF4LLM` library with `PyMuPDF Layout` imported.
91
+
92
+ Examples:
93
+ Setup:
94
+
95
+ .. code-block:: bash
96
+
97
+ pip install -U langchain-pymupdf-layout
98
+
99
+ Load a blob from a PDF file:
100
+
101
+ .. code-block:: python
102
+
103
+ from langchain_core.documents.base import Blob
104
+
105
+ blob = Blob.from_path("./test.pdf")
106
+
107
+ Instantiate the parser:
108
+
109
+ .. code-block:: python
110
+
111
+ from langchain_pymupdf_layout import PyMuPDFLayoutParser
112
+
113
+ parser = PyMuPDFLayoutParser()
114
+
115
+ Lazily parse the blob:
116
+
117
+ .. code-block:: python
118
+
119
+ docs = []
120
+ docs_lazy = parser.lazy_parse(blob)
121
+
122
+ for doc in docs_lazy:
123
+ docs.append(doc)
124
+ print(docs[0].page_content[:100])
125
+ print(docs[0].metadata)
126
+ """
127
+
128
+ # PyMuPDF is not thread safe.
129
+ # See https://pymupdf.readthedocs.io/en/latest/recipes-multiprocessing.html
130
+ _lock = threading.Lock()
131
+
132
+ def __init__(
133
+ self,
134
+ *,
135
+ password: Optional[str] = None,
136
+ **pymupdf_layout_kwargs,
137
+ ) -> None:
138
+ """Initialize a parser to extract PDF content in markdown using PyMuPDF4LLM.
139
+
140
+ Args:
141
+ password: Optional password for opening encrypted PDFs.
142
+ **pymupdf_layout_kwargs: Additional keyword arguments to pass directly to the
143
+ `pymupdf4llm.to_markdown` function.
144
+ See: https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/api.html#to_markdown
145
+ And: https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
146
+
147
+ Returns:
148
+ This method does not directly return data. Use the `parse` or `lazy_parse`
149
+ methods to retrieve parsed documents with content and metadata.
150
+
151
+ Raises:
152
+ ValueError: If unsupported `pymupdf_layout_kwargs` are provided (e.g.,
153
+ `table_strategy`, `detect_bg_color`, `ignore_images`).
154
+ """
155
+
156
+ if "table_strategy" in pymupdf_layout_kwargs:
157
+ raise ValueError("PyMuPDF Layout argument: table_strategy should not be set")
158
+
159
+ if "ignore_images" in pymupdf_layout_kwargs:
160
+ raise ValueError("PyMuPDF Layout argument: ignore_images should not be set")
161
+
162
+ if "graphics_limit" in pymupdf_layout_kwargs:
163
+ raise ValueError("PyMuPDF Layout argument: graphics_limit should not be set")
164
+
165
+ if "detect_bg_color" in pymupdf_layout_kwargs:
166
+ raise ValueError("PyMuPDF Layout argument: detect_bg_color should not be set")
167
+
168
+ super().__init__()
169
+
170
+ self.password = password
171
+ self.pymupdf_layout_kwargs = pymupdf_layout_kwargs
172
+
173
+ def lazy_parse(self, blob: Blob) -> Iterator[Document]:
174
+ """Lazily parse a blob from a PDF document.
175
+
176
+ Args:
177
+ blob: The blob from a PDF document to parse.
178
+
179
+ Raises:
180
+ ImportError: If the `pymupdf4llm` package is not found.
181
+
182
+ Yield:
183
+ An iterator over the parsed documents with PDF content.
184
+ """
185
+
186
+ with PyMuPDFLayoutParser._lock:
187
+ with blob.as_bytes_io() as file_path:
188
+ if blob.data is None:
189
+ doc = pymupdf.open(file_path)
190
+ else:
191
+ doc = pymupdf.open(stream=file_path, filetype="pdf")
192
+ if doc.is_encrypted:
193
+ doc.authenticate(self.password)
194
+ doc_metadata = self._extract_metadata(doc, blob)
195
+
196
+ yield Document(
197
+ page_content=self._get_md(doc),
198
+ metadata=_validate_metadata(doc_metadata),
199
+ )
200
+
201
+ def _get_md(
202
+ self,
203
+ doc: pymupdf.Document,
204
+ #pages: list | range | None = None,
205
+ ) -> str:
206
+ """Get the content of the page in markdown using PyMuPDF4LLM
207
+
208
+ Args:
209
+ doc: The PyMuPDF document object.
210
+ pages: The pages. If omitted (None) all pages are processed.
211
+
212
+ Returns:
213
+ str: The content of the page in markdown.
214
+ """
215
+ try:
216
+ import pymupdf4llm
217
+ except ImportError:
218
+ raise ImportError(
219
+ "pymupdf4llm package not found, please install it "
220
+ "with `pip install pymupdf4llm`"
221
+ )
222
+
223
+ pymupdf4llm_params: dict[str, Any] = {
224
+ **self.pymupdf_layout_kwargs,
225
+ }
226
+
227
+ md = pymupdf4llm.to_markdown(
228
+ doc,
229
+ **pymupdf4llm_params,
230
+ )
231
+
232
+ return md
233
+
234
+ def _extract_metadata(self, doc: pymupdf.Document, blob: Blob) -> dict:
235
+ """Extract metadata from the PDF document.
236
+
237
+ Args:
238
+ doc: The PyMuPDF document object.
239
+ blob: The blob being parsed.
240
+
241
+ Returns:
242
+ dict: The extracted metadata from the PDF.
243
+ """
244
+ metadata = _purge_metadata(
245
+ {
246
+ **{
247
+ "producer": "PyMuPDF Layout",
248
+ "creator": "PyMuPDF Layout",
249
+ "creationdate": "",
250
+ "source": blob.source,
251
+ "file_path": blob.source,
252
+ "total_pages": len(doc),
253
+ },
254
+ **{
255
+ k: doc.metadata[k]
256
+ for k in doc.metadata
257
+ if isinstance(doc.metadata[k], (str, int))
258
+ },
259
+ }
260
+ )
261
+ for k in ("modDate", "creationDate"):
262
+ if k in doc.metadata:
263
+ metadata[k] = doc.metadata[k]
264
+ return metadata
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: langchain-pymupdf-layout
3
+ Version: 0.1.0
4
+ Summary: An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis.
5
+ Author-email: Jamie Lemon <jamie.lemon@artifex.com>
6
+ License: # PolyForm Noncommercial License 1.0.0
7
+
8
+ <https://polyformproject.org/licenses/noncommercial/1.0.0>
9
+
10
+ ## Acceptance
11
+
12
+ In order to get any license under these terms, you must agree
13
+ to them as both strict obligations and conditions to all
14
+ your licenses.
15
+
16
+ ## Copyright License
17
+
18
+ The licensor grants you a copyright license for the
19
+ software to do everything you might do with the software
20
+ that would otherwise infringe the licensor's copyright
21
+ in it for any permitted purpose. However, you may
22
+ only distribute the software according to [Distribution
23
+ License](#distribution-license) and make changes or new works
24
+ based on the software according to [Changes and New Works
25
+ License](#changes-and-new-works-license).
26
+
27
+ ## Distribution License
28
+
29
+ The licensor grants you an additional copyright license
30
+ to distribute copies of the software. Your license
31
+ to distribute covers distributing the software with
32
+ changes and new works permitted by [Changes and New Works
33
+ License](#changes-and-new-works-license).
34
+
35
+ ## Notices
36
+
37
+ You must ensure that anyone who gets a copy of any part of
38
+ the software from you also gets a copy of these terms or the
39
+ URL for them above, as well as copies of any plain-text lines
40
+ beginning with `Required Notice:` that the licensor provided
41
+ with the software. For example:
42
+
43
+ > Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
44
+
45
+ ## Changes and New Works License
46
+
47
+ The licensor grants you an additional copyright license to
48
+ make changes and new works based on the software for any
49
+ permitted purpose.
50
+
51
+ ## Patent License
52
+
53
+ The licensor grants you a patent license for the software that
54
+ covers patent claims the licensor can license, or becomes able
55
+ to license, that you would infringe by using the software.
56
+
57
+ ## Noncommercial Purposes
58
+
59
+ Any noncommercial purpose is a permitted purpose.
60
+
61
+ ## Personal Uses
62
+
63
+ Personal use for research, experiment, and testing for
64
+ the benefit of public knowledge, personal study, private
65
+ entertainment, hobby projects, amateur pursuits, or religious
66
+ observance, without any anticipated commercial application,
67
+ is use for a permitted purpose.
68
+
69
+ ## Noncommercial Organizations
70
+
71
+ Use by any charitable organization, educational institution,
72
+ public research organization, public safety or health
73
+ organization, environmental protection organization,
74
+ or government institution is use for a permitted purpose
75
+ regardless of the source of funding or obligations resulting
76
+ from the funding.
77
+
78
+ ## Fair Use
79
+
80
+ You may have "fair use" rights for the software under the
81
+ law. These terms do not limit them.
82
+
83
+ ## No Other Rights
84
+
85
+ These terms do not allow you to sublicense or transfer any of
86
+ your licenses to anyone else, or prevent the licensor from
87
+ granting licenses to anyone else. These terms do not imply
88
+ any other licenses.
89
+
90
+ ## Patent Defense
91
+
92
+ If you make any written claim that the software infringes or
93
+ contributes to infringement of any patent, your patent license
94
+ for the software granted under these terms ends immediately. If
95
+ your company makes such a claim, your patent license ends
96
+ immediately for work on behalf of your company.
97
+
98
+ ## Violations
99
+
100
+ The first time you are notified in writing that you have
101
+ violated any of these terms, or done anything with the software
102
+ not covered by your licenses, your licenses can nonetheless
103
+ continue if you come into full compliance with these terms,
104
+ and take practical steps to correct past violations, within
105
+ 32 days of receiving notice. Otherwise, all your licenses
106
+ end immediately.
107
+
108
+ ## No Liability
109
+
110
+ ***As far as the law allows, the software comes as is, without
111
+ any warranty or condition, and the licensor will not be liable
112
+ to you for any damages arising out of these terms or the use
113
+ or nature of the software, under any kind of legal claim.***
114
+
115
+ ## Definitions
116
+
117
+ The **licensor** is the individual or entity offering these
118
+ terms, and the **software** is the software the licensor makes
119
+ available under these terms.
120
+
121
+ **You** refers to the individual or entity agreeing to these
122
+ terms.
123
+
124
+ **Your company** is any legal entity, sole proprietorship,
125
+ or other kind of organization that you work for, plus all
126
+ organizations that have control over, are under the control of,
127
+ or are under common control with that organization. **Control**
128
+ means ownership of substantially all the assets of an entity,
129
+ or the power to direct its management and policies by vote,
130
+ contract, or otherwise. Control can be direct or indirect.
131
+
132
+ **Your licenses** are all the licenses granted to you for the
133
+ software under these terms.
134
+
135
+ **Use** means anything you do with the software requiring one
136
+ of your licenses.
137
+
138
+ Requires-Python: >=3.7
139
+ Description-Content-Type: text/markdown
140
+ License-File: LICENSE
141
+ Requires-Dist: langchain-core<2.0.0,>=1.0.0
142
+ Requires-Dist: pymupdf>=0.26.6
143
+ Requires-Dist: pymupdf-layout>=0.26.6
144
+ Requires-Dist: pymupdf4llm>=0.2.0
145
+ Dynamic: license-file
146
+
147
+ # langchain-pymupdf-layout
148
+
149
+ An integration package connecting **PyMuPDF Layout** to **LangChain**.
150
+
151
+ Load PDF content to Markdown using AI-based, CPU only, layout analysis.
152
+
153
+ [![LangChain](https://img.shields.io/badge/LangChain-v1.0+-blue)](https://github.com/langchain-ai/langchain)
154
+ [![Python version](https://img.shields.io/badge/python-3.11+-blue)](https://pypi.org/project/pymupdf-layout/)
155
+ [![License PolyForm Noncommercial](https://img.shields.io/badge/license-Polyform_Noncommercial-purple)](https://polyformproject.org/licenses/noncommercial/1.0.0/)
156
+
157
+ ## Features
158
+
159
+ - 📚 Structured data extraction from your documents
160
+ - 🧐 Advanced document page layout understanding, including semantic markup for titles, headings, headers, footers, tables, images and text styling
161
+ - 🔍 Detect and isolate header and footer patterns on each page
162
+
163
+ For more detailed information visit the [official PyMuPDF Layout documentation webpage](https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html).
164
+
165
+ ## Requirements
166
+
167
+ - Python 3.11 or higher
168
+ - LangChain Core v1.0.0 or higher
169
+ - PyMuPDF v0.26.6 or higher
170
+ - PyMuPDF4LLM v0.2.0 or higher
171
+ - PyMuPDF Layout c0.26.6 or higher
172
+
173
+ ## Installation
174
+
175
+ Install the package using pip to start using the Document Loader:
176
+
177
+ ```bash
178
+ pip install -U langchain-pymupdf-layout
179
+
180
+ ```
181
+
182
+ ## Usage
183
+
184
+ You can easily integrate and use the PyMuPDF Layout Loader in your Python application for loading and parsing PDFs.
185
+
186
+ Below is an example of how to set up and utilize this loader:
187
+
188
+ ```python
189
+ from langchain_pymupdf_layout import version
190
+
191
+ print(version()) # Output: version number
192
+
193
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
194
+
195
+ from langchain_pymupdf_layout import PyMuPDFLayoutLoader
196
+
197
+ loader = PyMuPDFLayoutLoader(
198
+ file_path="https://www.adobe.com/support/products/enterprise/knowledgecenter/media/c4611_sample_explain.pdf",
199
+ show_progress=False,
200
+ # See other loader options on https://pymupdf.readthedocs.io/en/latest/pymupdf-layout/index.html#pymupdf-layout-and-parameter-caveats
201
+ )
202
+
203
+ documents = loader.load()
204
+
205
+ # Chunk
206
+ text_splitter = RecursiveCharacterTextSplitter(
207
+ chunk_size=1000,
208
+ chunk_overlap=200
209
+ )
210
+
211
+ chunks = text_splitter.split_documents(documents)
212
+
213
+ print(f"Loaded {len(documents)} document(s)")
214
+ print(f"Created {len(chunks)} chunk(s)")
215
+
216
+ content = chunks[0].page_content
217
+ print(f"\ncontent:\n{content}")
218
+
219
+ ```
@@ -0,0 +1,11 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ langchain_pymupdf_layout/__init__.py
5
+ langchain_pymupdf_layout/pymupdf_layout_loader.py
6
+ langchain_pymupdf_layout/pymupdf_layout_parser.py
7
+ langchain_pymupdf_layout.egg-info/PKG-INFO
8
+ langchain_pymupdf_layout.egg-info/SOURCES.txt
9
+ langchain_pymupdf_layout.egg-info/dependency_links.txt
10
+ langchain_pymupdf_layout.egg-info/requires.txt
11
+ langchain_pymupdf_layout.egg-info/top_level.txt
@@ -0,0 +1,4 @@
1
+ langchain-core<2.0.0,>=1.0.0
2
+ pymupdf>=0.26.6
3
+ pymupdf-layout>=0.26.6
4
+ pymupdf4llm>=0.2.0
@@ -0,0 +1 @@
1
+ langchain_pymupdf_layout
@@ -0,0 +1,19 @@
1
+ [build-system]
2
+ requires = ["setuptools>=45", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "langchain-pymupdf-layout"
7
+ version = "0.1.0"
8
+ description = "An integration package connecting PyMuPDF Layout to LangChain. Load PDF content to Markdown using AI-based, CPU only, layout analysis."
9
+ readme = "README.md"
10
+ authors = [{name = "Jamie Lemon", email = "jamie.lemon@artifex.com"}]
11
+ license = {file = "LICENSE"}
12
+ requires-python = ">=3.7"
13
+
14
+ dependencies = [
15
+ "langchain-core>=1.0.0,<2.0.0",
16
+ "pymupdf>=0.26.6",
17
+ "pymupdf-layout>=0.26.6",
18
+ "pymupdf4llm>=0.2.0"
19
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+