langchain-fastocr 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_fastocr-0.1.0/.gitignore +12 -0
- langchain_fastocr-0.1.0/CHANGELOG.md +13 -0
- langchain_fastocr-0.1.0/LICENSE +21 -0
- langchain_fastocr-0.1.0/PKG-INFO +168 -0
- langchain_fastocr-0.1.0/README.md +133 -0
- langchain_fastocr-0.1.0/docs/images/logo-dark.png +0 -0
- langchain_fastocr-0.1.0/docs/images/logo-light.png +0 -0
- langchain_fastocr-0.1.0/docs/images/readme-hero.gif +0 -0
- langchain_fastocr-0.1.0/docs/images/readme-hero.png +0 -0
- langchain_fastocr-0.1.0/pyproject.toml +74 -0
- langchain_fastocr-0.1.0/src/langchain_fastocr/__init__.py +6 -0
- langchain_fastocr-0.1.0/src/langchain_fastocr/_version.py +1 -0
- langchain_fastocr-0.1.0/src/langchain_fastocr/loader.py +90 -0
- langchain_fastocr-0.1.0/src/langchain_fastocr/py.typed +0 -0
- langchain_fastocr-0.1.0/tests/test_loader.py +172 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this package are listed here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow
|
|
5
|
+
[semantic versioning](https://semver.org).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-10-24
|
|
8
|
+
|
|
9
|
+
First release. `FastOCRLoader`, a LangChain document loader for scanned PDFs and images
|
|
10
|
+
(PDF, PNG, JPG, JPEG, GIF, WebP, BMP). One document per file, with `source`, `file_name`,
|
|
11
|
+
`fastocr_document_id`, `pages_billed` and `status` in the metadata. `output="pdf"` also saves
|
|
12
|
+
a searchable PDF beside the input. Needs `fastocr-sdk` 0.2.0 or later. API requests send the
|
|
13
|
+
User-Agent `langchain-fastocr/<version>` followed by the SDK's own.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 FastOCR
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: langchain-fastocr
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain document loader for the FastOCR Document OCR API: load scanned PDFs and images as text, in 100+ languages.
|
|
5
|
+
Project-URL: Homepage, https://fastocr.org
|
|
6
|
+
Project-URL: Documentation, https://fastocr.org/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/fastocr-org/langchain-fastocr
|
|
8
|
+
Project-URL: Changelog, https://github.com/fastocr-org/langchain-fastocr/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Issues, https://github.com/fastocr-org/langchain-fastocr/issues
|
|
10
|
+
Author-email: FastOCR <support@fastocr.org>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: document-loader,fastocr,langchain,ocr,pdf,rag,scanned-pdf
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: fastocr-sdk<0.3,>=0.2.0
|
|
28
|
+
Requires-Dist: langchain-core>=0.3.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=7.4.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: respx>=0.21.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
<div align="center">
|
|
37
|
+
|
|
38
|
+
<a href="https://fastocr.org">
|
|
39
|
+
<picture>
|
|
40
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/logo-dark.png">
|
|
41
|
+
<img src="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/logo-light.png" alt="FastOCR" width="200">
|
|
42
|
+
</picture>
|
|
43
|
+
</a>
|
|
44
|
+
|
|
45
|
+
<h3>A LangChain document loader that reads scanned PDFs and images with FastOCR.</h3>
|
|
46
|
+
|
|
47
|
+
<p>
|
|
48
|
+
<a href="https://github.com/fastocr-org/langchain-fastocr/actions/workflows/ci.yml"><img src="https://github.com/fastocr-org/langchain-fastocr/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
49
|
+
<a href="https://pypi.org/project/langchain-fastocr/"><img src="https://img.shields.io/pypi/v/langchain-fastocr.svg?label=pypi" alt="PyPI"></a>
|
|
50
|
+
<a href="https://pypi.org/project/langchain-fastocr/"><img src="https://img.shields.io/pypi/pyversions/langchain-fastocr.svg" alt="Python"></a>
|
|
51
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT"></a>
|
|
52
|
+
</p>
|
|
53
|
+
|
|
54
|
+
<p>
|
|
55
|
+
<a href="https://fastocr.org">Website</a> ·
|
|
56
|
+
<a href="https://fastocr.org/docs">Docs</a> ·
|
|
57
|
+
<a href="https://fastocr.org/developers/keys">Get an API key</a>
|
|
58
|
+
</p>
|
|
59
|
+
|
|
60
|
+
<img src="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/readme-hero.gif" alt="FastOCR reads a scanned page in English, Arabic and Hindi and returns its text" width="800">
|
|
61
|
+
|
|
62
|
+
</div>
|
|
63
|
+
|
|
64
|
+
## Quickstart
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install langchain-fastocr
|
|
68
|
+
export FASTOCR_API_KEY="your_api_key"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from langchain_fastocr import FastOCRLoader
|
|
73
|
+
|
|
74
|
+
documents = FastOCRLoader("scanned_contract.pdf").load()
|
|
75
|
+
print(documents[0].page_content)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Python 3.10 or newer. The distribution is `langchain-fastocr` and the import is `langchain_fastocr`.
|
|
79
|
+
|
|
80
|
+
## What you get back
|
|
81
|
+
|
|
82
|
+
`FastOCRLoader` returns one `Document` per file.
|
|
83
|
+
|
|
84
|
+
| You get | Where |
|
|
85
|
+
|---|---|
|
|
86
|
+
| The text of the whole file | `page_content` |
|
|
87
|
+
| The file's path and name | `metadata["source"]`, `metadata["file_name"]` |
|
|
88
|
+
| The FastOCR document id | `metadata["fastocr_document_id"]` |
|
|
89
|
+
| Pages billed | `metadata["pages_billed"]` |
|
|
90
|
+
| `completed` or `partial` | `metadata["status"]` |
|
|
91
|
+
| A searchable PDF, with `output="pdf"` | saved as `<name>.searchable.pdf` beside the input; its path is in `metadata["searchable_pdf_path"]` |
|
|
92
|
+
|
|
93
|
+
If the account runs out of API pages partway through a file, you still get the text of the pages that were processed, with `status` set to `partial` and a warning in the log. A document that fails raises the error from [`fastocr-sdk`](https://pypi.org/project/fastocr-sdk/) with its message and code.
|
|
94
|
+
|
|
95
|
+
## Languages and files
|
|
96
|
+
|
|
97
|
+
- 100+ languages, detected automatically. No language setting is needed.
|
|
98
|
+
- PDF files up to 1 GiB, and PNG, JPG, JPEG, GIF, WebP and BMP images up to 20 MiB. An image counts as one page.
|
|
99
|
+
|
|
100
|
+
## Get an API key and pages
|
|
101
|
+
|
|
102
|
+
Any signed-in FastOCR account can create an API key and start right away.
|
|
103
|
+
|
|
104
|
+
1. Sign in and create an API key in the [developer console](https://fastocr.org/developers/keys). You accept the [API terms](https://fastocr.org/api-terms) with your first key.
|
|
105
|
+
2. Set the key as `FASTOCR_API_KEY`, or pass `api_key=` to the loader.
|
|
106
|
+
3. When you need more than the free pages, buy a prepaid API page pack by card in the [developer console](https://fastocr.org/developers/keys#buy-api-pages).
|
|
107
|
+
|
|
108
|
+
Each account gets 50 free document pages per month.
|
|
109
|
+
|
|
110
|
+
## Examples
|
|
111
|
+
|
|
112
|
+
### RAG over scanned PDFs
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from langchain_core.vectorstores import InMemoryVectorStore
|
|
116
|
+
from langchain_fastocr import FastOCRLoader
|
|
117
|
+
from langchain_openai import OpenAIEmbeddings
|
|
118
|
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
119
|
+
|
|
120
|
+
documents = FastOCRLoader("scanned_contract.pdf").load()
|
|
121
|
+
chunks = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150).split_documents(documents)
|
|
122
|
+
|
|
123
|
+
store = InMemoryVectorStore.from_documents(chunks, OpenAIEmbeddings())
|
|
124
|
+
for hit in store.as_retriever(search_kwargs={"k": 3}).invoke("When can the contract be terminated?"):
|
|
125
|
+
print(hit.metadata["file_name"], hit.page_content[:200])
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Each chunk keeps its file's metadata, so answers can cite the source file. This example also needs `pip install langchain-openai langchain-text-splitters numpy` and an OpenAI key; any other embeddings class works the same way.
|
|
129
|
+
|
|
130
|
+
### A folder of scans
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from pathlib import Path
|
|
134
|
+
|
|
135
|
+
from langchain_fastocr import FastOCRLoader
|
|
136
|
+
|
|
137
|
+
scans = sorted(p for p in Path("scans").iterdir() if p.suffix.lower() in {".pdf", ".png", ".jpg", ".jpeg"})
|
|
138
|
+
for document in FastOCRLoader(scans).lazy_load():
|
|
139
|
+
print(document.metadata["file_name"], document.metadata["pages_billed"])
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
`lazy_load()` yields each document as soon as its file is finished. The loader runs one file at a time.
|
|
143
|
+
|
|
144
|
+
### Searchable PDFs for an archive
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from langchain_fastocr import FastOCRLoader
|
|
148
|
+
|
|
149
|
+
documents = FastOCRLoader(["invoice-001.pdf", "invoice-002.pdf"], output="pdf").load()
|
|
150
|
+
for document in documents:
|
|
151
|
+
print(document.metadata["searchable_pdf_path"])
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Each searchable PDF is saved beside its input, as `invoice-001.searchable.pdf`. An existing file with that name is replaced. Index the text and keep the PDF.
|
|
155
|
+
|
|
156
|
+
## Options
|
|
157
|
+
|
|
158
|
+
| Argument | Default | Meaning |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| `file_path` | required | One path or a list of paths |
|
|
161
|
+
| `api_key` | `FASTOCR_API_KEY` | Your FastOCR API key |
|
|
162
|
+
| `output` | `"text"` | `"pdf"` also saves a searchable PDF beside each input |
|
|
163
|
+
| `base_url` | `FASTOCR_BASE_URL`, then `https://api.fastocr.org` | API address |
|
|
164
|
+
| `timeout` | `300.0` | Seconds to wait for each document |
|
|
165
|
+
|
|
166
|
+
## Support and license
|
|
167
|
+
|
|
168
|
+
Questions: support@fastocr.org. Bugs and requests: [GitHub issues](https://github.com/fastocr-org/langchain-fastocr/issues). MIT license; see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<a href="https://fastocr.org">
|
|
4
|
+
<picture>
|
|
5
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/logo-dark.png">
|
|
6
|
+
<img src="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/logo-light.png" alt="FastOCR" width="200">
|
|
7
|
+
</picture>
|
|
8
|
+
</a>
|
|
9
|
+
|
|
10
|
+
<h3>A LangChain document loader that reads scanned PDFs and images with FastOCR.</h3>
|
|
11
|
+
|
|
12
|
+
<p>
|
|
13
|
+
<a href="https://github.com/fastocr-org/langchain-fastocr/actions/workflows/ci.yml"><img src="https://github.com/fastocr-org/langchain-fastocr/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
14
|
+
<a href="https://pypi.org/project/langchain-fastocr/"><img src="https://img.shields.io/pypi/v/langchain-fastocr.svg?label=pypi" alt="PyPI"></a>
|
|
15
|
+
<a href="https://pypi.org/project/langchain-fastocr/"><img src="https://img.shields.io/pypi/pyversions/langchain-fastocr.svg" alt="Python"></a>
|
|
16
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT"></a>
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
<p>
|
|
20
|
+
<a href="https://fastocr.org">Website</a> ·
|
|
21
|
+
<a href="https://fastocr.org/docs">Docs</a> ·
|
|
22
|
+
<a href="https://fastocr.org/developers/keys">Get an API key</a>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
<img src="https://raw.githubusercontent.com/fastocr-org/langchain-fastocr/main/docs/images/readme-hero.gif" alt="FastOCR reads a scanned page in English, Arabic and Hindi and returns its text" width="800">
|
|
26
|
+
|
|
27
|
+
</div>
|
|
28
|
+
|
|
29
|
+
## Quickstart
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install langchain-fastocr
|
|
33
|
+
export FASTOCR_API_KEY="your_api_key"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from langchain_fastocr import FastOCRLoader
|
|
38
|
+
|
|
39
|
+
documents = FastOCRLoader("scanned_contract.pdf").load()
|
|
40
|
+
print(documents[0].page_content)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Python 3.10 or newer. The distribution is `langchain-fastocr` and the import is `langchain_fastocr`.
|
|
44
|
+
|
|
45
|
+
## What you get back
|
|
46
|
+
|
|
47
|
+
`FastOCRLoader` returns one `Document` per file.
|
|
48
|
+
|
|
49
|
+
| You get | Where |
|
|
50
|
+
|---|---|
|
|
51
|
+
| The text of the whole file | `page_content` |
|
|
52
|
+
| The file's path and name | `metadata["source"]`, `metadata["file_name"]` |
|
|
53
|
+
| The FastOCR document id | `metadata["fastocr_document_id"]` |
|
|
54
|
+
| Pages billed | `metadata["pages_billed"]` |
|
|
55
|
+
| `completed` or `partial` | `metadata["status"]` |
|
|
56
|
+
| A searchable PDF, with `output="pdf"` | saved as `<name>.searchable.pdf` beside the input; its path is in `metadata["searchable_pdf_path"]` |
|
|
57
|
+
|
|
58
|
+
If the account runs out of API pages partway through a file, you still get the text of the pages that were processed, with `status` set to `partial` and a warning in the log. A document that fails raises the error from [`fastocr-sdk`](https://pypi.org/project/fastocr-sdk/) with its message and code.
|
|
59
|
+
|
|
60
|
+
## Languages and files
|
|
61
|
+
|
|
62
|
+
- 100+ languages, detected automatically. No language setting is needed.
|
|
63
|
+
- PDF files up to 1 GiB, and PNG, JPG, JPEG, GIF, WebP and BMP images up to 20 MiB. An image counts as one page.
|
|
64
|
+
|
|
65
|
+
## Get an API key and pages
|
|
66
|
+
|
|
67
|
+
Any signed-in FastOCR account can create an API key and start right away.
|
|
68
|
+
|
|
69
|
+
1. Sign in and create an API key in the [developer console](https://fastocr.org/developers/keys). You accept the [API terms](https://fastocr.org/api-terms) with your first key.
|
|
70
|
+
2. Set the key as `FASTOCR_API_KEY`, or pass `api_key=` to the loader.
|
|
71
|
+
3. When you need more than the free pages, buy a prepaid API page pack by card in the [developer console](https://fastocr.org/developers/keys#buy-api-pages).
|
|
72
|
+
|
|
73
|
+
Each account gets 50 free document pages per month.
|
|
74
|
+
|
|
75
|
+
## Examples
|
|
76
|
+
|
|
77
|
+
### RAG over scanned PDFs
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from langchain_core.vectorstores import InMemoryVectorStore
|
|
81
|
+
from langchain_fastocr import FastOCRLoader
|
|
82
|
+
from langchain_openai import OpenAIEmbeddings
|
|
83
|
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
84
|
+
|
|
85
|
+
documents = FastOCRLoader("scanned_contract.pdf").load()
|
|
86
|
+
chunks = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150).split_documents(documents)
|
|
87
|
+
|
|
88
|
+
store = InMemoryVectorStore.from_documents(chunks, OpenAIEmbeddings())
|
|
89
|
+
for hit in store.as_retriever(search_kwargs={"k": 3}).invoke("When can the contract be terminated?"):
|
|
90
|
+
print(hit.metadata["file_name"], hit.page_content[:200])
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Each chunk keeps its file's metadata, so answers can cite the source file. This example also needs `pip install langchain-openai langchain-text-splitters numpy` and an OpenAI key; any other embeddings class works the same way.
|
|
94
|
+
|
|
95
|
+
### A folder of scans
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from pathlib import Path
|
|
99
|
+
|
|
100
|
+
from langchain_fastocr import FastOCRLoader
|
|
101
|
+
|
|
102
|
+
scans = sorted(p for p in Path("scans").iterdir() if p.suffix.lower() in {".pdf", ".png", ".jpg", ".jpeg"})
|
|
103
|
+
for document in FastOCRLoader(scans).lazy_load():
|
|
104
|
+
print(document.metadata["file_name"], document.metadata["pages_billed"])
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`lazy_load()` yields each document as soon as its file is finished. The loader runs one file at a time.
|
|
108
|
+
|
|
109
|
+
### Searchable PDFs for an archive
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from langchain_fastocr import FastOCRLoader
|
|
113
|
+
|
|
114
|
+
documents = FastOCRLoader(["invoice-001.pdf", "invoice-002.pdf"], output="pdf").load()
|
|
115
|
+
for document in documents:
|
|
116
|
+
print(document.metadata["searchable_pdf_path"])
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Each searchable PDF is saved beside its input, as `invoice-001.searchable.pdf`. An existing file with that name is replaced. Index the text and keep the PDF.
|
|
120
|
+
|
|
121
|
+
## Options
|
|
122
|
+
|
|
123
|
+
| Argument | Default | Meaning |
|
|
124
|
+
|---|---|---|
|
|
125
|
+
| `file_path` | required | One path or a list of paths |
|
|
126
|
+
| `api_key` | `FASTOCR_API_KEY` | Your FastOCR API key |
|
|
127
|
+
| `output` | `"text"` | `"pdf"` also saves a searchable PDF beside each input |
|
|
128
|
+
| `base_url` | `FASTOCR_BASE_URL`, then `https://api.fastocr.org` | API address |
|
|
129
|
+
| `timeout` | `300.0` | Seconds to wait for each document |
|
|
130
|
+
|
|
131
|
+
## Support and license
|
|
132
|
+
|
|
133
|
+
Questions: support@fastocr.org. Bugs and requests: [GitHub issues](https://github.com/fastocr-org/langchain-fastocr/issues). MIT license; see [LICENSE](LICENSE).
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "langchain-fastocr"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "LangChain document loader for the FastOCR Document OCR API: load scanned PDFs and images as text, in 100+ languages."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = ["langchain", "document-loader", "ocr", "pdf", "scanned-pdf", "rag", "fastocr"]
|
|
14
|
+
authors = [
|
|
15
|
+
{ name = "FastOCR", email = "support@fastocr.org" }
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Programming Language :: Python :: 3.14",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
dependencies = [
|
|
32
|
+
"fastocr-sdk>=0.2.0,<0.3",
|
|
33
|
+
"langchain-core>=0.3.0",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = [
|
|
38
|
+
"pytest>=7.4.0",
|
|
39
|
+
"respx>=0.21.0",
|
|
40
|
+
"ruff>=0.6",
|
|
41
|
+
"mypy>=1.10",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://fastocr.org"
|
|
46
|
+
Documentation = "https://fastocr.org/docs"
|
|
47
|
+
Repository = "https://github.com/fastocr-org/langchain-fastocr"
|
|
48
|
+
Changelog = "https://github.com/fastocr-org/langchain-fastocr/blob/main/CHANGELOG.md"
|
|
49
|
+
Issues = "https://github.com/fastocr-org/langchain-fastocr/issues"
|
|
50
|
+
|
|
51
|
+
[tool.hatch.version]
|
|
52
|
+
path = "src/langchain_fastocr/_version.py"
|
|
53
|
+
|
|
54
|
+
[tool.hatch.build.targets.wheel]
|
|
55
|
+
packages = ["src/langchain_fastocr"]
|
|
56
|
+
|
|
57
|
+
[tool.hatch.build.targets.sdist]
|
|
58
|
+
include = ["src", "tests", "docs", "README.md", "LICENSE", "CHANGELOG.md"]
|
|
59
|
+
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
testpaths = ["tests"]
|
|
62
|
+
|
|
63
|
+
[tool.ruff]
|
|
64
|
+
line-length = 120
|
|
65
|
+
target-version = "py310"
|
|
66
|
+
src = ["src", "tests"]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint]
|
|
69
|
+
select = ["E4", "E7", "E9", "F", "I"]
|
|
70
|
+
|
|
71
|
+
[tool.mypy]
|
|
72
|
+
python_version = "3.10"
|
|
73
|
+
strict = true
|
|
74
|
+
files = ["src"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Document loader that reads scanned PDFs and images with FastOCR."""
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Dict, Iterator, Literal, Optional, Sequence, Union
|
|
5
|
+
|
|
6
|
+
from fastocr_sdk import FastOCR
|
|
7
|
+
from langchain_core.document_loaders import BaseLoader
|
|
8
|
+
from langchain_core.documents import Document
|
|
9
|
+
|
|
10
|
+
from ._version import __version__
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
USER_AGENT = f"langchain-fastocr/{__version__}"
|
|
15
|
+
SUPPORTED_SUFFIXES = (".pdf", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")
|
|
16
|
+
|
|
17
|
+
PathLike = Union[str, Path]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FastOCRLoader(BaseLoader):
|
|
21
|
+
"""Load scanned PDFs and images as LangChain documents, one document per file.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
file_path: One path or a list of paths (PDF, PNG, JPG, JPEG, GIF, WebP or BMP).
|
|
25
|
+
api_key: A FastOCR API key. Defaults to the FASTOCR_API_KEY environment variable.
|
|
26
|
+
output: "text" returns the text. "pdf" also saves a searchable PDF beside each
|
|
27
|
+
input file as <name>.searchable.pdf and records its path in the metadata.
|
|
28
|
+
base_url: Overrides the API address. Defaults to FASTOCR_BASE_URL, then https://api.fastocr.org.
|
|
29
|
+
timeout: Seconds to wait for each document to finish.
|
|
30
|
+
|
|
31
|
+
Each document's metadata has source, file_name, fastocr_document_id, pages_billed and
|
|
32
|
+
status. A "partial" status means the account ran out of API pages partway: the text
|
|
33
|
+
covers only the processed pages and a warning is logged. A failed document raises the
|
|
34
|
+
SDK's DocumentFailedError.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
file_path: Union[PathLike, Sequence[PathLike]],
|
|
40
|
+
*,
|
|
41
|
+
api_key: Optional[str] = None,
|
|
42
|
+
output: Literal["text", "pdf"] = "text",
|
|
43
|
+
base_url: Optional[str] = None,
|
|
44
|
+
timeout: float = 300.0,
|
|
45
|
+
) -> None:
|
|
46
|
+
if output not in ("text", "pdf"):
|
|
47
|
+
raise ValueError("output must be 'text' or 'pdf'")
|
|
48
|
+
paths = [file_path] if isinstance(file_path, (str, Path)) else list(file_path)
|
|
49
|
+
self.paths = [Path(path).expanduser() for path in paths]
|
|
50
|
+
for path in self.paths:
|
|
51
|
+
if path.suffix.lower() not in SUPPORTED_SUFFIXES:
|
|
52
|
+
raise ValueError(f"{path.name}: FastOCR accepts PDF, PNG, JPG, JPEG, GIF, WebP and BMP files")
|
|
53
|
+
self.api_key = api_key
|
|
54
|
+
self.output = output
|
|
55
|
+
self.base_url = base_url
|
|
56
|
+
self.timeout = timeout
|
|
57
|
+
|
|
58
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
59
|
+
with self._client() as client:
|
|
60
|
+
for path in self.paths:
|
|
61
|
+
yield self._load_one(client, path)
|
|
62
|
+
|
|
63
|
+
def _client(self) -> FastOCR:
|
|
64
|
+
client = FastOCR(api_key=self.api_key, base_url=self.base_url)
|
|
65
|
+
sdk_agent = client._http.headers.get("User-Agent", "")
|
|
66
|
+
client._http.headers["User-Agent"] = f"{USER_AGENT} {sdk_agent}".strip()
|
|
67
|
+
return client
|
|
68
|
+
|
|
69
|
+
def _load_one(self, client: FastOCR, path: Path) -> Document:
|
|
70
|
+
job = client.documents.process(path, timeout=self.timeout)
|
|
71
|
+
if job.is_partial:
|
|
72
|
+
logger.warning(
|
|
73
|
+
"%s: only %s of %s pages were processed because the account ran out of API pages",
|
|
74
|
+
path.name,
|
|
75
|
+
job.pages_billed,
|
|
76
|
+
job.pages_total,
|
|
77
|
+
)
|
|
78
|
+
metadata: Dict[str, Any] = {
|
|
79
|
+
"source": str(path),
|
|
80
|
+
"file_name": path.name,
|
|
81
|
+
"fastocr_document_id": job.id,
|
|
82
|
+
"pages_billed": job.pages_billed,
|
|
83
|
+
"status": job.status,
|
|
84
|
+
}
|
|
85
|
+
text = client.documents.get_text(job.id)
|
|
86
|
+
if self.output == "pdf":
|
|
87
|
+
destination = path.with_name(f"{path.stem}.searchable.pdf")
|
|
88
|
+
client.documents.download_searchable_pdf(job.id, destination)
|
|
89
|
+
metadata["searchable_pdf_path"] = str(destination)
|
|
90
|
+
return Document(page_content=text, metadata=metadata)
|
|
File without changes
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""The loader end to end, with the FastOCR API mocked at the HTTP layer."""
|
|
2
|
+
import logging
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
import pytest
|
|
6
|
+
import respx
|
|
7
|
+
from fastocr_sdk import AuthenticationError, DocumentFailedError
|
|
8
|
+
from langchain_core.document_loaders import BaseLoader
|
|
9
|
+
|
|
10
|
+
from langchain_fastocr import FastOCRLoader, __version__
|
|
11
|
+
|
|
12
|
+
BASE_URL = "https://api.test"
|
|
13
|
+
UPLOAD_URL = "https://s3.test/uploads/original?X-Amz-Signature=sig"
|
|
14
|
+
TEXT_URL = "https://s3.test/out/ocr-text.txt?X-Amz-Signature=sig"
|
|
15
|
+
PDF_URL = "https://s3.test/out/searchable.pdf?X-Amz-Signature=sig"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def job(doc_id, status="completed", pages=2, total=2):
|
|
19
|
+
return {
|
|
20
|
+
"id": doc_id,
|
|
21
|
+
"status": status,
|
|
22
|
+
"pages_billed": pages,
|
|
23
|
+
"pages_total": total,
|
|
24
|
+
"outputs": {"text": {"available": True}, "pdf": {"available": True}},
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.fixture
|
|
29
|
+
def api():
|
|
30
|
+
with respx.mock(assert_all_called=False) as router:
|
|
31
|
+
yield router
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def mock_document(api, doc_id, finished, text):
|
|
35
|
+
api.put(UPLOAD_URL).respond(200)
|
|
36
|
+
api.post(f"{BASE_URL}/v1/documents/{doc_id}/start").respond(202, json={"status": "processing"})
|
|
37
|
+
api.get(f"{BASE_URL}/v1/documents/{doc_id}").respond(200, json=finished)
|
|
38
|
+
api.get(f"{BASE_URL}/v1/documents/{doc_id}/output", params={"format": "text"}).respond(
|
|
39
|
+
200, json={"format": "text", "url": TEXT_URL + doc_id, "expires_at": "x"}
|
|
40
|
+
)
|
|
41
|
+
api.get(f"{BASE_URL}/v1/documents/{doc_id}/output", params={"format": "pdf"}).respond(
|
|
42
|
+
200, json={"format": "pdf", "url": PDF_URL, "expires_at": "x"}
|
|
43
|
+
)
|
|
44
|
+
api.get(TEXT_URL + doc_id).respond(200, content=text.encode("utf-8"))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def mock_creates(api, *doc_ids):
|
|
48
|
+
responses = [
|
|
49
|
+
httpx.Response(201, json={"id": doc_id, "status": "awaiting_upload", "upload_url": UPLOAD_URL, "expires_at": "x"})
|
|
50
|
+
for doc_id in doc_ids
|
|
51
|
+
]
|
|
52
|
+
return api.post(f"{BASE_URL}/v1/documents").mock(side_effect=responses)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def loader(paths, **options):
|
|
56
|
+
return FastOCRLoader(paths, api_key="fok_test", base_url=BASE_URL, **options)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_is_a_langchain_loader():
|
|
60
|
+
assert issubclass(FastOCRLoader, BaseLoader)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_loads_an_image(api, tmp_path):
|
|
64
|
+
image = tmp_path / "receipt.png"
|
|
65
|
+
image.write_bytes(b"\x89PNG test")
|
|
66
|
+
mock_creates(api, "doc_1")
|
|
67
|
+
mock_document(api, "doc_1", job("doc_1", pages=1, total=1), "مرحبا بالعالم")
|
|
68
|
+
|
|
69
|
+
[document] = loader(image).load()
|
|
70
|
+
|
|
71
|
+
assert document.page_content == "مرحبا بالعالم"
|
|
72
|
+
assert document.metadata == {
|
|
73
|
+
"source": str(image),
|
|
74
|
+
"file_name": "receipt.png",
|
|
75
|
+
"fastocr_document_id": "doc_1",
|
|
76
|
+
"pages_billed": 1,
|
|
77
|
+
"status": "completed",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_loads_a_list_of_files_one_document_each(api, tmp_path):
|
|
82
|
+
first, second = tmp_path / "a.pdf", tmp_path / "b.PDF"
|
|
83
|
+
first.write_bytes(b"%PDF-1.4 a")
|
|
84
|
+
second.write_bytes(b"%PDF-1.4 b")
|
|
85
|
+
mock_creates(api, "doc_a", "doc_b")
|
|
86
|
+
mock_document(api, "doc_a", job("doc_a"), "text of a")
|
|
87
|
+
mock_document(api, "doc_b", job("doc_b"), "text of b")
|
|
88
|
+
|
|
89
|
+
documents = loader([str(first), second]).load()
|
|
90
|
+
|
|
91
|
+
assert [d.page_content for d in documents] == ["text of a", "text of b"]
|
|
92
|
+
assert [d.metadata["file_name"] for d in documents] == ["a.pdf", "b.PDF"]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_pdf_output_saves_the_searchable_pdf_beside_the_input(api, tmp_path):
|
|
96
|
+
scan = tmp_path / "scan.v2.pdf"
|
|
97
|
+
scan.write_bytes(b"%PDF-1.4 scan")
|
|
98
|
+
mock_creates(api, "doc_1")
|
|
99
|
+
mock_document(api, "doc_1", job("doc_1"), "page text")
|
|
100
|
+
api.get(PDF_URL).respond(200, content=b"%PDF searchable")
|
|
101
|
+
|
|
102
|
+
[document] = loader(scan, output="pdf").load()
|
|
103
|
+
|
|
104
|
+
saved = tmp_path / "scan.v2.searchable.pdf"
|
|
105
|
+
assert saved.read_bytes() == b"%PDF searchable"
|
|
106
|
+
assert document.metadata["searchable_pdf_path"] == str(saved)
|
|
107
|
+
assert document.page_content == "page text"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_partial_result_keeps_the_text_and_logs_a_warning(api, tmp_path, caplog):
|
|
111
|
+
scan = tmp_path / "big.pdf"
|
|
112
|
+
scan.write_bytes(b"%PDF-1.4 big")
|
|
113
|
+
mock_creates(api, "doc_1")
|
|
114
|
+
mock_document(api, "doc_1", job("doc_1", status="partial", pages=3, total=10), "first three pages")
|
|
115
|
+
|
|
116
|
+
with caplog.at_level(logging.WARNING, logger="langchain_fastocr"):
|
|
117
|
+
[document] = loader(scan).load()
|
|
118
|
+
|
|
119
|
+
assert document.page_content == "first three pages"
|
|
120
|
+
assert document.metadata["status"] == "partial"
|
|
121
|
+
assert "3 of 10 pages" in caplog.text
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_failed_document_raises_the_sdk_error(api, tmp_path):
|
|
125
|
+
scan = tmp_path / "bad.pdf"
|
|
126
|
+
scan.write_bytes(b"%PDF-1.4 bad")
|
|
127
|
+
mock_creates(api, "doc_1")
|
|
128
|
+
failed = {"id": "doc_1", "status": "failed", "error": {"type": "x", "code": "buy_pages", "message": "No pages", "retryable": False}}
|
|
129
|
+
mock_document(api, "doc_1", failed, "")
|
|
130
|
+
|
|
131
|
+
with pytest.raises(DocumentFailedError) as raised:
|
|
132
|
+
loader(scan).load()
|
|
133
|
+
|
|
134
|
+
assert raised.value.out_of_pages
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_rejected_key_raises_authentication_error(api, tmp_path):
|
|
138
|
+
scan = tmp_path / "scan.pdf"
|
|
139
|
+
scan.write_bytes(b"%PDF-1.4 scan")
|
|
140
|
+
api.post(f"{BASE_URL}/v1/documents").respond(401, json={"message": "Unauthorized"})
|
|
141
|
+
|
|
142
|
+
with pytest.raises(AuthenticationError):
|
|
143
|
+
loader(scan).load()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_requests_name_this_package_ahead_of_the_sdk(api, tmp_path):
|
|
147
|
+
scan = tmp_path / "scan.pdf"
|
|
148
|
+
scan.write_bytes(b"%PDF-1.4 scan")
|
|
149
|
+
create = mock_creates(api, "doc_1")
|
|
150
|
+
mock_document(api, "doc_1", job("doc_1"), "text")
|
|
151
|
+
|
|
152
|
+
loader(scan).load()
|
|
153
|
+
|
|
154
|
+
agent = create.calls[0].request.headers["User-Agent"]
|
|
155
|
+
assert agent.startswith(f"langchain-fastocr/{__version__} fastocr-python/")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_unsupported_file_type_is_rejected_before_any_request(tmp_path):
|
|
159
|
+
with pytest.raises(ValueError, match="notes.docx"):
|
|
160
|
+
FastOCRLoader(tmp_path / "notes.docx", api_key="fok_test")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_api_key_defaults_to_the_environment(api, tmp_path, monkeypatch):
|
|
164
|
+
scan = tmp_path / "scan.pdf"
|
|
165
|
+
scan.write_bytes(b"%PDF-1.4 scan")
|
|
166
|
+
monkeypatch.setenv("FASTOCR_API_KEY", "fok_from_env")
|
|
167
|
+
create = mock_creates(api, "doc_1")
|
|
168
|
+
mock_document(api, "doc_1", job("doc_1"), "text")
|
|
169
|
+
|
|
170
|
+
FastOCRLoader(scan, base_url=BASE_URL).load()
|
|
171
|
+
|
|
172
|
+
assert create.calls[0].request.headers["Authorization"] == "Bearer fok_from_env"
|