fastdocparse 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastdocparse-0.3.0/PKG-INFO +163 -0
- fastdocparse-0.3.0/README.md +130 -0
- fastdocparse-0.3.0/pyproject.toml +66 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/__init__.py +29 -15
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/cache.py +8 -7
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/cli.py +102 -10
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/grounding.py +23 -25
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/json_repair.py +11 -7
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/llm_client.py +5 -4
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/ocr_engine.py +11 -7
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/parser.py +40 -28
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/pdf_utils.py +29 -20
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/prompt_compiler.py +6 -4
- fastdocparse-0.3.0/src/fastdocparse/result.py +75 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/schema.py +14 -12
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/schema_compiler.py +2 -2
- fastdocparse-0.3.0/src/fastdocparse.egg-info/PKG-INFO +163 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/tests/test_architecture.py +7 -2
- fastdocparse-0.3.0/tests/test_cli.py +241 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/tests/test_parser.py +5 -4
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/tests/test_schema_compiler.py +2 -1
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/tests/test_severe_edge_cases.py +65 -5
- fastdocparse-0.2.0/PKG-INFO +0 -136
- fastdocparse-0.2.0/README.md +0 -117
- fastdocparse-0.2.0/pyproject.toml +0 -35
- fastdocparse-0.2.0/src/fastdocparse/result.py +0 -42
- fastdocparse-0.2.0/src/fastdocparse.egg-info/PKG-INFO +0 -136
- fastdocparse-0.2.0/tests/test_cli.py +0 -129
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/LICENSE +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/setup.cfg +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/config.py +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/example_schemas.py +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/py.typed +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/schemas/invoice.json +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse/schemas/shipment_manifest.json +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse.egg-info/SOURCES.txt +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse.egg-info/dependency_links.txt +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse.egg-info/entry_points.txt +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse.egg-info/requires.txt +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/src/fastdocparse.egg-info/top_level.txt +0 -0
- {fastdocparse-0.2.0 → fastdocparse-0.3.0}/tests/test_grounding.py +0 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastdocparse
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Extract structured data from semi-structured documents using any OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/pranjalparmar/fastdocparse
|
|
7
|
+
Project-URL: Repository, https://github.com/pranjalparmar/fastdocparse
|
|
8
|
+
Project-URL: Issues, https://github.com/pranjalparmar/fastdocparse/issues
|
|
9
|
+
Project-URL: Documentation, https://github.com/pranjalparmar/fastdocparse/tree/main/docs
|
|
10
|
+
Keywords: document-extraction,llm,pdf,ocr,invoice-extraction,structured-data,openai,ollama,grounding
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: <3.13,>=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: pydantic>=2.0
|
|
24
|
+
Requires-Dist: openai>=1.0
|
|
25
|
+
Requires-Dist: pymupdf>=1.24
|
|
26
|
+
Requires-Dist: pillow>=10.0
|
|
27
|
+
Requires-Dist: rapidocr-onnxruntime>=1.3
|
|
28
|
+
Requires-Dist: typer>=0.12
|
|
29
|
+
Requires-Dist: PyYAML>=6.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# fastdocparse
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/fastdocparse/)
|
|
37
|
+
[](https://github.com/pranjalparmar/fastdocparse/actions/workflows/ci.yml)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
Extract structured data from semi-structured documents (invoices, bills, tax forms, resumes, bank statements, shipment manifests) using any OpenAI-compatible LLM (OpenAI, Ollama, vLLM, Groq, etc.), with **per-field grounding and confidence**, not just raw extraction.
|
|
41
|
+
|
|
42
|
+
## Why this, not just another parser
|
|
43
|
+
|
|
44
|
+
Most extractors give you a value and no way to know if it's real. This one tells you:
|
|
45
|
+
|
|
46
|
+
- **`grounded`**: the value was found verbatim (or near-verbatim) in the source document text.
|
|
47
|
+
- **`ungrounded`**: the value doesn't appear in the source, likely a hallucination. Flag for human review.
|
|
48
|
+
- **`missing_required`**: a field you marked required came back empty.
|
|
49
|
+
- **`invalid_format`**: the value doesn't match a pattern/enum constraint you declared (e.g. a shipment status outside the allowed list).
|
|
50
|
+
- **`failed_check`**: a custom cross-field rule failed (e.g. line items don't sum to the stated total).
|
|
51
|
+
|
|
52
|
+
No extra LLM call for any of this: it's deterministic, string/rule-based validation against text you already extracted.
|
|
53
|
+
|
|
54
|
+
**Where it fits:** semi-structured documents with recurring fields (invoices, bills, tax forms, resumes, statements), and prose documents where *proving* a value came from the source matters (contracts, legal clauses, insurance claims). It is not a vision-LLM pipeline. It works from extracted text (digital PDF text layer, or local OCR for scans/images), which is what keeps it fast, cheap, and usable with small local models. Messy handwritten forms or complex multi-column layouts are a known weaker spot (see [document-extractor-spec.md](document-extractor-spec.md)).
|
|
55
|
+
|
|
56
|
+
## How this compares
|
|
57
|
+
|
|
58
|
+
| Project | Approach | Where it beats fastdocparse | Where fastdocparse can beat it |
|
|
59
|
+
|---|---|---|---|
|
|
60
|
+
| **[Sparrow](https://github.com/katanaml/sparrow)** (katanaml) | Vision-LLM first (MLX/vLLM/Ollama/Mistral OCR), multi-service platform, API-first | Layout-aware (sees the page), mature, table templates for complex tables | No GPU required, single `pip install`, Pydantic-schema devex vs. raw JSON-string CLI args |
|
|
61
|
+
| **[LangExtract](https://github.com/google/langextract)** (Google) | Text-first, source-grounding (character-offset mapping), few-shot examples required | Real grounding/traceability, brand trust | Purpose-built for documents (OCR routing) vs. text-in/text-out |
|
|
62
|
+
|
|
63
|
+
**The honest edge:** *fast, cheap, local-model-friendly extraction for clean-to-moderate documents*, not "better than Sparrow at everything." This has not yet been validated with a real head-to-head benchmark ([tracking issue](https://github.com/pranjalparmar/fastdocparse/issues/19)). Treat it as a design goal, not a proven claim, until that lands.
|
|
64
|
+
|
|
65
|
+
## Two ways to use it
|
|
66
|
+
|
|
67
|
+
| | Who it's for | How |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| **CLI** | No coding needed | `fastdocparse extract <file> <schema.json>` / `fastdocparse --version` |
|
|
70
|
+
| **Python API** | Building it into your own app | `DocumentParser(client).extract(document_bytes, schema)` |
|
|
71
|
+
|
|
72
|
+
Defining *what* to extract also has two paths: hand-write a JSON/YAML schema file, or describe it in plain English and let the LLM draft the schema for you.
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install fastdocparse
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
For local development instead:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
git clone https://github.com/pranjalparmar/fastdocparse
|
|
84
|
+
cd fastdocparse
|
|
85
|
+
python -m venv venv
|
|
86
|
+
source venv/bin/activate # Windows: venv\Scripts\activate
|
|
87
|
+
pip install -e ".[dev]"
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
You also need access to an LLM. Either:
|
|
91
|
+
- An OpenAI API key (`export OPENAI_API_KEY=...` or pass `--api-key`), or
|
|
92
|
+
- A local model via [Ollama](https://ollama.com/): no API key, no cloud, documents never leave your machine.
|
|
93
|
+
|
|
94
|
+
## Quickstart: CLI (no coding)
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# 1. Extract using one of the bundled example schemas
|
|
98
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
99
|
+
--model gpt-4o-mini --api-key sk-...
|
|
100
|
+
|
|
101
|
+
# Or with a local model via Ollama (no API key needed):
|
|
102
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
103
|
+
--model llama3.2 --base-url http://localhost:11434/v1 --api-key ollama
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Output is JSON, printed to stdout (or saved with `--output result.json`):
|
|
107
|
+
|
|
108
|
+
```json
|
|
109
|
+
{
|
|
110
|
+
"_meta": { "truncated": false, "truncation_reason": null },
|
|
111
|
+
"invoice_number": { "value": "INV-9011", "confidence": "high", "flags": ["grounded"] },
|
|
112
|
+
"total_price": { "value": 100.0, "confidence": "high", "flags": ["grounded"] }
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Don't want to write JSON at all? Describe the fields in plain English instead:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
fastdocparse schema-from-text \
|
|
120
|
+
"I want the invoice number, total price, and vendor name. Invoice number and total are required." \
|
|
121
|
+
--output my_invoice_schema.json
|
|
122
|
+
|
|
123
|
+
# review my_invoice_schema.json, then:
|
|
124
|
+
fastdocparse extract my_invoice.pdf my_invoice_schema.json
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Quickstart: Python API
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
131
|
+
|
|
132
|
+
schema = Schema(
|
|
133
|
+
name="Invoice",
|
|
134
|
+
fields=[
|
|
135
|
+
Field(name="invoice_number", description="The invoice number", required=True),
|
|
136
|
+
Field(name="total_price", description="Total amount due", type="number", required=True),
|
|
137
|
+
],
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
client = LLMClient(model="gpt-4o-mini", api_key="sk-...")
|
|
141
|
+
# or: LLMClient(base_url="http://localhost:11434/v1", api_key="ollama", model="llama3.2")
|
|
142
|
+
|
|
143
|
+
parser = DocumentParser(client=client)
|
|
144
|
+
|
|
145
|
+
with open("invoice.pdf", "rb") as f:
|
|
146
|
+
result = parser.extract(f.read(), schema)
|
|
147
|
+
|
|
148
|
+
print(result["invoice_number"]) # {'value': 'INV-9011', 'confidence': 'high', 'flags': ['grounded']}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Full documentation
|
|
152
|
+
|
|
153
|
+
- [Getting Started](docs/getting-started.md): step-by-step install, CLI, and API walkthroughs
|
|
154
|
+
- [Schema Guide](docs/schema-guide.md): every field option (`type`, `required`, `pattern`, `enum`, `sub_fields`, few-shot `examples`), for JSON, YAML, and plain-English authoring
|
|
155
|
+
- [Output & Validation](docs/output-format.md): the full result shape, what each confidence flag means, and how to write custom cross-check rules
|
|
156
|
+
- [Architecture](docs/architecture.md): diagrams of the pipeline, the module dependency graph, and where to plug in a contribution
|
|
157
|
+
- [Project spec](document-extractor-spec.md): architecture, phased roadmap, honest competitive positioning
|
|
158
|
+
|
|
159
|
+
Want to contribute? Start with [docs/architecture.md](docs/architecture.md) for the map, then [CONTRIBUTING.md](CONTRIBUTING.md) for the process.
|
|
160
|
+
|
|
161
|
+
## Status
|
|
162
|
+
|
|
163
|
+
Core extraction, grounding, chunking, both CLI/API paths, and real packaging are implemented and tested (75 tests, `pytest -v`). Published on PyPI as [`fastdocparse`](https://pypi.org/project/fastdocparse/). `pip install fastdocparse` installs a working `fastdocparse` command and a proper `fastdocparse.*` import namespace, verified end to end with a clean-virtualenv install straight from the real public index. Not yet done: a hosted API. See [document-extractor-spec.md](document-extractor-spec.md) for the roadmap.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# fastdocparse
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/fastdocparse/)
|
|
4
|
+
[](https://github.com/pranjalparmar/fastdocparse/actions/workflows/ci.yml)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Extract structured data from semi-structured documents (invoices, bills, tax forms, resumes, bank statements, shipment manifests) using any OpenAI-compatible LLM (OpenAI, Ollama, vLLM, Groq, etc.), with **per-field grounding and confidence**, not just raw extraction.
|
|
8
|
+
|
|
9
|
+
## Why this, not just another parser
|
|
10
|
+
|
|
11
|
+
Most extractors give you a value and no way to know if it's real. This one tells you:
|
|
12
|
+
|
|
13
|
+
- **`grounded`**: the value was found verbatim (or near-verbatim) in the source document text.
|
|
14
|
+
- **`ungrounded`**: the value doesn't appear in the source, likely a hallucination. Flag for human review.
|
|
15
|
+
- **`missing_required`**: a field you marked required came back empty.
|
|
16
|
+
- **`invalid_format`**: the value doesn't match a pattern/enum constraint you declared (e.g. a shipment status outside the allowed list).
|
|
17
|
+
- **`failed_check`**: a custom cross-field rule failed (e.g. line items don't sum to the stated total).
|
|
18
|
+
|
|
19
|
+
No extra LLM call for any of this: it's deterministic, string/rule-based validation against text you already extracted.
|
|
20
|
+
|
|
21
|
+
**Where it fits:** semi-structured documents with recurring fields (invoices, bills, tax forms, resumes, statements), and prose documents where *proving* a value came from the source matters (contracts, legal clauses, insurance claims). It is not a vision-LLM pipeline. It works from extracted text (digital PDF text layer, or local OCR for scans/images), which is what keeps it fast, cheap, and usable with small local models. Messy handwritten forms or complex multi-column layouts are a known weaker spot (see [document-extractor-spec.md](document-extractor-spec.md)).
|
|
22
|
+
|
|
23
|
+
## How this compares
|
|
24
|
+
|
|
25
|
+
| Project | Approach | Where it beats fastdocparse | Where fastdocparse can beat it |
|
|
26
|
+
|---|---|---|---|
|
|
27
|
+
| **[Sparrow](https://github.com/katanaml/sparrow)** (katanaml) | Vision-LLM first (MLX/vLLM/Ollama/Mistral OCR), multi-service platform, API-first | Layout-aware (sees the page), mature, table templates for complex tables | No GPU required, single `pip install`, Pydantic-schema devex vs. raw JSON-string CLI args |
|
|
28
|
+
| **[LangExtract](https://github.com/google/langextract)** (Google) | Text-first, source-grounding (character-offset mapping), few-shot examples required | Real grounding/traceability, brand trust | Purpose-built for documents (OCR routing) vs. text-in/text-out |
|
|
29
|
+
|
|
30
|
+
**The honest edge:** *fast, cheap, local-model-friendly extraction for clean-to-moderate documents*, not "better than Sparrow at everything." This has not yet been validated with a real head-to-head benchmark ([tracking issue](https://github.com/pranjalparmar/fastdocparse/issues/19)). Treat it as a design goal, not a proven claim, until that lands.
|
|
31
|
+
|
|
32
|
+
## Two ways to use it
|
|
33
|
+
|
|
34
|
+
| | Who it's for | How |
|
|
35
|
+
|---|---|---|
|
|
36
|
+
| **CLI** | No coding needed | `fastdocparse extract <file> <schema.json>` / `fastdocparse --version` |
|
|
37
|
+
| **Python API** | Building it into your own app | `DocumentParser(client).extract(document_bytes, schema)` |
|
|
38
|
+
|
|
39
|
+
Defining *what* to extract also has two paths: hand-write a JSON/YAML schema file, or describe it in plain English and let the LLM draft the schema for you.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install fastdocparse
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
For local development instead:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
git clone https://github.com/pranjalparmar/fastdocparse
|
|
51
|
+
cd fastdocparse
|
|
52
|
+
python -m venv venv
|
|
53
|
+
source venv/bin/activate # Windows: venv\Scripts\activate
|
|
54
|
+
pip install -e ".[dev]"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
You also need access to an LLM. Either:
|
|
58
|
+
- An OpenAI API key (`export OPENAI_API_KEY=...` or pass `--api-key`), or
|
|
59
|
+
- A local model via [Ollama](https://ollama.com/): no API key, no cloud, documents never leave your machine.
|
|
60
|
+
|
|
61
|
+
## Quickstart: CLI (no coding)
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# 1. Extract using one of the bundled example schemas
|
|
65
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
66
|
+
--model gpt-4o-mini --api-key sk-...
|
|
67
|
+
|
|
68
|
+
# Or with a local model via Ollama (no API key needed):
|
|
69
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
70
|
+
--model llama3.2 --base-url http://localhost:11434/v1 --api-key ollama
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Output is JSON, printed to stdout (or saved with `--output result.json`):
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"_meta": { "truncated": false, "truncation_reason": null },
|
|
78
|
+
"invoice_number": { "value": "INV-9011", "confidence": "high", "flags": ["grounded"] },
|
|
79
|
+
"total_price": { "value": 100.0, "confidence": "high", "flags": ["grounded"] }
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Don't want to write JSON at all? Describe the fields in plain English instead:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
fastdocparse schema-from-text \
|
|
87
|
+
"I want the invoice number, total price, and vendor name. Invoice number and total are required." \
|
|
88
|
+
--output my_invoice_schema.json
|
|
89
|
+
|
|
90
|
+
# review my_invoice_schema.json, then:
|
|
91
|
+
fastdocparse extract my_invoice.pdf my_invoice_schema.json
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Quickstart: Python API
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
98
|
+
|
|
99
|
+
schema = Schema(
|
|
100
|
+
name="Invoice",
|
|
101
|
+
fields=[
|
|
102
|
+
Field(name="invoice_number", description="The invoice number", required=True),
|
|
103
|
+
Field(name="total_price", description="Total amount due", type="number", required=True),
|
|
104
|
+
],
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
client = LLMClient(model="gpt-4o-mini", api_key="sk-...")
|
|
108
|
+
# or: LLMClient(base_url="http://localhost:11434/v1", api_key="ollama", model="llama3.2")
|
|
109
|
+
|
|
110
|
+
parser = DocumentParser(client=client)
|
|
111
|
+
|
|
112
|
+
with open("invoice.pdf", "rb") as f:
|
|
113
|
+
result = parser.extract(f.read(), schema)
|
|
114
|
+
|
|
115
|
+
print(result["invoice_number"]) # {'value': 'INV-9011', 'confidence': 'high', 'flags': ['grounded']}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Full documentation
|
|
119
|
+
|
|
120
|
+
- [Getting Started](docs/getting-started.md): step-by-step install, CLI, and API walkthroughs
|
|
121
|
+
- [Schema Guide](docs/schema-guide.md): every field option (`type`, `required`, `pattern`, `enum`, `sub_fields`, few-shot `examples`), for JSON, YAML, and plain-English authoring
|
|
122
|
+
- [Output & Validation](docs/output-format.md): the full result shape, what each confidence flag means, and how to write custom cross-check rules
|
|
123
|
+
- [Architecture](docs/architecture.md): diagrams of the pipeline, the module dependency graph, and where to plug in a contribution
|
|
124
|
+
- [Project spec](document-extractor-spec.md): architecture, phased roadmap, honest competitive positioning
|
|
125
|
+
|
|
126
|
+
Want to contribute? Start with [docs/architecture.md](docs/architecture.md) for the map, then [CONTRIBUTING.md](CONTRIBUTING.md) for the process.
|
|
127
|
+
|
|
128
|
+
## Status
|
|
129
|
+
|
|
130
|
+
Core extraction, grounding, chunking, both CLI/API paths, and real packaging are implemented and tested (75 tests, `pytest -v`). Published on PyPI as [`fastdocparse`](https://pypi.org/project/fastdocparse/). `pip install fastdocparse` installs a working `fastdocparse` command and a proper `fastdocparse.*` import namespace, verified end to end with a clean-virtualenv install straight from the real public index. Not yet done: a hosted API. See [document-extractor-spec.md](document-extractor-spec.md) for the roadmap.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "fastdocparse"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Extract structured data from semi-structured documents using any OpenAI-compatible LLM, with per-field grounding and confidence."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.9,<3.13" # capped by rapidocr-onnxruntime's own upper bound (checked live against PyPI)
|
|
8
|
+
keywords = [
|
|
9
|
+
"document-extraction",
|
|
10
|
+
"llm",
|
|
11
|
+
"pdf",
|
|
12
|
+
"ocr",
|
|
13
|
+
"invoice-extraction",
|
|
14
|
+
"structured-data",
|
|
15
|
+
"openai",
|
|
16
|
+
"ollama",
|
|
17
|
+
"grounding",
|
|
18
|
+
]
|
|
19
|
+
classifiers = [
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.9",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Topic :: Text Processing :: Linguistic",
|
|
28
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
29
|
+
]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"pydantic>=2.0",
|
|
32
|
+
"openai>=1.0",
|
|
33
|
+
"pymupdf>=1.24",
|
|
34
|
+
"pillow>=10.0",
|
|
35
|
+
"rapidocr-onnxruntime>=1.3",
|
|
36
|
+
"typer>=0.12",
|
|
37
|
+
"PyYAML>=6.0",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.optional-dependencies]
|
|
41
|
+
dev = ["pytest"]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/pranjalparmar/fastdocparse"
|
|
45
|
+
Repository = "https://github.com/pranjalparmar/fastdocparse"
|
|
46
|
+
Issues = "https://github.com/pranjalparmar/fastdocparse/issues"
|
|
47
|
+
Documentation = "https://github.com/pranjalparmar/fastdocparse/tree/main/docs"
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
fastdocparse = "fastdocparse.cli:app"
|
|
51
|
+
|
|
52
|
+
[build-system]
|
|
53
|
+
requires = ["setuptools>=68"]
|
|
54
|
+
build-backend = "setuptools.build_meta"
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.packages.find]
|
|
57
|
+
where = ["src"]
|
|
58
|
+
|
|
59
|
+
[tool.setuptools.package-data]
|
|
60
|
+
fastdocparse = ["schemas/*.json", "py.typed"]
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint.per-file-ignores]
|
|
66
|
+
"src/fastdocparse/cli.py" = ["B008"]
|
|
@@ -10,41 +10,55 @@ OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
|
10
10
|
|
|
11
11
|
from .cache import Cache, InMemoryCache
|
|
12
12
|
from .config import ExtractionConfig
|
|
13
|
-
from .grounding import
|
|
13
|
+
from .grounding import (
|
|
14
|
+
Issue,
|
|
15
|
+
check_substring,
|
|
16
|
+
cross_check,
|
|
17
|
+
date_parseable_rule,
|
|
18
|
+
numeric_sum_rule,
|
|
19
|
+
validate_field_constraints,
|
|
20
|
+
)
|
|
14
21
|
from .llm_client import LLMClient, LLMClientError
|
|
15
|
-
from .parser import
|
|
22
|
+
from .parser import (
|
|
23
|
+
DocumentParser,
|
|
24
|
+
EmptyDocumentError,
|
|
25
|
+
UnknownIngestionKindError,
|
|
26
|
+
register_default_ingestion_handler,
|
|
27
|
+
)
|
|
16
28
|
from .result import ExtractionMeta, ExtractionResult, FieldResult
|
|
17
29
|
from .schema import Field, Schema
|
|
18
30
|
from .schema_compiler import compile_schema_from_description
|
|
19
31
|
|
|
20
32
|
try:
|
|
33
|
+
from importlib.metadata import PackageNotFoundError
|
|
21
34
|
from importlib.metadata import version as _pkg_version
|
|
35
|
+
|
|
22
36
|
# Looks up by the PyPI *distribution* name (pyproject.toml's [project] name) — keep
|
|
23
37
|
# this string in sync with that if the distribution is ever renamed again.
|
|
24
38
|
__version__ = _pkg_version("fastdocparse")
|
|
25
|
-
except
|
|
39
|
+
except PackageNotFoundError:
|
|
26
40
|
__version__ = "0.0.0+unknown"
|
|
27
41
|
|
|
28
42
|
__all__ = [
|
|
29
|
-
"
|
|
30
|
-
"Field",
|
|
31
|
-
"LLMClient",
|
|
32
|
-
"LLMClientError",
|
|
43
|
+
"Cache",
|
|
33
44
|
"DocumentParser",
|
|
34
45
|
"EmptyDocumentError",
|
|
35
|
-
"UnknownIngestionKindError",
|
|
36
|
-
"register_default_ingestion_handler",
|
|
37
46
|
"ExtractionConfig",
|
|
38
|
-
"
|
|
39
|
-
"InMemoryCache",
|
|
47
|
+
"ExtractionMeta",
|
|
40
48
|
"ExtractionResult",
|
|
49
|
+
"Field",
|
|
41
50
|
"FieldResult",
|
|
42
|
-
"
|
|
51
|
+
"InMemoryCache",
|
|
43
52
|
"Issue",
|
|
53
|
+
"LLMClient",
|
|
54
|
+
"LLMClientError",
|
|
55
|
+
"Schema",
|
|
56
|
+
"UnknownIngestionKindError",
|
|
44
57
|
"check_substring",
|
|
58
|
+
"compile_schema_from_description",
|
|
45
59
|
"cross_check",
|
|
46
|
-
"validate_field_constraints",
|
|
47
|
-
"numeric_sum_rule",
|
|
48
60
|
"date_parseable_rule",
|
|
49
|
-
"
|
|
61
|
+
"numeric_sum_rule",
|
|
62
|
+
"register_default_ingestion_handler",
|
|
63
|
+
"validate_field_constraints",
|
|
50
64
|
]
|
|
@@ -4,19 +4,20 @@ Caching is skipped whenever custom `rules` are passed to extract(), since a rule
|
|
|
4
4
|
arbitrary callable that can't be safely fingerprinted — caching would risk returning a
|
|
5
5
|
result validated under a different rule than the one just requested.
|
|
6
6
|
"""
|
|
7
|
+
from __future__ import annotations
|
|
7
8
|
|
|
8
9
|
import hashlib
|
|
9
10
|
import json
|
|
10
11
|
from collections import OrderedDict
|
|
11
|
-
from typing import Any, Callable,
|
|
12
|
+
from typing import Any, Callable, Protocol
|
|
12
13
|
|
|
13
14
|
from .config import ExtractionConfig
|
|
14
15
|
from .schema import Schema
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
class Cache(Protocol):
|
|
18
|
-
def get(self, key: str) ->
|
|
19
|
-
def set(self, key: str, value:
|
|
19
|
+
def get(self, key: str) -> dict[str, Any] | None: ...
|
|
20
|
+
def set(self, key: str, value: dict[str, Any]) -> None: ...
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
class InMemoryCache:
|
|
@@ -27,19 +28,19 @@ class InMemoryCache:
|
|
|
27
28
|
process exits before the cache could grow unbounded.
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
|
-
def __init__(self, max_size:
|
|
31
|
+
def __init__(self, max_size: int | None = None):
|
|
31
32
|
if max_size is not None and max_size <= 0:
|
|
32
33
|
raise ValueError(f"max_size must be positive, got {max_size}")
|
|
33
34
|
self._max_size = max_size
|
|
34
|
-
self._store:
|
|
35
|
+
self._store: OrderedDict[str, dict[str, Any]] = OrderedDict()
|
|
35
36
|
|
|
36
|
-
def get(self, key: str) ->
|
|
37
|
+
def get(self, key: str) -> dict[str, Any] | None:
|
|
37
38
|
if key not in self._store:
|
|
38
39
|
return None
|
|
39
40
|
self._store.move_to_end(key)
|
|
40
41
|
return self._store[key]
|
|
41
42
|
|
|
42
|
-
def set(self, key: str, value:
|
|
43
|
+
def set(self, key: str, value: dict[str, Any]) -> None:
|
|
43
44
|
self._store[key] = value
|
|
44
45
|
self._store.move_to_end(key)
|
|
45
46
|
if self._max_size is not None:
|
|
@@ -4,16 +4,17 @@ Usage:
|
|
|
4
4
|
fastdocparse extract document.pdf invoice_schema.json
|
|
5
5
|
fastdocparse extract receipt.jpg shipment_schema.json --model llama3 --base-url http://localhost:11434/v1
|
|
6
6
|
"""
|
|
7
|
+
from __future__ import annotations
|
|
7
8
|
|
|
8
9
|
import importlib
|
|
9
10
|
import json
|
|
10
11
|
import os
|
|
11
12
|
from pathlib import Path
|
|
12
|
-
from typing import Optional
|
|
13
13
|
|
|
14
14
|
import typer
|
|
15
15
|
from pydantic import ValidationError
|
|
16
16
|
|
|
17
|
+
from . import __version__
|
|
17
18
|
from .llm_client import LLMClient, LLMClientError
|
|
18
19
|
from .parser import DocumentParser, EmptyDocumentError, UnknownIngestionKindError
|
|
19
20
|
from .schema import Schema
|
|
@@ -48,17 +49,82 @@ IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
|
|
|
48
49
|
SUPPORTED_EXTENSIONS = IMAGE_EXTENSIONS | {".pdf"}
|
|
49
50
|
|
|
50
51
|
|
|
52
|
+
def _check_llm_credentials(base_url: str | None, api_key: str | None) -> None:
|
|
53
|
+
"""Catch the common "nothing configured" case before making any network call, rather
|
|
54
|
+
than letting it surface as an OpenAI auth error after a request round-trip. A
|
|
55
|
+
`base_url` (local/self-hosted endpoint) needs no real key, so this only fires when
|
|
56
|
+
neither a base_url nor a key is present from any source (flag or env var). A blank
|
|
57
|
+
or whitespace-only value (e.g. an env var set to "") counts as absent, not present,
|
|
58
|
+
since otherwise it would silently reach LLMClient's own dummy-key fallback and
|
|
59
|
+
proceed to a doomed network call instead of failing here with a clear message."""
|
|
60
|
+
if (base_url and base_url.strip()) or (api_key and api_key.strip()):
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
typer.echo(
|
|
64
|
+
"Error: no LLM credentials configured.\n"
|
|
65
|
+
"\n"
|
|
66
|
+
"To use OpenAI, set an API key:\n"
|
|
67
|
+
" export OPENAI_API_KEY=sk-...\n"
|
|
68
|
+
" (or pass --api-key)\n"
|
|
69
|
+
"\n"
|
|
70
|
+
"To use a local model via Ollama instead, pass --base-url pointing at your\n"
|
|
71
|
+
"Ollama server. No real key is needed there; --api-key accepts any non-empty\n"
|
|
72
|
+
"value in that case.\n"
|
|
73
|
+
"\n"
|
|
74
|
+
"Run 'fastdocparse extract --help' for the full list of options.",
|
|
75
|
+
err=True,
|
|
76
|
+
)
|
|
77
|
+
raise typer.Exit(code=1)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _version_callback(value: bool) -> None:
|
|
81
|
+
if not value:
|
|
82
|
+
return
|
|
83
|
+
|
|
84
|
+
typer.echo(f"fastdocparse {__version__}")
|
|
85
|
+
raise typer.Exit()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@app.callback()
|
|
89
|
+
def main(
|
|
90
|
+
version: bool = typer.Option(
|
|
91
|
+
False,
|
|
92
|
+
"--version",
|
|
93
|
+
callback=_version_callback,
|
|
94
|
+
is_eager=True,
|
|
95
|
+
help="Show the version and exit.",
|
|
96
|
+
),
|
|
97
|
+
) -> None:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
51
101
|
@app.command()
|
|
52
102
|
def extract(
|
|
53
103
|
file: Path = typer.Argument(..., exists=True, readable=True, help="Path to the PDF/PNG/JPG document."),
|
|
54
|
-
schema: Path = typer.Argument(
|
|
55
|
-
model: str = typer.Option("gpt-4o-mini", "--model", "-m", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
56
|
-
base_url:
|
|
57
|
-
api_key:
|
|
58
|
-
output:
|
|
59
|
-
kind:
|
|
104
|
+
schema: Path | None = typer.Argument(None, exists=True, readable=True, help="Path to a .json or .yaml schema file listing the fields to extract."),
|
|
105
|
+
model: str = typer.Option("gpt-4o-mini", "--model", "-m", envvar="FASTDOCPARSE_MODEL", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
106
|
+
base_url: str | None = typer.Option(None, "--base-url", envvar="FASTDOCPARSE_BASE_URL", help="OpenAI-compatible API base URL. Omit for OpenAI; use e.g. http://localhost:11434/v1 for Ollama."),
|
|
107
|
+
api_key: str | None = typer.Option(None, "--api-key", envvar=["LLM_API_KEY", "OPENAI_API_KEY"], help="API key. Not needed for local Ollama."),
|
|
108
|
+
output: Path | None = typer.Option(None, "--output", "-o", help="Write the JSON result to this file instead of printing it."),
|
|
109
|
+
kind: str | None = typer.Option(None, "--kind", help="Override ingestion routing (e.g. 'docx' for a custom handler loaded via FASTDOCPARSE_PLUGINS). Defaults to auto-detecting pdf/image from the file extension."),
|
|
60
110
|
):
|
|
61
111
|
"""Extract the fields defined in SCHEMA from FILE and print the result as JSON."""
|
|
112
|
+
if schema is None:
|
|
113
|
+
typer.echo(
|
|
114
|
+
"Error: a schema file is required.\n"
|
|
115
|
+
"\n"
|
|
116
|
+
"Quickest way to create one — describe your fields in plain English:\n"
|
|
117
|
+
" fastdocparse schema-from-text \"invoice number, total amount, and line items\" --output my_schema.json\n"
|
|
118
|
+
"\n"
|
|
119
|
+
"Or start from a bundled example schema:\n"
|
|
120
|
+
" fastdocparse list-schemas\n"
|
|
121
|
+
"\n"
|
|
122
|
+
"Then extract with:\n"
|
|
123
|
+
" fastdocparse extract " + str(file) + " my_schema.json",
|
|
124
|
+
err=True,
|
|
125
|
+
)
|
|
126
|
+
raise typer.Exit(code=1)
|
|
127
|
+
|
|
62
128
|
if kind is None and file.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
63
129
|
typer.echo(f"Unsupported file type {file.suffix!r}. Supported: .pdf, .png, .jpg, .jpeg (or pass --kind).", err=True)
|
|
64
130
|
raise typer.Exit(code=1)
|
|
@@ -69,6 +135,8 @@ def extract(
|
|
|
69
135
|
typer.echo(f"Could not load schema from {schema}: {e}", err=True)
|
|
70
136
|
raise typer.Exit(code=1)
|
|
71
137
|
|
|
138
|
+
_check_llm_credentials(base_url, api_key)
|
|
139
|
+
|
|
72
140
|
client = LLMClient(base_url=base_url, api_key=api_key, model=model)
|
|
73
141
|
document_parser = DocumentParser(client=client)
|
|
74
142
|
|
|
@@ -108,9 +176,9 @@ def extract(
|
|
|
108
176
|
def schema_from_text(
|
|
109
177
|
description: str = typer.Argument(..., help="Plain-English description of the fields you want extracted, e.g. \"invoice number, total price, and a list of line items with product name and quantity\"."),
|
|
110
178
|
output: Path = typer.Option(..., "--output", "-o", help="Where to save the generated schema (.json)."),
|
|
111
|
-
model: str = typer.Option("gpt-4o-mini", "--model", "-m", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
112
|
-
base_url:
|
|
113
|
-
api_key:
|
|
179
|
+
model: str = typer.Option("gpt-4o-mini", "--model", "-m", envvar="FASTDOCPARSE_MODEL", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
180
|
+
base_url: str | None = typer.Option(None, "--base-url", envvar="FASTDOCPARSE_BASE_URL", help="OpenAI-compatible API base URL. Omit for OpenAI; use e.g. http://localhost:11434/v1 for Ollama."),
|
|
181
|
+
api_key: str | None = typer.Option(None, "--api-key", envvar=["LLM_API_KEY", "OPENAI_API_KEY"], help="API key. Not needed for local Ollama."),
|
|
114
182
|
):
|
|
115
183
|
"""Turn a plain-English description of the fields you want into a schema file.
|
|
116
184
|
|
|
@@ -118,6 +186,8 @@ def schema_from_text(
|
|
|
118
186
|
names and types from your description, and a wrong guess here affects every
|
|
119
187
|
document you later run against this schema.
|
|
120
188
|
"""
|
|
189
|
+
_check_llm_credentials(base_url, api_key)
|
|
190
|
+
|
|
121
191
|
client = LLMClient(base_url=base_url, api_key=api_key, model=model)
|
|
122
192
|
try:
|
|
123
193
|
doc_schema = compile_schema_from_description(description, client)
|
|
@@ -136,5 +206,27 @@ def schema_from_text(
|
|
|
136
206
|
typer.echo("Review it, then run: fastdocparse extract <your_document> " + str(output))
|
|
137
207
|
|
|
138
208
|
|
|
209
|
+
@app.command(name="list-schemas")
|
|
210
|
+
def list_schemas():
|
|
211
|
+
"""List the bundled example schemas you can copy as a starting point.
|
|
212
|
+
|
|
213
|
+
Copy any of the listed files into your project and pass the copy to `extract`.
|
|
214
|
+
This is the fastest way to get started without an LLM-generated schema.
|
|
215
|
+
"""
|
|
216
|
+
schemas_dir = Path(__file__).parent / "schemas"
|
|
217
|
+
schema_files = sorted(schemas_dir.glob("*.json"))
|
|
218
|
+
if not schema_files:
|
|
219
|
+
typer.echo("No bundled schemas found.", err=True)
|
|
220
|
+
raise typer.Exit(code=1)
|
|
221
|
+
|
|
222
|
+
typer.echo("Bundled example schemas (copy one as your starting point):\n")
|
|
223
|
+
for path in schema_files:
|
|
224
|
+
typer.echo(f" {path}")
|
|
225
|
+
typer.echo(
|
|
226
|
+
"\nTo use one directly:\n"
|
|
227
|
+
" fastdocparse extract document.pdf " + str(schema_files[0])
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
|
|
139
231
|
if __name__ == "__main__":
|
|
140
232
|
app()
|