fastdocparse 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastdocparse-0.1.0/src/fastdocparse.egg-info → fastdocparse-0.2.0}/PKG-INFO +18 -14
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/README.md +17 -13
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/pyproject.toml +3 -3
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/__init__.py +5 -3
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/cli.py +8 -8
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/ocr_engine.py +1 -1
- {fastdocparse-0.1.0 → fastdocparse-0.2.0/src/fastdocparse.egg-info}/PKG-INFO +18 -14
- fastdocparse-0.2.0/src/fastdocparse.egg-info/SOURCES.txt +33 -0
- fastdocparse-0.2.0/src/fastdocparse.egg-info/entry_points.txt +2 -0
- fastdocparse-0.2.0/src/fastdocparse.egg-info/top_level.txt +1 -0
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_architecture.py +16 -16
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_cli.py +8 -8
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_grounding.py +2 -2
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_parser.py +31 -32
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_schema_compiler.py +2 -2
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/tests/test_severe_edge_cases.py +8 -8
- fastdocparse-0.1.0/src/fastdocparse.egg-info/SOURCES.txt +0 -33
- fastdocparse-0.1.0/src/fastdocparse.egg-info/entry_points.txt +0 -2
- fastdocparse-0.1.0/src/fastdocparse.egg-info/top_level.txt +0 -1
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/LICENSE +0 -0
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/setup.cfg +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/cache.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/config.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/example_schemas.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/grounding.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/json_repair.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/llm_client.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/parser.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/pdf_utils.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/prompt_compiler.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/py.typed +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/result.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schema.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schema_compiler.py +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schemas/invoice.json +0 -0
- {fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schemas/shipment_manifest.json +0 -0
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/src/fastdocparse.egg-info/dependency_links.txt +0 -0
- {fastdocparse-0.1.0 → fastdocparse-0.2.0}/src/fastdocparse.egg-info/requires.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastdocparse
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Extract structured data from semi-structured documents using any OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Requires-Python: <3.13,>=3.9
|
|
@@ -17,7 +17,7 @@ Provides-Extra: dev
|
|
|
17
17
|
Requires-Dist: pytest; extra == "dev"
|
|
18
18
|
Dynamic: license-file
|
|
19
19
|
|
|
20
|
-
#
|
|
20
|
+
# fastdocparse
|
|
21
21
|
|
|
22
22
|
Extract structured data from semi-structured documents — invoices, bills, tax forms, resumes, bank statements, shipment manifests — using any OpenAI-compatible LLM (OpenAI, Ollama, vLLM, Groq, etc.), with **per-field grounding and confidence**, not just raw extraction.
|
|
23
23
|
|
|
@@ -39,7 +39,7 @@ No extra LLM call for any of this — it's deterministic, string/rule-based vali
|
|
|
39
39
|
|
|
40
40
|
| | Who it's for | How |
|
|
41
41
|
|---|---|---|
|
|
42
|
-
| **CLI** | No coding needed | `
|
|
42
|
+
| **CLI** | No coding needed | `fastdocparse extract <file> <schema.json>` |
|
|
43
43
|
| **Python API** | Building it into your own app | `DocumentParser(client).extract(document_bytes, schema)` |
|
|
44
44
|
|
|
45
45
|
Defining *what* to extract also has two paths — hand-write a JSON/YAML schema file, or describe it in plain English and let the LLM draft the schema for you.
|
|
@@ -47,15 +47,19 @@ Defining *what* to extract also has two paths — hand-write a JSON/YAML schema
|
|
|
47
47
|
## Install
|
|
48
48
|
|
|
49
49
|
```bash
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
pip install fastdocparse
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For local development instead:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
git clone https://github.com/pranjalparmar/fastdocparse
|
|
57
|
+
cd fastdocparse
|
|
52
58
|
python -m venv venv
|
|
53
59
|
source venv/bin/activate # Windows: venv\Scripts\activate
|
|
54
|
-
pip install -e .
|
|
60
|
+
pip install -e ".[dev]"
|
|
55
61
|
```
|
|
56
62
|
|
|
57
|
-
(Not yet published to PyPI — see [Status](#status). Until then, install from a local clone as above.)
|
|
58
|
-
|
|
59
63
|
You also need access to an LLM. Either:
|
|
60
64
|
- An OpenAI API key (`export OPENAI_API_KEY=...` or pass `--api-key`), or
|
|
61
65
|
- A local model via [Ollama](https://ollama.com/) — no API key, no cloud, documents never leave your machine.
|
|
@@ -64,11 +68,11 @@ You also need access to an LLM. Either:
|
|
|
64
68
|
|
|
65
69
|
```bash
|
|
66
70
|
# 1. Extract using one of the bundled example schemas
|
|
67
|
-
|
|
71
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
68
72
|
--model gpt-4o-mini --api-key sk-...
|
|
69
73
|
|
|
70
74
|
# Or with a local model via Ollama (no API key needed):
|
|
71
|
-
|
|
75
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
72
76
|
--model llama3.2 --base-url http://localhost:11434/v1 --api-key ollama
|
|
73
77
|
```
|
|
74
78
|
|
|
@@ -85,18 +89,18 @@ Output is JSON, printed to stdout (or saved with `--output result.json`):
|
|
|
85
89
|
Don't want to write JSON at all? Describe the fields in plain English instead:
|
|
86
90
|
|
|
87
91
|
```bash
|
|
88
|
-
|
|
92
|
+
fastdocparse schema-from-text \
|
|
89
93
|
"I want the invoice number, total price, and vendor name. Invoice number and total are required." \
|
|
90
94
|
--output my_invoice_schema.json
|
|
91
95
|
|
|
92
96
|
# review my_invoice_schema.json, then:
|
|
93
|
-
|
|
97
|
+
fastdocparse extract my_invoice.pdf my_invoice_schema.json
|
|
94
98
|
```
|
|
95
99
|
|
|
96
100
|
## Quickstart — Python API
|
|
97
101
|
|
|
98
102
|
```python
|
|
99
|
-
from
|
|
103
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
100
104
|
|
|
101
105
|
schema = Schema(
|
|
102
106
|
name="Invoice",
|
|
@@ -129,4 +133,4 @@ Want to contribute? Start with [docs/architecture.md](docs/architecture.md) for
|
|
|
129
133
|
|
|
130
134
|
## Status
|
|
131
135
|
|
|
132
|
-
Core extraction, grounding, chunking, both CLI/API paths, and real packaging (`pip install
|
|
136
|
+
Core extraction, grounding, chunking, both CLI/API paths, and real packaging are implemented and tested (74 tests, `pytest -v`). Published on PyPI as [`fastdocparse`](https://pypi.org/project/fastdocparse/) — `pip install fastdocparse` installs a working `fastdocparse` command and a proper `fastdocparse.*` import namespace, verified end to end with a clean-virtualenv install straight from the real public index. Not yet done: a hosted API — see [document-extractor-spec.md](document-extractor-spec.md) for the roadmap.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# fastdocparse
|
|
2
2
|
|
|
3
3
|
Extract structured data from semi-structured documents — invoices, bills, tax forms, resumes, bank statements, shipment manifests — using any OpenAI-compatible LLM (OpenAI, Ollama, vLLM, Groq, etc.), with **per-field grounding and confidence**, not just raw extraction.
|
|
4
4
|
|
|
@@ -20,7 +20,7 @@ No extra LLM call for any of this — it's deterministic, string/rule-based vali
|
|
|
20
20
|
|
|
21
21
|
| | Who it's for | How |
|
|
22
22
|
|---|---|---|
|
|
23
|
-
| **CLI** | No coding needed | `
|
|
23
|
+
| **CLI** | No coding needed | `fastdocparse extract <file> <schema.json>` |
|
|
24
24
|
| **Python API** | Building it into your own app | `DocumentParser(client).extract(document_bytes, schema)` |
|
|
25
25
|
|
|
26
26
|
Defining *what* to extract also has two paths — hand-write a JSON/YAML schema file, or describe it in plain English and let the LLM draft the schema for you.
|
|
@@ -28,15 +28,19 @@ Defining *what* to extract also has two paths — hand-write a JSON/YAML schema
|
|
|
28
28
|
## Install
|
|
29
29
|
|
|
30
30
|
```bash
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
pip install fastdocparse
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
For local development instead:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
git clone https://github.com/pranjalparmar/fastdocparse
|
|
38
|
+
cd fastdocparse
|
|
33
39
|
python -m venv venv
|
|
34
40
|
source venv/bin/activate # Windows: venv\Scripts\activate
|
|
35
|
-
pip install -e .
|
|
41
|
+
pip install -e ".[dev]"
|
|
36
42
|
```
|
|
37
43
|
|
|
38
|
-
(Not yet published to PyPI — see [Status](#status). Until then, install from a local clone as above.)
|
|
39
|
-
|
|
40
44
|
You also need access to an LLM. Either:
|
|
41
45
|
- An OpenAI API key (`export OPENAI_API_KEY=...` or pass `--api-key`), or
|
|
42
46
|
- A local model via [Ollama](https://ollama.com/) — no API key, no cloud, documents never leave your machine.
|
|
@@ -45,11 +49,11 @@ You also need access to an LLM. Either:
|
|
|
45
49
|
|
|
46
50
|
```bash
|
|
47
51
|
# 1. Extract using one of the bundled example schemas
|
|
48
|
-
|
|
52
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
49
53
|
--model gpt-4o-mini --api-key sk-...
|
|
50
54
|
|
|
51
55
|
# Or with a local model via Ollama (no API key needed):
|
|
52
|
-
|
|
56
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
53
57
|
--model llama3.2 --base-url http://localhost:11434/v1 --api-key ollama
|
|
54
58
|
```
|
|
55
59
|
|
|
@@ -66,18 +70,18 @@ Output is JSON, printed to stdout (or saved with `--output result.json`):
|
|
|
66
70
|
Don't want to write JSON at all? Describe the fields in plain English instead:
|
|
67
71
|
|
|
68
72
|
```bash
|
|
69
|
-
|
|
73
|
+
fastdocparse schema-from-text \
|
|
70
74
|
"I want the invoice number, total price, and vendor name. Invoice number and total are required." \
|
|
71
75
|
--output my_invoice_schema.json
|
|
72
76
|
|
|
73
77
|
# review my_invoice_schema.json, then:
|
|
74
|
-
|
|
78
|
+
fastdocparse extract my_invoice.pdf my_invoice_schema.json
|
|
75
79
|
```
|
|
76
80
|
|
|
77
81
|
## Quickstart — Python API
|
|
78
82
|
|
|
79
83
|
```python
|
|
80
|
-
from
|
|
84
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
81
85
|
|
|
82
86
|
schema = Schema(
|
|
83
87
|
name="Invoice",
|
|
@@ -110,4 +114,4 @@ Want to contribute? Start with [docs/architecture.md](docs/architecture.md) for
|
|
|
110
114
|
|
|
111
115
|
## Status
|
|
112
116
|
|
|
113
|
-
Core extraction, grounding, chunking, both CLI/API paths, and real packaging (`pip install
|
|
117
|
+
Core extraction, grounding, chunking, both CLI/API paths, and real packaging are implemented and tested (74 tests, `pytest -v`). Published on PyPI as [`fastdocparse`](https://pypi.org/project/fastdocparse/) — `pip install fastdocparse` installs a working `fastdocparse` command and a proper `fastdocparse.*` import namespace, verified end to end with a clean-virtualenv install straight from the real public index. Not yet done: a hosted API — see [document-extractor-spec.md](document-extractor-spec.md) for the roadmap.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fastdocparse"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Extract structured data from semi-structured documents using any OpenAI-compatible LLM, with per-field grounding and confidence."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -19,7 +19,7 @@ dependencies = [
|
|
|
19
19
|
dev = ["pytest"]
|
|
20
20
|
|
|
21
21
|
[project.scripts]
|
|
22
|
-
|
|
22
|
+
fastdocparse = "fastdocparse.cli:app"
|
|
23
23
|
|
|
24
24
|
[build-system]
|
|
25
25
|
requires = ["setuptools>=68"]
|
|
@@ -29,7 +29,7 @@ build-backend = "setuptools.build_meta"
|
|
|
29
29
|
where = ["src"]
|
|
30
30
|
|
|
31
31
|
[tool.setuptools.package-data]
|
|
32
|
-
|
|
32
|
+
fastdocparse = ["schemas/*.json", "py.typed"]
|
|
33
33
|
|
|
34
34
|
[tool.pytest.ini_options]
|
|
35
35
|
testpaths = ["tests"]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""fastdocparse — extract structured data from semi-structured documents using any
|
|
2
2
|
OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
3
3
|
|
|
4
|
-
from
|
|
4
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
5
5
|
|
|
6
6
|
schema = Schema(name="Invoice", fields=[Field(name="total", description="Grand total", type="number")])
|
|
7
7
|
client = LLMClient(model="gpt-4o-mini", api_key="sk-...")
|
|
@@ -19,7 +19,9 @@ from .schema_compiler import compile_schema_from_description
|
|
|
19
19
|
|
|
20
20
|
try:
|
|
21
21
|
from importlib.metadata import version as _pkg_version
|
|
22
|
-
|
|
22
|
+
# Looks up by the PyPI *distribution* name (pyproject.toml's [project] name) — keep
|
|
23
|
+
# this string in sync with that if the distribution is ever renamed again.
|
|
24
|
+
__version__ = _pkg_version("fastdocparse")
|
|
23
25
|
except Exception:
|
|
24
26
|
__version__ = "0.0.0+unknown"
|
|
25
27
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Command-line entrypoint: extract fields from a document without writing code.
|
|
2
2
|
|
|
3
3
|
Usage:
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
fastdocparse extract document.pdf invoice_schema.json
|
|
5
|
+
fastdocparse extract receipt.jpg shipment_schema.json --model llama3 --base-url http://localhost:11434/v1
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import importlib
|
|
@@ -21,18 +21,18 @@ from .schema_compiler import compile_schema_from_description
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def _load_plugins() -> None:
|
|
24
|
-
"""Import modules listed in
|
|
24
|
+
"""Import modules listed in FASTDOCPARSE_PLUGINS (comma-separated) so they can call
|
|
25
25
|
parser.register_default_ingestion_handler() on import — the only way a custom
|
|
26
26
|
ingestion kind (DOCX, XLSX, ...) becomes reachable from this CLI, since a fresh CLI
|
|
27
27
|
process otherwise only knows the built-in "pdf"/"image" handlers.
|
|
28
28
|
|
|
29
|
-
Security note: this imports and runs arbitrary Python from wherever
|
|
29
|
+
Security note: this imports and runs arbitrary Python from wherever FASTDOCPARSE_PLUGINS
|
|
30
30
|
points, at CLI startup, with no sandboxing — the same trust model as PYTHONSTARTUP or
|
|
31
31
|
DJANGO_SETTINGS_MODULE. That's fine for a user pointing it at their own plugin on their
|
|
32
32
|
own machine, which is the only supported use. Never let this env var be set from an
|
|
33
33
|
untrusted source (e.g. a request parameter in a hosted service built on this CLI).
|
|
34
34
|
"""
|
|
35
|
-
plugin_spec = os.environ.get("
|
|
35
|
+
plugin_spec = os.environ.get("FASTDOCPARSE_PLUGINS", "")
|
|
36
36
|
for module_name in filter(None, (p.strip() for p in plugin_spec.split(","))):
|
|
37
37
|
importlib.import_module(module_name)
|
|
38
38
|
|
|
@@ -56,7 +56,7 @@ def extract(
|
|
|
56
56
|
base_url: Optional[str] = typer.Option(None, "--base-url", help="OpenAI-compatible API base URL. Omit for OpenAI; use e.g. http://localhost:11434/v1 for Ollama."),
|
|
57
57
|
api_key: Optional[str] = typer.Option(None, "--api-key", envvar="LLM_API_KEY", help="API key. Not needed for local Ollama."),
|
|
58
58
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Write the JSON result to this file instead of printing it."),
|
|
59
|
-
kind: Optional[str] = typer.Option(None, "--kind", help="Override ingestion routing (e.g. 'docx' for a custom handler loaded via
|
|
59
|
+
kind: Optional[str] = typer.Option(None, "--kind", help="Override ingestion routing (e.g. 'docx' for a custom handler loaded via FASTDOCPARSE_PLUGINS). Defaults to auto-detecting pdf/image from the file extension."),
|
|
60
60
|
):
|
|
61
61
|
"""Extract the fields defined in SCHEMA from FILE and print the result as JSON."""
|
|
62
62
|
if kind is None and file.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
@@ -84,7 +84,7 @@ def extract(
|
|
|
84
84
|
typer.echo(f"Could not complete extraction: {e}", err=True)
|
|
85
85
|
raise typer.Exit(code=1)
|
|
86
86
|
except UnknownIngestionKindError as e:
|
|
87
|
-
typer.echo(f"{e} (check --kind is spelled correctly and its plugin is loaded via
|
|
87
|
+
typer.echo(f"{e} (check --kind is spelled correctly and its plugin is loaded via FASTDOCPARSE_PLUGINS)", err=True)
|
|
88
88
|
raise typer.Exit(code=1)
|
|
89
89
|
except ValueError as e:
|
|
90
90
|
typer.echo(f"Extraction failed: {e}", err=True)
|
|
@@ -133,7 +133,7 @@ def schema_from_text(
|
|
|
133
133
|
raise typer.Exit(code=1)
|
|
134
134
|
|
|
135
135
|
typer.echo(f"Saved schema '{doc_schema.name}' with {len(doc_schema.fields)} field(s) to {output}")
|
|
136
|
-
typer.echo("Review it, then run:
|
|
136
|
+
typer.echo("Review it, then run: fastdocparse extract <your_document> " + str(output))
|
|
137
137
|
|
|
138
138
|
|
|
139
139
|
if __name__ == "__main__":
|
|
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
|
|
|
15
15
|
# module without executing it, so this costs nothing at import time. The actual
|
|
16
16
|
# `import rapidocr_onnxruntime` (which pulls in onnxruntime, opencv, numpy — measured
|
|
17
17
|
# at ~1.2s) and RapidOCR() construction only happen lazily, on first real OCR call, via
|
|
18
|
-
# _get_rapid_ocr() below. Without this, every `import
|
|
18
|
+
# _get_rapid_ocr() below. Without this, every `import fastdocparse` would eagerly pay
|
|
19
19
|
# that cost even for code that never touches OCR (e.g. just building a Schema).
|
|
20
20
|
HAS_RAPID_OCR = importlib.util.find_spec("rapidocr_onnxruntime") is not None
|
|
21
21
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastdocparse
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Extract structured data from semi-structured documents using any OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Requires-Python: <3.13,>=3.9
|
|
@@ -17,7 +17,7 @@ Provides-Extra: dev
|
|
|
17
17
|
Requires-Dist: pytest; extra == "dev"
|
|
18
18
|
Dynamic: license-file
|
|
19
19
|
|
|
20
|
-
#
|
|
20
|
+
# fastdocparse
|
|
21
21
|
|
|
22
22
|
Extract structured data from semi-structured documents — invoices, bills, tax forms, resumes, bank statements, shipment manifests — using any OpenAI-compatible LLM (OpenAI, Ollama, vLLM, Groq, etc.), with **per-field grounding and confidence**, not just raw extraction.
|
|
23
23
|
|
|
@@ -39,7 +39,7 @@ No extra LLM call for any of this — it's deterministic, string/rule-based vali
|
|
|
39
39
|
|
|
40
40
|
| | Who it's for | How |
|
|
41
41
|
|---|---|---|
|
|
42
|
-
| **CLI** | No coding needed | `
|
|
42
|
+
| **CLI** | No coding needed | `fastdocparse extract <file> <schema.json>` |
|
|
43
43
|
| **Python API** | Building it into your own app | `DocumentParser(client).extract(document_bytes, schema)` |
|
|
44
44
|
|
|
45
45
|
Defining *what* to extract also has two paths — hand-write a JSON/YAML schema file, or describe it in plain English and let the LLM draft the schema for you.
|
|
@@ -47,15 +47,19 @@ Defining *what* to extract also has two paths — hand-write a JSON/YAML schema
|
|
|
47
47
|
## Install
|
|
48
48
|
|
|
49
49
|
```bash
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
pip install fastdocparse
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For local development instead:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
git clone https://github.com/pranjalparmar/fastdocparse
|
|
57
|
+
cd fastdocparse
|
|
52
58
|
python -m venv venv
|
|
53
59
|
source venv/bin/activate # Windows: venv\Scripts\activate
|
|
54
|
-
pip install -e .
|
|
60
|
+
pip install -e ".[dev]"
|
|
55
61
|
```
|
|
56
62
|
|
|
57
|
-
(Not yet published to PyPI — see [Status](#status). Until then, install from a local clone as above.)
|
|
58
|
-
|
|
59
63
|
You also need access to an LLM. Either:
|
|
60
64
|
- An OpenAI API key (`export OPENAI_API_KEY=...` or pass `--api-key`), or
|
|
61
65
|
- A local model via [Ollama](https://ollama.com/) — no API key, no cloud, documents never leave your machine.
|
|
@@ -64,11 +68,11 @@ You also need access to an LLM. Either:
|
|
|
64
68
|
|
|
65
69
|
```bash
|
|
66
70
|
# 1. Extract using one of the bundled example schemas
|
|
67
|
-
|
|
71
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
68
72
|
--model gpt-4o-mini --api-key sk-...
|
|
69
73
|
|
|
70
74
|
# Or with a local model via Ollama (no API key needed):
|
|
71
|
-
|
|
75
|
+
fastdocparse extract sample_invoice.png src/fastdocparse/schemas/invoice.json \
|
|
72
76
|
--model llama3.2 --base-url http://localhost:11434/v1 --api-key ollama
|
|
73
77
|
```
|
|
74
78
|
|
|
@@ -85,18 +89,18 @@ Output is JSON, printed to stdout (or saved with `--output result.json`):
|
|
|
85
89
|
Don't want to write JSON at all? Describe the fields in plain English instead:
|
|
86
90
|
|
|
87
91
|
```bash
|
|
88
|
-
|
|
92
|
+
fastdocparse schema-from-text \
|
|
89
93
|
"I want the invoice number, total price, and vendor name. Invoice number and total are required." \
|
|
90
94
|
--output my_invoice_schema.json
|
|
91
95
|
|
|
92
96
|
# review my_invoice_schema.json, then:
|
|
93
|
-
|
|
97
|
+
fastdocparse extract my_invoice.pdf my_invoice_schema.json
|
|
94
98
|
```
|
|
95
99
|
|
|
96
100
|
## Quickstart — Python API
|
|
97
101
|
|
|
98
102
|
```python
|
|
99
|
-
from
|
|
103
|
+
from fastdocparse import Schema, Field, LLMClient, DocumentParser
|
|
100
104
|
|
|
101
105
|
schema = Schema(
|
|
102
106
|
name="Invoice",
|
|
@@ -129,4 +133,4 @@ Want to contribute? Start with [docs/architecture.md](docs/architecture.md) for
|
|
|
129
133
|
|
|
130
134
|
## Status
|
|
131
135
|
|
|
132
|
-
Core extraction, grounding, chunking, both CLI/API paths, and real packaging (`pip install
|
|
136
|
+
Core extraction, grounding, chunking, both CLI/API paths, and real packaging are implemented and tested (74 tests, `pytest -v`). Published on PyPI as [`fastdocparse`](https://pypi.org/project/fastdocparse/) — `pip install fastdocparse` installs a working `fastdocparse` command and a proper `fastdocparse.*` import namespace, verified end to end with a clean-virtualenv install straight from the real public index. Not yet done: a hosted API — see [document-extractor-spec.md](document-extractor-spec.md) for the roadmap.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/fastdocparse/__init__.py
|
|
5
|
+
src/fastdocparse/cache.py
|
|
6
|
+
src/fastdocparse/cli.py
|
|
7
|
+
src/fastdocparse/config.py
|
|
8
|
+
src/fastdocparse/example_schemas.py
|
|
9
|
+
src/fastdocparse/grounding.py
|
|
10
|
+
src/fastdocparse/json_repair.py
|
|
11
|
+
src/fastdocparse/llm_client.py
|
|
12
|
+
src/fastdocparse/ocr_engine.py
|
|
13
|
+
src/fastdocparse/parser.py
|
|
14
|
+
src/fastdocparse/pdf_utils.py
|
|
15
|
+
src/fastdocparse/prompt_compiler.py
|
|
16
|
+
src/fastdocparse/py.typed
|
|
17
|
+
src/fastdocparse/result.py
|
|
18
|
+
src/fastdocparse/schema.py
|
|
19
|
+
src/fastdocparse/schema_compiler.py
|
|
20
|
+
src/fastdocparse.egg-info/PKG-INFO
|
|
21
|
+
src/fastdocparse.egg-info/SOURCES.txt
|
|
22
|
+
src/fastdocparse.egg-info/dependency_links.txt
|
|
23
|
+
src/fastdocparse.egg-info/entry_points.txt
|
|
24
|
+
src/fastdocparse.egg-info/requires.txt
|
|
25
|
+
src/fastdocparse.egg-info/top_level.txt
|
|
26
|
+
src/fastdocparse/schemas/invoice.json
|
|
27
|
+
src/fastdocparse/schemas/shipment_manifest.json
|
|
28
|
+
tests/test_architecture.py
|
|
29
|
+
tests/test_cli.py
|
|
30
|
+
tests/test_grounding.py
|
|
31
|
+
tests/test_parser.py
|
|
32
|
+
tests/test_schema_compiler.py
|
|
33
|
+
tests/test_severe_edge_cases.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fastdocparse
|
|
@@ -8,11 +8,11 @@ from unittest.mock import MagicMock, patch
|
|
|
8
8
|
|
|
9
9
|
import pytest
|
|
10
10
|
|
|
11
|
-
from
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
from
|
|
11
|
+
from fastdocparse.cache import InMemoryCache, make_cache_key
|
|
12
|
+
from fastdocparse.config import ExtractionConfig
|
|
13
|
+
from fastdocparse.example_schemas import INVOICE_SCHEMA
|
|
14
|
+
from fastdocparse.parser import INGESTION_HANDLERS, DocumentParser, register_default_ingestion_handler
|
|
15
|
+
from fastdocparse.schema import Field, Schema
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def test_cache_hit_avoids_second_llm_call():
|
|
@@ -22,7 +22,7 @@ def test_cache_hit_avoids_second_llm_call():
|
|
|
22
22
|
cache = InMemoryCache()
|
|
23
23
|
parser = DocumentParser(client=mock_client, cache=cache)
|
|
24
24
|
|
|
25
|
-
with patch("
|
|
25
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Total amount is 15.00 for this bill."):
|
|
26
26
|
first = parser.extract(b"dummy-doc", schema)
|
|
27
27
|
second = parser.extract(b"dummy-doc", schema)
|
|
28
28
|
|
|
@@ -41,7 +41,7 @@ def test_cache_is_skipped_when_custom_rules_passed():
|
|
|
41
41
|
def noop_rule(extracted):
|
|
42
42
|
return None
|
|
43
43
|
|
|
44
|
-
with patch("
|
|
44
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Total amount is 15.00 for this bill."):
|
|
45
45
|
parser.extract(b"dummy-doc", schema, rules=[noop_rule])
|
|
46
46
|
parser.extract(b"dummy-doc", schema, rules=[noop_rule])
|
|
47
47
|
|
|
@@ -78,7 +78,7 @@ def test_cache_key_differs_by_handler_identity():
|
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
def test_unknown_kind_raises_dedicated_error_not_bare_valueerror():
|
|
81
|
-
from
|
|
81
|
+
from fastdocparse.parser import UnknownIngestionKindError
|
|
82
82
|
|
|
83
83
|
schema = Schema(name="T", fields=[Field(name="x", description="x")])
|
|
84
84
|
mock_client = MagicMock()
|
|
@@ -104,9 +104,9 @@ def test_concurrent_chunks_produce_same_result_as_sequential():
|
|
|
104
104
|
mock_client_par.extract.side_effect = fake_extract
|
|
105
105
|
parser_par = DocumentParser(client=mock_client_par, config=ExtractionConfig(max_concurrent_chunks=4))
|
|
106
106
|
|
|
107
|
-
with patch("
|
|
108
|
-
patch("
|
|
109
|
-
patch("
|
|
107
|
+
with patch("fastdocparse.parser.chunk_document_text", return_value=chunks), \
|
|
108
|
+
patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy"), \
|
|
109
|
+
patch("fastdocparse.parser.pymupdf.open"):
|
|
110
110
|
result_seq = parser_seq.extract(b"dummy", schema)
|
|
111
111
|
result_par = parser_par.extract(b"dummy", schema)
|
|
112
112
|
|
|
@@ -122,9 +122,9 @@ def test_merge_prefers_chunk_grounded_value_over_hallucinated_first_chunk():
|
|
|
122
122
|
mock_client.extract.side_effect = ['{"total": 999.0}', '{"total": 100.0}']
|
|
123
123
|
parser = DocumentParser(client=mock_client)
|
|
124
124
|
|
|
125
|
-
with patch("
|
|
126
|
-
patch("
|
|
127
|
-
patch("
|
|
125
|
+
with patch("fastdocparse.parser.chunk_document_text", return_value=["unrelated filler text", "Grand Total: 100.0"]), \
|
|
126
|
+
patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy"), \
|
|
127
|
+
patch("fastdocparse.parser.pymupdf.open"):
|
|
128
128
|
res = parser.extract(b"dummy", schema)
|
|
129
129
|
|
|
130
130
|
assert res["total"]["value"] == 100.0
|
|
@@ -215,7 +215,7 @@ def test_extraction_config_is_frozen():
|
|
|
215
215
|
|
|
216
216
|
|
|
217
217
|
def test_extraction_result_from_raw_requires_meta():
|
|
218
|
-
from
|
|
218
|
+
from fastdocparse.result import ExtractionResult
|
|
219
219
|
|
|
220
220
|
with pytest.raises(ValueError):
|
|
221
221
|
ExtractionResult.from_raw({"total": {"value": 1, "confidence": "high", "flags": []}})
|
|
@@ -227,7 +227,7 @@ def test_aextract_matches_sync_extract():
|
|
|
227
227
|
mock_client.extract.return_value = '{"total": 100.0}'
|
|
228
228
|
parser = DocumentParser(client=mock_client)
|
|
229
229
|
|
|
230
|
-
with patch("
|
|
230
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Grand Total: 100.0. Padding to skip OCR fallback."):
|
|
231
231
|
sync_result = parser.extract(b"dummy", schema)
|
|
232
232
|
async_result = asyncio.run(parser.aextract(b"dummy", schema))
|
|
233
233
|
|
|
@@ -6,12 +6,12 @@ from unittest.mock import MagicMock, patch
|
|
|
6
6
|
|
|
7
7
|
from typer.testing import CliRunner
|
|
8
8
|
|
|
9
|
-
from
|
|
9
|
+
from fastdocparse.cli import app
|
|
10
10
|
|
|
11
11
|
runner = CliRunner()
|
|
12
12
|
REPO_ROOT = Path(__file__).parent.parent
|
|
13
13
|
SAMPLE_IMAGE = REPO_ROOT / "sample_invoice.png"
|
|
14
|
-
INVOICE_SCHEMA_PATH = REPO_ROOT / "src" / "
|
|
14
|
+
INVOICE_SCHEMA_PATH = REPO_ROOT / "src" / "fastdocparse" / "schemas" / "invoice.json"
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def _mock_openai_returning(content: str):
|
|
@@ -26,7 +26,7 @@ def test_extract_command_success():
|
|
|
26
26
|
"exporter_address": None, "importer_name": None, "importer_address": None,
|
|
27
27
|
"currency": None, "total_price": None, "line_items": [],
|
|
28
28
|
})
|
|
29
|
-
with patch("
|
|
29
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning(fake_result)):
|
|
30
30
|
result = runner.invoke(app, ["extract", str(SAMPLE_IMAGE), str(INVOICE_SCHEMA_PATH)])
|
|
31
31
|
|
|
32
32
|
assert result.exit_code == 0
|
|
@@ -62,7 +62,7 @@ def test_extract_command_writes_output_file(tmp_path):
|
|
|
62
62
|
"currency": None, "total_price": None, "line_items": [],
|
|
63
63
|
})
|
|
64
64
|
output_path = tmp_path / "result.json"
|
|
65
|
-
with patch("
|
|
65
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning(fake_result)):
|
|
66
66
|
result = runner.invoke(app, ["extract", str(SAMPLE_IMAGE), str(INVOICE_SCHEMA_PATH), "--output", str(output_path)])
|
|
67
67
|
|
|
68
68
|
assert result.exit_code == 0
|
|
@@ -76,7 +76,7 @@ def test_schema_from_text_success(tmp_path):
|
|
|
76
76
|
"fields": [{"name": "bill_of_lading", "description": "B/L number", "required": True}],
|
|
77
77
|
})
|
|
78
78
|
output_path = tmp_path / "generated.json"
|
|
79
|
-
with patch("
|
|
79
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning(fake_schema)):
|
|
80
80
|
result = runner.invoke(app, ["schema-from-text", "the bill of lading number", "--output", str(output_path)])
|
|
81
81
|
|
|
82
82
|
assert result.exit_code == 0
|
|
@@ -92,7 +92,7 @@ def test_extract_command_creates_missing_output_directory(tmp_path):
|
|
|
92
92
|
"currency": None, "total_price": None, "line_items": [],
|
|
93
93
|
})
|
|
94
94
|
output_path = tmp_path / "nested" / "dir" / "result.json"
|
|
95
|
-
with patch("
|
|
95
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning(fake_result)):
|
|
96
96
|
result = runner.invoke(app, ["extract", str(SAMPLE_IMAGE), str(INVOICE_SCHEMA_PATH), "--output", str(output_path)])
|
|
97
97
|
|
|
98
98
|
assert result.exit_code == 0
|
|
@@ -112,7 +112,7 @@ def test_schema_from_text_creates_missing_output_directory(tmp_path):
|
|
|
112
112
|
"fields": [{"name": "bill_of_lading", "description": "B/L number", "required": True}],
|
|
113
113
|
})
|
|
114
114
|
output_path = tmp_path / "nested" / "dir" / "generated.json"
|
|
115
|
-
with patch("
|
|
115
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning(fake_schema)):
|
|
116
116
|
result = runner.invoke(app, ["schema-from-text", "the bill of lading number", "--output", str(output_path)])
|
|
117
117
|
|
|
118
118
|
assert result.exit_code == 0
|
|
@@ -121,7 +121,7 @@ def test_schema_from_text_creates_missing_output_directory(tmp_path):
|
|
|
121
121
|
|
|
122
122
|
def test_schema_from_text_reports_generation_failure_cleanly(tmp_path):
|
|
123
123
|
output_path = tmp_path / "generated.json"
|
|
124
|
-
with patch("
|
|
124
|
+
with patch("fastdocparse.llm_client.OpenAI", return_value=_mock_openai_returning("not json at all")):
|
|
125
125
|
result = runner.invoke(app, ["schema-from-text", "vague request", "--output", str(output_path)])
|
|
126
126
|
|
|
127
127
|
assert result.exit_code == 1
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
|
-
from
|
|
6
|
-
from
|
|
5
|
+
from fastdocparse.config import ExtractionConfig
|
|
6
|
+
from fastdocparse.grounding import Issue, date_parseable_rule, numeric_sum_rule
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
def test_numeric_sum_rule_flags_mismatch():
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"""Tests for the generalized document extractor Phase 1 & 2."""
|
|
2
2
|
|
|
3
|
-
import json
|
|
4
3
|
import pytest
|
|
5
4
|
from unittest.mock import MagicMock, patch
|
|
6
5
|
|
|
7
|
-
from
|
|
8
|
-
from
|
|
9
|
-
from
|
|
10
|
-
from
|
|
11
|
-
from
|
|
6
|
+
from fastdocparse.schema import Schema, Field
|
|
7
|
+
from fastdocparse.example_schemas import INVOICE_SCHEMA
|
|
8
|
+
from fastdocparse.llm_client import LLMClient
|
|
9
|
+
from fastdocparse.parser import DocumentParser, _parse_json_from_llm
|
|
10
|
+
from fastdocparse.grounding import Issue
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
def test_tc1_1_invoice_schema_extraction():
|
|
@@ -27,7 +26,7 @@ def test_tc1_1_invoice_schema_extraction():
|
|
|
27
26
|
|
|
28
27
|
parser = DocumentParser(client=mock_client)
|
|
29
28
|
|
|
30
|
-
with patch("
|
|
29
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Invoice INV-123 Total 100.0. This is a very long text to avoid OCR fallback."):
|
|
31
30
|
res = parser.extract(b"dummy_pdf_bytes", schema)
|
|
32
31
|
|
|
33
32
|
assert res["invoice_number"]["value"] == "INV-123"
|
|
@@ -47,7 +46,7 @@ def test_tc1_2_arbitrary_schema():
|
|
|
47
46
|
mock_client.extract.return_value = '{"weather": "sunny", "mood": "happy"}'
|
|
48
47
|
parser = DocumentParser(client=mock_client)
|
|
49
48
|
|
|
50
|
-
with patch("
|
|
49
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="It is sunny and I am happy today and everything is great"):
|
|
51
50
|
res = parser.extract(b"dummy", schema)
|
|
52
51
|
|
|
53
52
|
assert "weather" in res
|
|
@@ -61,7 +60,7 @@ def test_tc1_3_scanned_receipt():
|
|
|
61
60
|
mock_client.extract.return_value = '{"total": "15.00"}'
|
|
62
61
|
parser = DocumentParser(client=mock_client)
|
|
63
62
|
|
|
64
|
-
with patch("
|
|
63
|
+
with patch("fastdocparse.parser.extract_text_from_image_ocr", return_value="Total: 15.00") as mock_ocr:
|
|
65
64
|
res = parser.extract(b"dummy_img", schema, is_image=True)
|
|
66
65
|
|
|
67
66
|
mock_ocr.assert_called_once()
|
|
@@ -106,7 +105,7 @@ def test_tc1_6_missing_field():
|
|
|
106
105
|
mock_client.extract.return_value = '{"found_field": "yes"}'
|
|
107
106
|
parser = DocumentParser(client=mock_client)
|
|
108
107
|
|
|
109
|
-
with patch("
|
|
108
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy yes"):
|
|
110
109
|
res = parser.extract(b"dummy", schema)
|
|
111
110
|
|
|
112
111
|
assert res["found_field"]["value"] == "yes"
|
|
@@ -119,7 +118,7 @@ def test_tc2_1_grounded_value():
|
|
|
119
118
|
mock_client.extract.return_value = '{"total": "15.00"}'
|
|
120
119
|
parser = DocumentParser(client=mock_client)
|
|
121
120
|
|
|
122
|
-
with patch("
|
|
121
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Total amount is 15.00 for this bill."):
|
|
123
122
|
res = parser.extract(b"dummy", schema)
|
|
124
123
|
|
|
125
124
|
assert res["total"]["value"] == "15.00"
|
|
@@ -133,7 +132,7 @@ def test_tc2_2_ungrounded_value():
|
|
|
133
132
|
mock_client.extract.return_value = '{"total": "99.99"}'
|
|
134
133
|
parser = DocumentParser(client=mock_client)
|
|
135
134
|
|
|
136
|
-
with patch("
|
|
135
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Total amount is 15.00 for this bill."):
|
|
137
136
|
res = parser.extract(b"dummy", schema)
|
|
138
137
|
|
|
139
138
|
assert res["total"]["value"] == "99.99"
|
|
@@ -160,7 +159,7 @@ def test_tc2_3_cross_check_failure():
|
|
|
160
159
|
return [Issue(field="total", message="Mismatch"), Issue(field="line_items", message="Mismatch")]
|
|
161
160
|
return None
|
|
162
161
|
|
|
163
|
-
with patch("
|
|
162
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="total 100.0 items 20.0 50.0 and here is some extra padding text"):
|
|
164
163
|
res = parser.extract(b"dummy", schema, rules=[sum_check])
|
|
165
164
|
|
|
166
165
|
assert "failed_check" in res["total"]["flags"]
|
|
@@ -179,7 +178,7 @@ def test_tc2_4_unparseable_date_flagged():
|
|
|
179
178
|
return [Issue(field="date", message="Invalid date")]
|
|
180
179
|
return None
|
|
181
180
|
|
|
182
|
-
with patch("
|
|
181
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="date Not a date. Here is some more text to make it longer."):
|
|
183
182
|
res = parser.extract(b"dummy", schema, rules=[date_check])
|
|
184
183
|
|
|
185
184
|
assert "failed_check" in res["date"]["flags"]
|
|
@@ -191,14 +190,14 @@ def test_tc2_5_no_extra_llm_calls():
|
|
|
191
190
|
mock_client.extract.return_value = '{"total": "15.00"}'
|
|
192
191
|
parser = DocumentParser(client=mock_client)
|
|
193
192
|
|
|
194
|
-
with patch("
|
|
195
|
-
|
|
196
|
-
|
|
193
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Total amount is 15.00 for this bill."):
|
|
194
|
+
parser.extract(b"dummy", schema)
|
|
195
|
+
|
|
197
196
|
mock_client.extract.assert_called_once()
|
|
198
197
|
|
|
199
198
|
def test_tc3_1_schema_examples_compile():
|
|
200
199
|
"""TC3.1 - Prompt compiler includes examples if present and omits if not."""
|
|
201
|
-
from
|
|
200
|
+
from fastdocparse.prompt_compiler import compile_prompt
|
|
202
201
|
|
|
203
202
|
# 0 examples
|
|
204
203
|
schema_0 = Schema(name="NoExamples", fields=[Field(name="test", description="test field")])
|
|
@@ -225,7 +224,7 @@ def test_tc3_2_invoice_schema():
|
|
|
225
224
|
mock_client.extract.return_value = '{"invoice_number": "INV-123", "total_price": 100.0, "exporter_name": "Acme"}'
|
|
226
225
|
parser = DocumentParser(client=mock_client)
|
|
227
226
|
|
|
228
|
-
with patch("
|
|
227
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="Acme Corp INV-123 100.0. Added text to avoid OCR fallback."):
|
|
229
228
|
res = parser.extract(b"dummy", INVOICE_SCHEMA)
|
|
230
229
|
|
|
231
230
|
assert res["invoice_number"]["value"] == "INV-123"
|
|
@@ -257,7 +256,7 @@ def test_tc4_1_structured_mode_enabled():
|
|
|
257
256
|
mock_client.extract.return_value = '{"items": []}'
|
|
258
257
|
parser = DocumentParser(client=mock_client)
|
|
259
258
|
|
|
260
|
-
with patch("
|
|
259
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy") as mock_extract:
|
|
261
260
|
parser.extract(b"dummy", schema)
|
|
262
261
|
|
|
263
262
|
mock_extract.assert_called_once_with(b"dummy", max_pages=15, structured_mode=True)
|
|
@@ -269,19 +268,19 @@ def test_tc4_2_structured_mode_disabled():
|
|
|
269
268
|
mock_client.extract.return_value = '{"text": "val"}'
|
|
270
269
|
parser = DocumentParser(client=mock_client)
|
|
271
270
|
|
|
272
|
-
with patch("
|
|
271
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy") as mock_extract:
|
|
273
272
|
parser.extract(b"dummy", schema)
|
|
274
273
|
|
|
275
274
|
mock_extract.assert_called_once_with(b"dummy", max_pages=15, structured_mode=False)
|
|
276
275
|
|
|
277
276
|
def test_tc4_3_ocr_structured_formatting():
|
|
278
277
|
"""TC4.3 - Verify OCR engine formats output with X coordinates in structured_mode."""
|
|
279
|
-
from
|
|
278
|
+
from fastdocparse.ocr_engine import extract_text_from_image_ocr, HAS_RAPID_OCR
|
|
280
279
|
if not HAS_RAPID_OCR:
|
|
281
280
|
pytest.skip("RapidOCR not installed")
|
|
282
281
|
|
|
283
282
|
# We mock the rapidocr return value to simulate an image containing two columns
|
|
284
|
-
with patch("
|
|
283
|
+
with patch("fastdocparse.ocr_engine._rapid_ocr") as mock_ocr, patch("fastdocparse.ocr_engine.Image.open"):
|
|
285
284
|
# result is a tuple (list_of_boxes, _)
|
|
286
285
|
# Each box format: ([ [x0,y0], [x1,y1], [x2,y2], [x3,y3] ], text, confidence)
|
|
287
286
|
mock_ocr.return_value = (
|
|
@@ -313,9 +312,9 @@ def test_tc5_1_list_chunking():
|
|
|
313
312
|
]
|
|
314
313
|
parser = DocumentParser(client=mock_client)
|
|
315
314
|
|
|
316
|
-
with patch("
|
|
317
|
-
patch("
|
|
318
|
-
patch("
|
|
315
|
+
with patch("fastdocparse.parser.chunk_document_text", return_value=["chunk1", "chunk2"]), \
|
|
316
|
+
patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy"), \
|
|
317
|
+
patch("fastdocparse.parser.pymupdf.open"):
|
|
319
318
|
res = parser.extract(b"dummy", schema)
|
|
320
319
|
|
|
321
320
|
assert res["items"]["value"] == ["item1", "item2"]
|
|
@@ -331,9 +330,9 @@ def test_tc5_2_single_value_chunking():
|
|
|
331
330
|
]
|
|
332
331
|
parser = DocumentParser(client=mock_client)
|
|
333
332
|
|
|
334
|
-
with patch("
|
|
335
|
-
patch("
|
|
336
|
-
patch("
|
|
333
|
+
with patch("fastdocparse.parser.chunk_document_text", return_value=["chunk1", "chunk2"]), \
|
|
334
|
+
patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy"), \
|
|
335
|
+
patch("fastdocparse.parser.pymupdf.open"):
|
|
337
336
|
res = parser.extract(b"dummy", schema)
|
|
338
337
|
|
|
339
338
|
assert res["total"]["value"] == 100
|
|
@@ -348,9 +347,9 @@ def test_tc5_3_truncation_flag():
|
|
|
348
347
|
mock_doc = MagicMock()
|
|
349
348
|
mock_doc.__len__.return_value = 20
|
|
350
349
|
|
|
351
|
-
with patch("
|
|
352
|
-
patch("
|
|
353
|
-
patch("
|
|
350
|
+
with patch("fastdocparse.parser.pymupdf.open", return_value=mock_doc), \
|
|
351
|
+
patch("fastdocparse.parser.extract_text_from_pdf", return_value="dummy dummy dummy dummy dummy dummy dummy dummy"), \
|
|
352
|
+
patch("fastdocparse.parser.chunk_document_text", return_value=["chunk1"]):
|
|
354
353
|
res = parser.extract(b"dummy", schema)
|
|
355
354
|
|
|
356
355
|
assert res["_meta"]["truncated"] is True
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
import pytest
|
|
4
4
|
from unittest.mock import MagicMock
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
from
|
|
6
|
+
from fastdocparse.llm_client import LLMClient
|
|
7
|
+
from fastdocparse.schema_compiler import compile_schema_from_description
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def test_compile_schema_from_description_success():
|
|
@@ -10,11 +10,11 @@ from unittest.mock import MagicMock, patch
|
|
|
10
10
|
|
|
11
11
|
import pytest
|
|
12
12
|
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
from
|
|
16
|
-
from
|
|
17
|
-
from
|
|
13
|
+
from fastdocparse.grounding import validate_field_constraints
|
|
14
|
+
from fastdocparse.json_repair import parse_json_from_llm
|
|
15
|
+
from fastdocparse.parser import DocumentParser, EmptyDocumentError
|
|
16
|
+
from fastdocparse.pdf_utils import chunk_document_text
|
|
17
|
+
from fastdocparse.schema import Schema, Field
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def test_catastrophic_backtracking_pattern_does_not_hang():
|
|
@@ -33,8 +33,8 @@ def test_catastrophic_backtracking_pattern_does_not_hang():
|
|
|
33
33
|
"""
|
|
34
34
|
code = (
|
|
35
35
|
"import time\n"
|
|
36
|
-
"from
|
|
37
|
-
"from
|
|
36
|
+
"from fastdocparse.grounding import validate_field_constraints\n"
|
|
37
|
+
"from fastdocparse.schema import Schema, Field\n"
|
|
38
38
|
"schema = Schema(name='T', fields=[Field(name='x', description='x', pattern=r'^(a+)+$')])\n"
|
|
39
39
|
"start = time.time()\n"
|
|
40
40
|
"validate_field_constraints(schema, {'x': 'a' * 30 + '!'})\n"
|
|
@@ -67,7 +67,7 @@ def test_empty_string_value_not_trivially_grounded():
|
|
|
67
67
|
mock_client.extract.return_value = '{"x": ""}'
|
|
68
68
|
schema = Schema(name="T", fields=[Field(name="x", description="x", required=True)])
|
|
69
69
|
parser = DocumentParser(client=mock_client)
|
|
70
|
-
with patch("
|
|
70
|
+
with patch("fastdocparse.parser.extract_text_from_pdf", return_value="padding padding padding padding padding"):
|
|
71
71
|
res = parser.extract(b"dummy", schema)
|
|
72
72
|
assert "grounded" not in res["x"]["flags"]
|
|
73
73
|
assert res["x"]["value"] is None
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.md
|
|
3
|
-
pyproject.toml
|
|
4
|
-
src/docextract/__init__.py
|
|
5
|
-
src/docextract/cache.py
|
|
6
|
-
src/docextract/cli.py
|
|
7
|
-
src/docextract/config.py
|
|
8
|
-
src/docextract/example_schemas.py
|
|
9
|
-
src/docextract/grounding.py
|
|
10
|
-
src/docextract/json_repair.py
|
|
11
|
-
src/docextract/llm_client.py
|
|
12
|
-
src/docextract/ocr_engine.py
|
|
13
|
-
src/docextract/parser.py
|
|
14
|
-
src/docextract/pdf_utils.py
|
|
15
|
-
src/docextract/prompt_compiler.py
|
|
16
|
-
src/docextract/py.typed
|
|
17
|
-
src/docextract/result.py
|
|
18
|
-
src/docextract/schema.py
|
|
19
|
-
src/docextract/schema_compiler.py
|
|
20
|
-
src/docextract/schemas/invoice.json
|
|
21
|
-
src/docextract/schemas/shipment_manifest.json
|
|
22
|
-
src/fastdocparse.egg-info/PKG-INFO
|
|
23
|
-
src/fastdocparse.egg-info/SOURCES.txt
|
|
24
|
-
src/fastdocparse.egg-info/dependency_links.txt
|
|
25
|
-
src/fastdocparse.egg-info/entry_points.txt
|
|
26
|
-
src/fastdocparse.egg-info/requires.txt
|
|
27
|
-
src/fastdocparse.egg-info/top_level.txt
|
|
28
|
-
tests/test_architecture.py
|
|
29
|
-
tests/test_cli.py
|
|
30
|
-
tests/test_grounding.py
|
|
31
|
-
tests/test_parser.py
|
|
32
|
-
tests/test_schema_compiler.py
|
|
33
|
-
tests/test_severe_edge_cases.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
docextract
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/example_schemas.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/prompt_compiler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schema_compiler.py
RENAMED
|
File without changes
|
{fastdocparse-0.1.0/src/docextract → fastdocparse-0.2.0/src/fastdocparse}/schemas/invoice.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|