fastdocparse 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docextract/__init__.py +48 -0
- docextract/cache.py +72 -0
- docextract/cli.py +140 -0
- docextract/config.py +38 -0
- docextract/example_schemas.py +13 -0
- docextract/grounding.py +306 -0
- docextract/json_repair.py +79 -0
- docextract/llm_client.py +82 -0
- docextract/ocr_engine.py +105 -0
- docextract/parser.py +313 -0
- docextract/pdf_utils.py +262 -0
- docextract/prompt_compiler.py +86 -0
- docextract/py.typed +0 -0
- docextract/result.py +42 -0
- docextract/schema.py +94 -0
- docextract/schema_compiler.py +68 -0
- docextract/schemas/invoice.json +73 -0
- docextract/schemas/shipment_manifest.json +40 -0
- fastdocparse-0.1.0.dist-info/METADATA +132 -0
- fastdocparse-0.1.0.dist-info/RECORD +24 -0
- fastdocparse-0.1.0.dist-info/WHEEL +5 -0
- fastdocparse-0.1.0.dist-info/entry_points.txt +2 -0
- fastdocparse-0.1.0.dist-info/licenses/LICENSE +21 -0
- fastdocparse-0.1.0.dist-info/top_level.txt +1 -0
docextract/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""docextract — extract structured data from semi-structured documents using any
|
|
2
|
+
OpenAI-compatible LLM, with per-field grounding and confidence.
|
|
3
|
+
|
|
4
|
+
from docextract import Schema, Field, LLMClient, DocumentParser
|
|
5
|
+
|
|
6
|
+
schema = Schema(name="Invoice", fields=[Field(name="total", description="Grand total", type="number")])
|
|
7
|
+
client = LLMClient(model="gpt-4o-mini", api_key="sk-...")
|
|
8
|
+
result = DocumentParser(client).extract(document_bytes, schema)
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from .cache import Cache, InMemoryCache
|
|
12
|
+
from .config import ExtractionConfig
|
|
13
|
+
from .grounding import Issue, check_substring, cross_check, date_parseable_rule, numeric_sum_rule, validate_field_constraints
|
|
14
|
+
from .llm_client import LLMClient, LLMClientError
|
|
15
|
+
from .parser import DocumentParser, EmptyDocumentError, UnknownIngestionKindError, register_default_ingestion_handler
|
|
16
|
+
from .result import ExtractionMeta, ExtractionResult, FieldResult
|
|
17
|
+
from .schema import Field, Schema
|
|
18
|
+
from .schema_compiler import compile_schema_from_description
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from importlib.metadata import version as _pkg_version
|
|
22
|
+
__version__ = _pkg_version("docextract")
|
|
23
|
+
except Exception:
|
|
24
|
+
__version__ = "0.0.0+unknown"
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Schema",
|
|
28
|
+
"Field",
|
|
29
|
+
"LLMClient",
|
|
30
|
+
"LLMClientError",
|
|
31
|
+
"DocumentParser",
|
|
32
|
+
"EmptyDocumentError",
|
|
33
|
+
"UnknownIngestionKindError",
|
|
34
|
+
"register_default_ingestion_handler",
|
|
35
|
+
"ExtractionConfig",
|
|
36
|
+
"Cache",
|
|
37
|
+
"InMemoryCache",
|
|
38
|
+
"ExtractionResult",
|
|
39
|
+
"FieldResult",
|
|
40
|
+
"ExtractionMeta",
|
|
41
|
+
"Issue",
|
|
42
|
+
"check_substring",
|
|
43
|
+
"cross_check",
|
|
44
|
+
"validate_field_constraints",
|
|
45
|
+
"numeric_sum_rule",
|
|
46
|
+
"date_parseable_rule",
|
|
47
|
+
"compile_schema_from_description",
|
|
48
|
+
]
|
docextract/cache.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Pluggable caching for DocumentParser.extract(). Off by default — pass a cache instance to opt in.
|
|
2
|
+
|
|
3
|
+
Caching is skipped whenever custom `rules` are passed to extract(), since a rule is an
|
|
4
|
+
arbitrary callable that can't be safely fingerprinted — caching would risk returning a
|
|
5
|
+
result validated under a different rule than the one just requested.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
from collections import OrderedDict
|
|
11
|
+
from typing import Any, Callable, Dict, Optional, Protocol
|
|
12
|
+
|
|
13
|
+
from .config import ExtractionConfig
|
|
14
|
+
from .schema import Schema
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Cache(Protocol):
|
|
18
|
+
def get(self, key: str) -> Optional[Dict[str, Any]]: ...
|
|
19
|
+
def set(self, key: str, value: Dict[str, Any]) -> None: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class InMemoryCache:
|
|
23
|
+
"""Process-local cache. Good for a long-running batch job or server process; lost on restart.
|
|
24
|
+
|
|
25
|
+
Pass max_size to cap memory use — oldest-accessed entries are evicted once the cap is
|
|
26
|
+
hit (LRU). Leave it None (default) only for short-lived scripts/batch jobs where the
|
|
27
|
+
process exits before the cache could grow unbounded.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, max_size: Optional[int] = None):
|
|
31
|
+
if max_size is not None and max_size <= 0:
|
|
32
|
+
raise ValueError(f"max_size must be positive, got {max_size}")
|
|
33
|
+
self._max_size = max_size
|
|
34
|
+
self._store: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
|
|
35
|
+
|
|
36
|
+
def get(self, key: str) -> Optional[Dict[str, Any]]:
|
|
37
|
+
if key not in self._store:
|
|
38
|
+
return None
|
|
39
|
+
self._store.move_to_end(key)
|
|
40
|
+
return self._store[key]
|
|
41
|
+
|
|
42
|
+
def set(self, key: str, value: Dict[str, Any]) -> None:
|
|
43
|
+
self._store[key] = value
|
|
44
|
+
self._store.move_to_end(key)
|
|
45
|
+
if self._max_size is not None:
|
|
46
|
+
while len(self._store) > self._max_size:
|
|
47
|
+
self._store.popitem(last=False)
|
|
48
|
+
|
|
49
|
+
def __len__(self) -> int:
|
|
50
|
+
return len(self._store)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def make_cache_key(
|
|
54
|
+
document_bytes: bytes,
|
|
55
|
+
schema: Schema,
|
|
56
|
+
kind: str,
|
|
57
|
+
config: ExtractionConfig,
|
|
58
|
+
handler: Callable[..., str],
|
|
59
|
+
) -> str:
|
|
60
|
+
"""Fingerprint a (document, schema, ingestion kind, config, handler) combination.
|
|
61
|
+
|
|
62
|
+
handler is included, not just kind, because handlers are registered per
|
|
63
|
+
DocumentParser instance — two instances can register different handlers under the
|
|
64
|
+
same kind name, and a cache shared between them must not conflate the two.
|
|
65
|
+
Identity is process-local (qualified name + id()), which matches InMemoryCache's own
|
|
66
|
+
process-local lifetime — it isn't meant to survive a restart anyway.
|
|
67
|
+
"""
|
|
68
|
+
document_digest = hashlib.sha256(document_bytes).hexdigest()
|
|
69
|
+
schema_digest = hashlib.sha256(schema.model_dump_json().encode()).hexdigest()
|
|
70
|
+
config_digest = hashlib.sha256(json.dumps(config.__dict__, sort_keys=True).encode()).hexdigest()
|
|
71
|
+
handler_identity = f"{getattr(handler, '__module__', '?')}.{getattr(handler, '__qualname__', repr(handler))}:{id(handler)}"
|
|
72
|
+
return hashlib.sha256(f"{document_digest}:{schema_digest}:{kind}:{config_digest}:{handler_identity}".encode()).hexdigest()
|
docextract/cli.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Command-line entrypoint: extract fields from a document without writing code.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
docextract extract document.pdf invoice_schema.json
|
|
5
|
+
docextract extract receipt.jpg shipment_schema.json --model llama3 --base-url http://localhost:11434/v1
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import importlib
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
from pydantic import ValidationError
|
|
16
|
+
|
|
17
|
+
from .llm_client import LLMClient, LLMClientError
|
|
18
|
+
from .parser import DocumentParser, EmptyDocumentError, UnknownIngestionKindError
|
|
19
|
+
from .schema import Schema
|
|
20
|
+
from .schema_compiler import compile_schema_from_description
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _load_plugins() -> None:
|
|
24
|
+
"""Import modules listed in DOCEXTRACT_PLUGINS (comma-separated) so they can call
|
|
25
|
+
parser.register_default_ingestion_handler() on import — the only way a custom
|
|
26
|
+
ingestion kind (DOCX, XLSX, ...) becomes reachable from this CLI, since a fresh CLI
|
|
27
|
+
process otherwise only knows the built-in "pdf"/"image" handlers.
|
|
28
|
+
|
|
29
|
+
Security note: this imports and runs arbitrary Python from wherever DOCEXTRACT_PLUGINS
|
|
30
|
+
points, at CLI startup, with no sandboxing — the same trust model as PYTHONSTARTUP or
|
|
31
|
+
DJANGO_SETTINGS_MODULE. That's fine for a user pointing it at their own plugin on their
|
|
32
|
+
own machine, which is the only supported use. Never let this env var be set from an
|
|
33
|
+
untrusted source (e.g. a request parameter in a hosted service built on this CLI).
|
|
34
|
+
"""
|
|
35
|
+
plugin_spec = os.environ.get("DOCEXTRACT_PLUGINS", "")
|
|
36
|
+
for module_name in filter(None, (p.strip() for p in plugin_spec.split(","))):
|
|
37
|
+
importlib.import_module(module_name)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_load_plugins()
|
|
41
|
+
|
|
42
|
+
app = typer.Typer(
|
|
43
|
+
add_completion=False,
|
|
44
|
+
help="Extract structured data from a PDF/PNG/JPG using a schema file you define — no coding required.",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
|
|
48
|
+
SUPPORTED_EXTENSIONS = IMAGE_EXTENSIONS | {".pdf"}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command()
|
|
52
|
+
def extract(
|
|
53
|
+
file: Path = typer.Argument(..., exists=True, readable=True, help="Path to the PDF/PNG/JPG document."),
|
|
54
|
+
schema: Path = typer.Argument(..., exists=True, readable=True, help="Path to a .json or .yaml schema file listing the fields to extract."),
|
|
55
|
+
model: str = typer.Option("gpt-4o-mini", "--model", "-m", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
56
|
+
base_url: Optional[str] = typer.Option(None, "--base-url", help="OpenAI-compatible API base URL. Omit for OpenAI; use e.g. http://localhost:11434/v1 for Ollama."),
|
|
57
|
+
api_key: Optional[str] = typer.Option(None, "--api-key", envvar="LLM_API_KEY", help="API key. Not needed for local Ollama."),
|
|
58
|
+
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Write the JSON result to this file instead of printing it."),
|
|
59
|
+
kind: Optional[str] = typer.Option(None, "--kind", help="Override ingestion routing (e.g. 'docx' for a custom handler loaded via DOCEXTRACT_PLUGINS). Defaults to auto-detecting pdf/image from the file extension."),
|
|
60
|
+
):
|
|
61
|
+
"""Extract the fields defined in SCHEMA from FILE and print the result as JSON."""
|
|
62
|
+
if kind is None and file.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
63
|
+
typer.echo(f"Unsupported file type {file.suffix!r}. Supported: .pdf, .png, .jpg, .jpeg (or pass --kind).", err=True)
|
|
64
|
+
raise typer.Exit(code=1)
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
doc_schema = Schema.from_file(schema)
|
|
68
|
+
except (ValidationError, ValueError, OSError) as e:
|
|
69
|
+
typer.echo(f"Could not load schema from {schema}: {e}", err=True)
|
|
70
|
+
raise typer.Exit(code=1)
|
|
71
|
+
|
|
72
|
+
client = LLMClient(base_url=base_url, api_key=api_key, model=model)
|
|
73
|
+
document_parser = DocumentParser(client=client)
|
|
74
|
+
|
|
75
|
+
is_image = file.suffix.lower() in IMAGE_EXTENSIONS
|
|
76
|
+
document_bytes = file.read_bytes()
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
result = document_parser.extract(document_bytes, doc_schema, is_image=is_image, kind=kind)
|
|
80
|
+
except EmptyDocumentError as e:
|
|
81
|
+
typer.echo(f"Extraction failed: {e}", err=True)
|
|
82
|
+
raise typer.Exit(code=1)
|
|
83
|
+
except LLMClientError as e:
|
|
84
|
+
typer.echo(f"Could not complete extraction: {e}", err=True)
|
|
85
|
+
raise typer.Exit(code=1)
|
|
86
|
+
except UnknownIngestionKindError as e:
|
|
87
|
+
typer.echo(f"{e} (check --kind is spelled correctly and its plugin is loaded via DOCEXTRACT_PLUGINS)", err=True)
|
|
88
|
+
raise typer.Exit(code=1)
|
|
89
|
+
except ValueError as e:
|
|
90
|
+
typer.echo(f"Extraction failed: {e}", err=True)
|
|
91
|
+
raise typer.Exit(code=1)
|
|
92
|
+
|
|
93
|
+
result_json = json.dumps(result, indent=2, default=str)
|
|
94
|
+
|
|
95
|
+
if output:
|
|
96
|
+
try:
|
|
97
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
output.write_text(result_json)
|
|
99
|
+
except OSError as e:
|
|
100
|
+
typer.echo(f"Could not write output to {output}: {e}", err=True)
|
|
101
|
+
raise typer.Exit(code=1)
|
|
102
|
+
typer.echo(f"Wrote result to {output}")
|
|
103
|
+
else:
|
|
104
|
+
typer.echo(result_json)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@app.command(name="schema-from-text")
|
|
108
|
+
def schema_from_text(
|
|
109
|
+
description: str = typer.Argument(..., help="Plain-English description of the fields you want extracted, e.g. \"invoice number, total price, and a list of line items with product name and quantity\"."),
|
|
110
|
+
output: Path = typer.Option(..., "--output", "-o", help="Where to save the generated schema (.json)."),
|
|
111
|
+
model: str = typer.Option("gpt-4o-mini", "--model", "-m", help="Model name, e.g. gpt-4o-mini, llama3."),
|
|
112
|
+
base_url: Optional[str] = typer.Option(None, "--base-url", help="OpenAI-compatible API base URL. Omit for OpenAI; use e.g. http://localhost:11434/v1 for Ollama."),
|
|
113
|
+
api_key: Optional[str] = typer.Option(None, "--api-key", envvar="LLM_API_KEY", help="API key. Not needed for local Ollama."),
|
|
114
|
+
):
|
|
115
|
+
"""Turn a plain-English description of the fields you want into a schema file.
|
|
116
|
+
|
|
117
|
+
Review the generated file before using it with `extract` — the LLM proposes field
|
|
118
|
+
names and types from your description, and a wrong guess here affects every
|
|
119
|
+
document you later run against this schema.
|
|
120
|
+
"""
|
|
121
|
+
client = LLMClient(base_url=base_url, api_key=api_key, model=model)
|
|
122
|
+
try:
|
|
123
|
+
doc_schema = compile_schema_from_description(description, client)
|
|
124
|
+
except (ValueError, LLMClientError) as e:
|
|
125
|
+
typer.echo(f"Could not generate a schema: {e}", err=True)
|
|
126
|
+
raise typer.Exit(code=1)
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
130
|
+
output.write_text(json.dumps(doc_schema.model_dump(), indent=2))
|
|
131
|
+
except OSError as e:
|
|
132
|
+
typer.echo(f"Could not write schema to {output}: {e}", err=True)
|
|
133
|
+
raise typer.Exit(code=1)
|
|
134
|
+
|
|
135
|
+
typer.echo(f"Saved schema '{doc_schema.name}' with {len(doc_schema.fields)} field(s) to {output}")
|
|
136
|
+
typer.echo("Review it, then run: docextract extract <your_document> " + str(output))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
if __name__ == "__main__":
|
|
140
|
+
app()
|
docextract/config.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Tunable extraction parameters, previously hardcoded inline in parser.py."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class ExtractionConfig:
|
|
8
|
+
"""Knobs for a DocumentParser run. Override per-instance instead of editing source."""
|
|
9
|
+
|
|
10
|
+
# The single page cap: ingestion reads (and chunks/merges across) up to this
|
|
11
|
+
# many pages. A document with more pages than this is flagged truncated —
|
|
12
|
+
# there used to be two separate, disconnected limits here (a smaller one
|
|
13
|
+
# actually used for ingestion, a larger one only used for the truncation
|
|
14
|
+
# flag), which meant a document could silently lose pages while still being
|
|
15
|
+
# reported as "not truncated." One limit now, so the flag is honest.
|
|
16
|
+
max_pages: int = 15
|
|
17
|
+
chunk_max_tokens: int = 3000
|
|
18
|
+
pdf_render_dpi: int = 150
|
|
19
|
+
max_image_dim: int = 1536
|
|
20
|
+
ocr_min_confidence: float = 0.3
|
|
21
|
+
# Chunks are processed sequentially by default (1) — safe, deterministic order.
|
|
22
|
+
# Raise this to fan LLM calls for independent chunks out over threads; chunks are
|
|
23
|
+
# I/O-bound network calls, so a plain ThreadPoolExecutor is enough (no asyncio needed).
|
|
24
|
+
max_concurrent_chunks: int = 1
|
|
25
|
+
|
|
26
|
+
def __post_init__(self):
|
|
27
|
+
if self.max_pages <= 0:
|
|
28
|
+
raise ValueError(f"max_pages must be positive, got {self.max_pages}")
|
|
29
|
+
if self.chunk_max_tokens <= 0:
|
|
30
|
+
raise ValueError(f"chunk_max_tokens must be positive, got {self.chunk_max_tokens}")
|
|
31
|
+
if self.pdf_render_dpi <= 0:
|
|
32
|
+
raise ValueError(f"pdf_render_dpi must be positive, got {self.pdf_render_dpi}")
|
|
33
|
+
if self.max_image_dim <= 0:
|
|
34
|
+
raise ValueError(f"max_image_dim must be positive, got {self.max_image_dim}")
|
|
35
|
+
if not 0.0 <= self.ocr_min_confidence <= 1.0:
|
|
36
|
+
raise ValueError(f"ocr_min_confidence must be between 0 and 1, got {self.ocr_min_confidence}")
|
|
37
|
+
if self.max_concurrent_chunks <= 0:
|
|
38
|
+
raise ValueError(f"max_concurrent_chunks must be positive, got {self.max_concurrent_chunks}")
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Pre-built schemas with few-shot examples for common document types.
|
|
2
|
+
|
|
3
|
+
Loaded from schemas/*.json rather than declared twice — schemas/invoice.json is the
|
|
4
|
+
single source of truth; editing it updates both the CLI template and this import.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .schema import Schema
|
|
10
|
+
|
|
11
|
+
_SCHEMAS_DIR = Path(__file__).parent / "schemas"
|
|
12
|
+
|
|
13
|
+
INVOICE_SCHEMA = Schema.from_file(_SCHEMAS_DIR / "invoice.json")
|
docextract/grounding.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Grounding and validation checks for extracted data."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import signal
|
|
6
|
+
import threading
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from typing import Any, Dict, List, Callable, Optional
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from .schema import Schema
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _is_present(value: Any) -> bool:
|
|
16
|
+
"""False for None and for empty/whitespace-only strings — both mean "nothing was
|
|
17
|
+
really extracted here," which a bare `value is None` check misses. A required field
|
|
18
|
+
that comes back "" must not silently pass as present (and must not be trivially
|
|
19
|
+
"grounded" either, since an empty string is a substring of anything)."""
|
|
20
|
+
if value is None:
|
|
21
|
+
return False
|
|
22
|
+
if isinstance(value, str) and not value.strip():
|
|
23
|
+
return False
|
|
24
|
+
return True
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _PatternTimeout(Exception):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _regex_matches_with_timeout(pattern: str, value: str, timeout: float = 1.0) -> Optional[bool]:
|
|
32
|
+
"""Guard re.fullmatch against a catastrophically backtracking pattern (e.g.
|
|
33
|
+
"^(a+)+$" against a long near-miss string), since `pattern` is arbitrary user- or
|
|
34
|
+
LLM-supplied regex that can otherwise hang indefinitely.
|
|
35
|
+
|
|
36
|
+
Two approaches were tried and rejected before this one:
|
|
37
|
+
- A background *thread* with join(timeout): doesn't work. CPython's `re` engine
|
|
38
|
+
holds the GIL for the whole backtracking search, so a hung regex thread starves
|
|
39
|
+
the main thread too and the timeout never fires (confirmed by testing).
|
|
40
|
+
- A separate *process* with a hard kill: works in isolation, but this application
|
|
41
|
+
loads a native ML runtime at import time (RapidOCR/onnxruntime, for local OCR).
|
|
42
|
+
Spawning a subprocess alongside that runtime crashed the whole process with a
|
|
43
|
+
native SIGABRT on exit ("recursive_mutex lock failed") — confirmed by running it
|
|
44
|
+
through the real CLI, not just a unit test. Not safe here.
|
|
45
|
+
|
|
46
|
+
SIGALRM actually interrupts a mid-backtrack regex call in CPython (confirmed by
|
|
47
|
+
testing) and needs no subprocess — but signal handlers only work on the main thread.
|
|
48
|
+
On a non-main thread (e.g. inside aextract()'s background-thread wrapper) or on a
|
|
49
|
+
platform without SIGALRM (Windows), this runs the match with no timeout at all —
|
|
50
|
+
a narrower, documented gap rather than a mechanism that looked safe but wasn't.
|
|
51
|
+
"""
|
|
52
|
+
can_use_signal = hasattr(signal, "SIGALRM") and threading.current_thread() is threading.main_thread()
|
|
53
|
+
|
|
54
|
+
if not can_use_signal:
|
|
55
|
+
try:
|
|
56
|
+
return re.fullmatch(pattern, value) is not None
|
|
57
|
+
except re.error:
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
def _on_alarm(signum, frame):
|
|
61
|
+
raise _PatternTimeout()
|
|
62
|
+
|
|
63
|
+
previous_handler = signal.signal(signal.SIGALRM, _on_alarm)
|
|
64
|
+
signal.setitimer(signal.ITIMER_REAL, timeout)
|
|
65
|
+
try:
|
|
66
|
+
return re.fullmatch(pattern, value) is not None
|
|
67
|
+
except _PatternTimeout:
|
|
68
|
+
logger.warning(
|
|
69
|
+
"Pattern check timed out after %.1fs (pattern=%r) — possible catastrophic "
|
|
70
|
+
"backtracking; skipping this field's format check.", timeout, pattern
|
|
71
|
+
)
|
|
72
|
+
return None
|
|
73
|
+
except re.error:
|
|
74
|
+
return None
|
|
75
|
+
finally:
|
|
76
|
+
signal.setitimer(signal.ITIMER_REAL, 0)
|
|
77
|
+
signal.signal(signal.SIGALRM, previous_handler)
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class Issue:
|
|
81
|
+
field: str
|
|
82
|
+
message: str
|
|
83
|
+
severity: str = "warning"
|
|
84
|
+
kind: str = "cross_check" # "cross_check" | "missing_required" | "invalid_format"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
_NUMBER_RE = re.compile(r'-?\d[\d,]*\.?\d*')
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _as_number(value: Any) -> Optional[float]:
|
|
91
|
+
"""Parse an int/float or a numeric-looking string (currency symbols, thousands separators) into a float."""
|
|
92
|
+
if isinstance(value, bool):
|
|
93
|
+
return None
|
|
94
|
+
if isinstance(value, (int, float)):
|
|
95
|
+
return float(value)
|
|
96
|
+
if isinstance(value, str):
|
|
97
|
+
cleaned = re.sub(r'^[^\d\-]*', '', value.strip()).replace(',', '')
|
|
98
|
+
try:
|
|
99
|
+
return float(cleaned)
|
|
100
|
+
except ValueError:
|
|
101
|
+
return None
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _extract_numbers(text: str) -> List[float]:
|
|
106
|
+
numbers = []
|
|
107
|
+
for match in _NUMBER_RE.findall(text):
|
|
108
|
+
try:
|
|
109
|
+
numbers.append(float(match.replace(',', '')))
|
|
110
|
+
except ValueError:
|
|
111
|
+
continue
|
|
112
|
+
return numbers
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
_DATE_CANDIDATE_RE = re.compile(
|
|
116
|
+
r'\d{4}[/-]\d{1,2}[/-]\d{1,2}' # 2021-04-22, 2021/04/22
|
|
117
|
+
r'|\d{1,2}[/-]\d{1,2}[/-]\d{2,4}' # 22/04/2021, 04-22-2021
|
|
118
|
+
r'|\d{1,2}\s?[A-Za-z]{3,9}\s?\d{2,4}' # 22Apr2021, 22 April 2021
|
|
119
|
+
r'|[A-Za-z]{3,9}\s\d{1,2},?\s\d{2,4}' # April 22, 2021
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
_DATE_FORMATS = [
|
|
123
|
+
"%Y-%m-%d", "%Y/%m/%d",
|
|
124
|
+
"%m/%d/%Y", "%d/%m/%Y", "%m-%d-%Y", "%d-%m-%Y",
|
|
125
|
+
"%d%b%Y", "%d %b %Y", "%d%B%Y", "%d %B %Y",
|
|
126
|
+
"%b %d, %Y", "%B %d, %Y", "%b %d %Y", "%B %d %Y",
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _parse_date_any(text: str):
|
|
131
|
+
"""Try each known date format in turn; return a date object or None."""
|
|
132
|
+
text = text.strip()
|
|
133
|
+
for fmt in _DATE_FORMATS:
|
|
134
|
+
try:
|
|
135
|
+
return datetime.strptime(text, fmt).date()
|
|
136
|
+
except ValueError:
|
|
137
|
+
continue
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def check_substring(value: Any, source_text: str, numeric: bool = False, date: bool = False) -> bool:
|
|
142
|
+
"""
|
|
143
|
+
Check if the value is grounded (found) in the source text using fuzzy substring matching.
|
|
144
|
+
Returns True if grounded, False otherwise.
|
|
145
|
+
|
|
146
|
+
numeric: opt in for fields declared type "number"/"currency" — compares by parsed
|
|
147
|
+
numeric value (tolerant of formatting like "1,234.50" vs 1234.5). Leave False for
|
|
148
|
+
text/ID fields (invoice numbers, container numbers, etc.) where an all-digit value
|
|
149
|
+
is still a string identity, not a number, and "0100" must not be treated as
|
|
150
|
+
equal to "100".
|
|
151
|
+
|
|
152
|
+
date: opt in for fields declared type "date" — compares by parsed calendar date, so
|
|
153
|
+
a source document's "22Apr2021" still grounds a normalized "2021-04-22" even though
|
|
154
|
+
the strings don't match at all. Without this, every reformatted-but-correct date gets
|
|
155
|
+
flagged ungrounded, which is a false alarm, not a real hallucination.
|
|
156
|
+
"""
|
|
157
|
+
if value is None:
|
|
158
|
+
return True
|
|
159
|
+
|
|
160
|
+
if isinstance(value, list):
|
|
161
|
+
if not value:
|
|
162
|
+
return True
|
|
163
|
+
return all(check_substring(item, source_text) for item in value)
|
|
164
|
+
|
|
165
|
+
if isinstance(value, dict):
|
|
166
|
+
if not value:
|
|
167
|
+
return True
|
|
168
|
+
return all(check_substring(v, source_text) for v in value.values())
|
|
169
|
+
|
|
170
|
+
val_str = str(value).lower()
|
|
171
|
+
source_lower = source_text.lower()
|
|
172
|
+
|
|
173
|
+
# Exact lowercase match
|
|
174
|
+
if val_str in source_lower:
|
|
175
|
+
return True
|
|
176
|
+
|
|
177
|
+
# Numeric comparison: catches formatting differences (1,234.50 vs 1234.5)
|
|
178
|
+
# that the naive digit-concatenation fallback below gets wrong or right by luck.
|
|
179
|
+
# Opt-in only (see docstring) so digit-string IDs aren't coerced into numbers.
|
|
180
|
+
if numeric:
|
|
181
|
+
num_val = _as_number(value)
|
|
182
|
+
if num_val is not None:
|
|
183
|
+
return any(abs(n - num_val) < 0.01 for n in _extract_numbers(source_text))
|
|
184
|
+
|
|
185
|
+
# Date comparison: catches reformatting (22Apr2021 -> 2021-04-22) that no string-based
|
|
186
|
+
# match, fuzzy or otherwise, can see past. Opt-in for the same reason numeric is.
|
|
187
|
+
if date:
|
|
188
|
+
target_date = _parse_date_any(str(value))
|
|
189
|
+
if target_date is not None:
|
|
190
|
+
for candidate in _DATE_CANDIDATE_RE.findall(source_text):
|
|
191
|
+
if _parse_date_any(candidate) == target_date:
|
|
192
|
+
return True
|
|
193
|
+
|
|
194
|
+
# Fuzzy match ignoring non-alphanumeric chars
|
|
195
|
+
val_clean = re.sub(r'\W+', '', val_str)
|
|
196
|
+
source_clean = re.sub(r'\W+', '', source_lower)
|
|
197
|
+
|
|
198
|
+
if val_clean and val_clean in source_clean:
|
|
199
|
+
return True
|
|
200
|
+
|
|
201
|
+
return False
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def validate_field_constraints(schema: Schema, extracted: Dict[str, Any]) -> List[Issue]:
|
|
205
|
+
"""Check schema-declared constraints (required, pattern, enum) against extracted values.
|
|
206
|
+
|
|
207
|
+
Runs automatically for every schema, independent of any user-supplied cross_check rules —
|
|
208
|
+
these constraints are properties of the schema itself (any domain), not custom business logic.
|
|
209
|
+
"""
|
|
210
|
+
issues: List[Issue] = []
|
|
211
|
+
for f in schema.fields:
|
|
212
|
+
value = extracted.get(f.name)
|
|
213
|
+
is_missing = not _is_present(value) or (f.type == "list" and not value)
|
|
214
|
+
|
|
215
|
+
if is_missing:
|
|
216
|
+
if f.required:
|
|
217
|
+
issues.append(Issue(
|
|
218
|
+
field=f.name,
|
|
219
|
+
message=f"'{f.name}' is required but was not found in the document",
|
|
220
|
+
severity="error",
|
|
221
|
+
kind="missing_required",
|
|
222
|
+
))
|
|
223
|
+
continue
|
|
224
|
+
|
|
225
|
+
if f.type == "list":
|
|
226
|
+
continue # pattern/enum constraints apply to sub_fields on their own, not implemented here
|
|
227
|
+
|
|
228
|
+
if f.pattern and isinstance(value, str) and _regex_matches_with_timeout(f.pattern, value) is False:
|
|
229
|
+
issues.append(Issue(
|
|
230
|
+
field=f.name,
|
|
231
|
+
message=f"'{value}' does not match required pattern {f.pattern!r} for '{f.name}'",
|
|
232
|
+
kind="invalid_format",
|
|
233
|
+
))
|
|
234
|
+
|
|
235
|
+
if f.enum and value not in f.enum:
|
|
236
|
+
issues.append(Issue(
|
|
237
|
+
field=f.name,
|
|
238
|
+
message=f"'{value}' is not one of the allowed values for '{f.name}': {f.enum}",
|
|
239
|
+
kind="invalid_format",
|
|
240
|
+
))
|
|
241
|
+
|
|
242
|
+
return issues
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def numeric_sum_rule(list_field: str, total_field: str, item_key: str = "amount", tolerance: float = 0.01) -> Callable[[Dict[str, Any]], Optional[List[Issue]]]:
|
|
246
|
+
"""Build a rule flagging total_field when it doesn't match the sum of item_key across list_field."""
|
|
247
|
+
|
|
248
|
+
def _rule(extracted: Dict[str, Any]) -> Optional[List[Issue]]:
|
|
249
|
+
total = extracted.get(total_field)
|
|
250
|
+
items = extracted.get(list_field)
|
|
251
|
+
if total is None or not items:
|
|
252
|
+
return None
|
|
253
|
+
try:
|
|
254
|
+
total_val = float(total)
|
|
255
|
+
calc_val = sum(float(item.get(item_key, 0) or 0) for item in items)
|
|
256
|
+
except (TypeError, ValueError):
|
|
257
|
+
return None
|
|
258
|
+
if abs(calc_val - total_val) > tolerance:
|
|
259
|
+
return [
|
|
260
|
+
Issue(field=total_field, message=f"{total_field} ({total_val}) does not match sum of {list_field}.{item_key} ({calc_val})"),
|
|
261
|
+
Issue(field=list_field, message=f"Sum of {item_key} ({calc_val}) does not match {total_field} ({total_val})"),
|
|
262
|
+
]
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
return _rule
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def date_parseable_rule(field_name: str, formats: Optional[List[str]] = None) -> Callable[[Dict[str, Any]], Optional[List[Issue]]]:
|
|
269
|
+
"""Build a rule flagging field_name when its value doesn't parse as a date in any given format."""
|
|
270
|
+
from datetime import datetime
|
|
271
|
+
|
|
272
|
+
candidates = formats or ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]
|
|
273
|
+
|
|
274
|
+
def _rule(extracted: Dict[str, Any]) -> Optional[List[Issue]]:
|
|
275
|
+
val = extracted.get(field_name)
|
|
276
|
+
if val is None:
|
|
277
|
+
return None
|
|
278
|
+
for fmt in candidates:
|
|
279
|
+
try:
|
|
280
|
+
datetime.strptime(str(val), fmt)
|
|
281
|
+
return None
|
|
282
|
+
except ValueError:
|
|
283
|
+
continue
|
|
284
|
+
return [Issue(field=field_name, message=f"'{val}' is not a parseable date")]
|
|
285
|
+
|
|
286
|
+
return _rule
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def cross_check(schema: Schema, extracted: Dict[str, Any], rules: List[Callable[[Dict[str, Any]], Optional[List[Issue]]]]) -> List[Issue]:
|
|
290
|
+
"""
|
|
291
|
+
Run custom cross-check rules on the extracted data.
|
|
292
|
+
Each rule is a callable taking the extracted dict and returning a list of Issues (or None).
|
|
293
|
+
"""
|
|
294
|
+
issues = []
|
|
295
|
+
if not rules:
|
|
296
|
+
return issues
|
|
297
|
+
|
|
298
|
+
for rule in rules:
|
|
299
|
+
try:
|
|
300
|
+
result = rule(extracted)
|
|
301
|
+
if result:
|
|
302
|
+
issues.extend(result)
|
|
303
|
+
except Exception:
|
|
304
|
+
logger.exception("Cross-check rule %r raised an exception", getattr(rule, "__name__", rule))
|
|
305
|
+
|
|
306
|
+
return issues
|