docspectra 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docspectra-0.1.0/.gitignore +16 -0
- docspectra-0.1.0/.python-version +1 -0
- docspectra-0.1.0/LICENSE +21 -0
- docspectra-0.1.0/PKG-INFO +72 -0
- docspectra-0.1.0/README.md +33 -0
- docspectra-0.1.0/docspectra/__init__.py +8 -0
- docspectra-0.1.0/docspectra/agents/__init__.py +209 -0
- docspectra-0.1.0/docspectra/parser/__init__.py +38 -0
- docspectra-0.1.0/docspectra/parser/contracts.py +64 -0
- docspectra-0.1.0/docspectra/parser/core.py +192 -0
- docspectra-0.1.0/docspectra/parser/sniff.py +97 -0
- docspectra-0.1.0/docspectra/providers/__init__.py +42 -0
- docspectra-0.1.0/docspectra/providers/_pdfscan.py +122 -0
- docspectra-0.1.0/docspectra/providers/anydoc.py +65 -0
- docspectra-0.1.0/docspectra/providers/mineru.py +213 -0
- docspectra-0.1.0/docspectra/providers/pdf_inspect.py +21 -0
- docspectra-0.1.0/docspectra/providers/txt.py +15 -0
- docspectra-0.1.0/docspectra/service/__init__.py +5 -0
- docspectra-0.1.0/docspectra/service/parse.py +60 -0
- docspectra-0.1.0/pyproject.toml +72 -0
- docspectra-0.1.0/pytest.ini +2 -0
- docspectra-0.1.0/tests/__init__.py +0 -0
- docspectra-0.1.0/tests/docx_builder.py +155 -0
- docspectra-0.1.0/tests/fake_providers.py +32 -0
- docspectra-0.1.0/tests/fixtures/legacy.doc +0 -0
- docspectra-0.1.0/tests/helpers.py +19 -0
- docspectra-0.1.0/tests/pdf_builder.py +76 -0
- docspectra-0.1.0/tests/test_agents.py +140 -0
- docspectra-0.1.0/tests/test_anydoc.py +78 -0
- docspectra-0.1.0/tests/test_boundary.py +36 -0
- docspectra-0.1.0/tests/test_mineru.py +199 -0
- docspectra-0.1.0/tests/test_parser.py +172 -0
- docspectra-0.1.0/tests/test_pdf_inspect.py +89 -0
- docspectra-0.1.0/tests/test_pdfscan.py +40 -0
- docspectra-0.1.0/tests/test_service.py +54 -0
- docspectra-0.1.0/tests/test_smoke.py +13 -0
- docspectra-0.1.0/tests/test_sniff.py +104 -0
- docspectra-0.1.0/tests/test_traced.py +75 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
docspectra-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Flow Jiang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: docspectra
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: DocSpectra: schema-driven document parsing — any file in, spectral JSON out
|
|
5
|
+
Project-URL: Homepage, https://github.com/flowjzh/docspectra
|
|
6
|
+
Project-URL: Repository, https://github.com/flowjzh/docspectra.git
|
|
7
|
+
Project-URL: Issues, https://github.com/flowjzh/docspectra/issues
|
|
8
|
+
Author-email: Flow Jiang <flowjzh@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: documents,extraction,json-schema,llm,parsing
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: jsonschema>=4.23
|
|
25
|
+
Requires-Dist: opentelemetry-api>=1.40.0
|
|
26
|
+
Requires-Dist: xtremeparse>=0.1.0
|
|
27
|
+
Provides-Extra: anydoc
|
|
28
|
+
Requires-Dist: firecrawl-anydoc; extra == 'anydoc'
|
|
29
|
+
Requires-Dist: pypdf; extra == 'anydoc'
|
|
30
|
+
Provides-Extra: mineru
|
|
31
|
+
Requires-Dist: httpx>=0.27; extra == 'mineru'
|
|
32
|
+
Requires-Dist: xtremeflow>=0.4.4; extra == 'mineru'
|
|
33
|
+
Provides-Extra: pdf-inspect
|
|
34
|
+
Requires-Dist: pdf-inspector; extra == 'pdf-inspect'
|
|
35
|
+
Requires-Dist: pypdf; extra == 'pdf-inspect'
|
|
36
|
+
Provides-Extra: pydantic-ai
|
|
37
|
+
Requires-Dist: pydantic-ai-slim[openai]>=2.30.0; extra == 'pydantic-ai'
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
40
|
+
# DocSpectra
|
|
41
|
+
|
|
42
|
+
<img width="600" alt="DocSpectra" src="https://github.com/user-attachments/assets/1d858fb3-67c5-41f6-b0db-3fb6ab39fe4a" />
|
|
43
|
+
|
|
44
|
+
> **Any file in. Spectral JSON out. Fast.**
|
|
45
|
+
|
|
46
|
+
DocSpectra is a schema-driven document-parsing library: hand it a file in
|
|
47
|
+
any supported format and a JSON Schema, get back schema-conforming JSON —
|
|
48
|
+
extracted by extreme-concurrency LLM specialists that chunk, route, fan
|
|
49
|
+
out and self-correct (powered by
|
|
50
|
+
[xtremeparse](https://github.com/flowjzh/xtremeparse)).
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
file + JSON schema ──► docspectra.parse ──► JSON
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Layout
|
|
57
|
+
|
|
58
|
+
- `docspectra/parser` — file → text (provider engines as optional extras:
|
|
59
|
+
`pdf-inspect`, `anydoc`, `mineru`; magic-byte type routing;
|
|
60
|
+
file-level decline chains — a provider whose gate fires on a specific
|
|
61
|
+
file declines and the next engine takes it)
|
|
62
|
+
- `docspectra/agents` — the only PydanticAI boundary (AgentRunner adapter)
|
|
63
|
+
- `docspectra/service` — `parse`: file + JSON schema → JSON
|
|
64
|
+
|
|
65
|
+
DocSpectra is deliberately generic: JSON Schema in and out, no document
|
|
66
|
+
vendors, no HTTP server, no domain tools. Applications assemble it with
|
|
67
|
+
their own schema sources and renderers (e.g. bridging
|
|
68
|
+
[DocXCast](https://github.com/flowjzh/docxcast) templates).
|
|
69
|
+
|
|
70
|
+
## Status
|
|
71
|
+
|
|
72
|
+
Parser layer implemented and tested.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# DocSpectra
|
|
2
|
+
|
|
3
|
+
<img width="600" alt="DocSpectra" src="https://github.com/user-attachments/assets/1d858fb3-67c5-41f6-b0db-3fb6ab39fe4a" />
|
|
4
|
+
|
|
5
|
+
> **Any file in. Spectral JSON out. Fast.**
|
|
6
|
+
|
|
7
|
+
DocSpectra is a schema-driven document-parsing library: hand it a file in
|
|
8
|
+
any supported format and a JSON Schema, get back schema-conforming JSON —
|
|
9
|
+
extracted by extreme-concurrency LLM specialists that chunk, route, fan
|
|
10
|
+
out and self-correct (powered by
|
|
11
|
+
[xtremeparse](https://github.com/flowjzh/xtremeparse)).
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
file + JSON schema ──► docspectra.parse ──► JSON
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Layout
|
|
18
|
+
|
|
19
|
+
- `docspectra/parser` — file → text (provider engines as optional extras:
|
|
20
|
+
`pdf-inspect`, `anydoc`, `mineru`; magic-byte type routing;
|
|
21
|
+
file-level decline chains — a provider whose gate fires on a specific
|
|
22
|
+
file declines and the next engine takes it)
|
|
23
|
+
- `docspectra/agents` — the only PydanticAI boundary (AgentRunner adapter)
|
|
24
|
+
- `docspectra/service` — `parse`: file + JSON schema → JSON
|
|
25
|
+
|
|
26
|
+
DocSpectra is deliberately generic: JSON Schema in and out, no document
|
|
27
|
+
vendors, no HTTP server, no domain tools. Applications assemble it with
|
|
28
|
+
their own schema sources and renderers (e.g. bridging
|
|
29
|
+
[DocXCast](https://github.com/flowjzh/docxcast) templates).
|
|
30
|
+
|
|
31
|
+
## Status
|
|
32
|
+
|
|
33
|
+
Parser layer implemented and tested.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""The only PydanticAI boundary: xtremeparse's AgentRunner adapted here.
|
|
2
|
+
Swapping agent frameworks means rewriting this module and nothing else."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
from typing import Sequence
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
from pydantic_ai import Agent, NativeOutput
|
|
12
|
+
from pydantic_ai.models import Model
|
|
13
|
+
from pydantic_ai.output import StructuredDict
|
|
14
|
+
|
|
15
|
+
from xtremeparse.contracts import AgentResult, Issue, JSONSchema
|
|
16
|
+
from xtremeparse.scheduling import report_tokens
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# qwen commercial/numeric models think by default and dashscope rejects
|
|
20
|
+
# tool_choice='required' (structured output) while thinking — same quirk
|
|
21
|
+
# argus works around; generalized to any qwen3.x numeric series
|
|
22
|
+
_THINKING_OFF = re.compile(
|
|
23
|
+
r'^qwen-(plus|flash|turbo)(-.*)?$|'
|
|
24
|
+
r'^qwen3(\.\d+)?-(omni-flash|vl-plus|flash|plus|\d+b(-a\d+b)?)(-.*)?$')
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class PydanticAIRunner:
|
|
28
|
+
"""Adapts the AgentRunner protocol to PydanticAI.
|
|
29
|
+
|
|
30
|
+
The shared ``content`` prefix rides its own system message (Agent
|
|
31
|
+
``instructions``) so provider prefix caching sees identical bytes
|
|
32
|
+
across every call of one extraction; the unit card, scope and
|
|
33
|
+
feedback trail in the user message — a history round already
|
|
34
|
+
carries the card and scope, so the caller passes ``''`` for them
|
|
35
|
+
and only the per-round feedback is fresh. Structured output goes
|
|
36
|
+
through
|
|
37
|
+
the provider's native JSON-schema mode — tool-mode definitions
|
|
38
|
+
serialize ahead of the system message and displace the shared
|
|
39
|
+
prefix, killing cache reuse across specialists. Array-shaped result
|
|
40
|
+
schemas are wrapped in an ``items`` object (native output requires
|
|
41
|
+
an object schema) and unwrapped on the way out. StructuredDict
|
|
42
|
+
guarantees only a str-keyed dict — deep conformance is the injected
|
|
43
|
+
validator's job. Sampling defaults to temperature 0 (extraction is
|
|
44
|
+
deterministic-intent); ``model_settings`` overrides any field.
|
|
45
|
+
Actual token usage is reported to the dispatching scheduler via
|
|
46
|
+
xtremeparse's ``report_tokens``. ``tools`` is accepted (frozen
|
|
47
|
+
protocol) but not implemented yet."""
|
|
48
|
+
|
|
49
|
+
def __init__(self, model: Model, *, model_settings: dict | None = None,
|
|
50
|
+
strict_output: bool = True):
|
|
51
|
+
self.model = model
|
|
52
|
+
# deterministic-intent extraction: sampling heat off by default
|
|
53
|
+
self.model_settings = {'temperature': 0, **(model_settings or {})}
|
|
54
|
+
# strict keeps the schema in a server-side grammar; without it some
|
|
55
|
+
# providers inject the schema into the prompt behind the payload and
|
|
56
|
+
# break the cacheable prefix — hosts whose strict grammar rejects
|
|
57
|
+
# the schema shape turn it off here
|
|
58
|
+
self.strict_output = strict_output
|
|
59
|
+
|
|
60
|
+
async def run(self, *, instructions: str, result_schema: JSONSchema, content: str,
|
|
61
|
+
scope: str = None, tools: list = None, history: list = None,
|
|
62
|
+
feedback: Sequence[Issue] = None) -> AgentResult:
|
|
63
|
+
if tools:
|
|
64
|
+
raise NotImplementedError('tool-calling is deferred; see design.md §5.1')
|
|
65
|
+
free = result_schema.get('type') == 'string' # free-text mode: lib parses
|
|
66
|
+
wrapped = not free and result_schema.get('type') == 'array'
|
|
67
|
+
schema = _wrap(result_schema) if wrapped else result_schema
|
|
68
|
+
output_type = str if free else NativeOutput(StructuredDict(schema),
|
|
69
|
+
strict=self.strict_output)
|
|
70
|
+
agent = Agent(self.model, output_type=output_type, instructions=content)
|
|
71
|
+
result = await agent.run(_tail(instructions, scope, feedback),
|
|
72
|
+
message_history=history,
|
|
73
|
+
model_settings=self._model_settings())
|
|
74
|
+
await report_tokens(result.usage.input_tokens + result.usage.output_tokens)
|
|
75
|
+
data = result.output.get('items') if wrapped else result.output
|
|
76
|
+
return AgentResult(data=data, history=result.all_messages())
|
|
77
|
+
|
|
78
|
+
def _model_settings(self):
|
|
79
|
+
settings = dict(self.model_settings)
|
|
80
|
+
if _THINKING_OFF.match(getattr(self.model, 'model_name', '')):
|
|
81
|
+
settings['extra_body'] = {'enable_thinking': False}
|
|
82
|
+
return settings
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def enable_tracing() -> None:
|
|
86
|
+
"""Turn on the framework's native OTel span emission for every agent
|
|
87
|
+
(process-wide). Every adapter in this boundary provides this hook so
|
|
88
|
+
hosts enable tracing without naming the framework."""
|
|
89
|
+
Agent.instrument_all(True)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _json_body(body: dict) -> bytes:
|
|
93
|
+
"""SDK serialization style — ensure_ascii would double CJK bytes."""
|
|
94
|
+
return json.dumps(body, ensure_ascii=False, separators=(',', ':')).encode()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _stripped_headers(headers: httpx.Headers) -> httpx.Headers:
|
|
98
|
+
"""Headers for a rewritten body: the length/encoding headers are stale."""
|
|
99
|
+
headers = headers.copy()
|
|
100
|
+
headers.pop('content-length', None)
|
|
101
|
+
headers.pop('content-encoding', None)
|
|
102
|
+
return headers
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def marked_request(request: httpx.Request) -> httpx.Request:
|
|
106
|
+
"""Rewrite a chat-completions request so its leading string-content
|
|
107
|
+
system message becomes a cache-marked content block (dashscope's
|
|
108
|
+
``cache_control`` explicit-cache extension). The marker must close
|
|
109
|
+
the shared payload at a message boundary — marking mid-message does
|
|
110
|
+
not match. Non-matching shapes pass through untouched."""
|
|
111
|
+
body = json.loads(request.read())
|
|
112
|
+
messages = body.get('messages')
|
|
113
|
+
if (messages and messages[0].get('role') == 'system'
|
|
114
|
+
and isinstance(messages[0].get('content'), str)):
|
|
115
|
+
messages[0]['content'] = [{'type': 'text', 'text': messages[0]['content'],
|
|
116
|
+
'cache_control': {'type': 'ephemeral'}}]
|
|
117
|
+
return httpx.Request(request.method, request.url,
|
|
118
|
+
headers=_stripped_headers(request.headers),
|
|
119
|
+
content=_json_body(body))
|
|
120
|
+
return request
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _unthinking(request: httpx.Request) -> httpx.Request:
|
|
124
|
+
"""Drops ``enable_thinking: false`` into a chat-completions body."""
|
|
125
|
+
body = json.loads(request.read())
|
|
126
|
+
return httpx.Request(request.method, request.url,
|
|
127
|
+
headers=_stripped_headers(request.headers),
|
|
128
|
+
content=_json_body(body | {'enable_thinking': False}))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _rename_cache_write(response: httpx.Response) -> httpx.Response:
|
|
132
|
+
"""Relabel dashscope's cache-write usage field to the OpenAI name
|
|
133
|
+
pydantic-ai reads (``prompt_tokens_details.cache_write_tokens``);
|
|
134
|
+
without the relabel the count never reaches the usage object or
|
|
135
|
+
the span."""
|
|
136
|
+
body = response.content
|
|
137
|
+
if b'"cache_creation_input_tokens"' not in body:
|
|
138
|
+
return response
|
|
139
|
+
return httpx.Response(response.status_code,
|
|
140
|
+
headers=_stripped_headers(response.headers),
|
|
141
|
+
content=body.replace(b'"cache_creation_input_tokens"',
|
|
142
|
+
b'"cache_write_tokens"'))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _WireTransport(httpx.AsyncHTTPTransport):
|
|
146
|
+
"""Applies provider wire transforms — requests before they leave,
|
|
147
|
+
chat-completion responses as they return; the openai client beneath
|
|
148
|
+
is untouched."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, transforms: list):
|
|
151
|
+
super().__init__() # the base owns the connection pool
|
|
152
|
+
self.transforms = transforms
|
|
153
|
+
|
|
154
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
155
|
+
if request.url.path.endswith('/chat/completions'):
|
|
156
|
+
for transform in self.transforms:
|
|
157
|
+
request = transform(request)
|
|
158
|
+
response = await super().handle_async_request(request)
|
|
159
|
+
if 'application/json' in response.headers.get('content-type', ''):
|
|
160
|
+
await response.aread() # a rewritable body must be read first
|
|
161
|
+
return _rename_cache_write(response)
|
|
162
|
+
return response
|
|
163
|
+
return await super().handle_async_request(request)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def openai_model(base_url: str, api_key: str, model: str, *, explicit_cache: bool = False,
|
|
167
|
+
thinking_off: bool = False,
|
|
168
|
+
http_client: httpx.AsyncClient | None = None):
|
|
169
|
+
"""A pydantic-ai Model for any OpenAI-compatible endpoint — the
|
|
170
|
+
provider mechanism lives inside the agents boundary; hosts pass
|
|
171
|
+
only deployment config. Requires the ``docspectra[pydantic-ai]`` extra.
|
|
172
|
+
``explicit_cache`` marks the leading system message as a provider
|
|
173
|
+
cache block (dashscope explicit cache): paired with a runner that
|
|
174
|
+
puts the shared payload in Agent instructions, the call that runs
|
|
175
|
+
first pays cache creation once and every call behind it hits; the
|
|
176
|
+
wire transport relabels dashscope's non-standard cache-write usage
|
|
177
|
+
field so the count also lands on the span. Providers that reject
|
|
178
|
+
the extension must leave it off.
|
|
179
|
+
``thinking_off`` drops ``enable_thinking: false`` into every request
|
|
180
|
+
— for consumers that build their own pydantic-ai agents outside the
|
|
181
|
+
runner's model_settings (pydantic-evals LLMJudge: tools +
|
|
182
|
+
tool_choice='required', which dashscope rejects while thinking).
|
|
183
|
+
A host ``http_client`` (proxies, connection tuning) is used
|
|
184
|
+
verbatim."""
|
|
185
|
+
try:
|
|
186
|
+
from pydantic_ai.models.openai import OpenAIChatModel
|
|
187
|
+
from pydantic_ai.providers.openai import OpenAIProvider
|
|
188
|
+
except ImportError as e:
|
|
189
|
+
raise ImportError('install the pydantic-ai extra: docspectra[pydantic-ai]') from e
|
|
190
|
+
if transforms := ([marked_request] if explicit_cache else []) \
|
|
191
|
+
+ ([_unthinking] if thinking_off else []):
|
|
192
|
+
http_client = httpx.AsyncClient(transport=_WireTransport(transforms))
|
|
193
|
+
return OpenAIChatModel(model, provider=OpenAIProvider(
|
|
194
|
+
base_url=base_url, api_key=api_key, http_client=http_client))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _wrap(schema: JSONSchema) -> JSONSchema:
|
|
198
|
+
return {'type': 'object', 'properties': {'items': schema},
|
|
199
|
+
'required': ['items'], 'additionalProperties': False}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _tail(instructions: str, scope, feedback) -> str:
|
|
203
|
+
parts = [instructions] if instructions else []
|
|
204
|
+
if scope is not None:
|
|
205
|
+
parts.append(f'\n\n---\nAssigned material:\n{scope}')
|
|
206
|
+
if feedback:
|
|
207
|
+
lines = '\n'.join(f'- {i.path}: {i.message}' for i in feedback)
|
|
208
|
+
parts.append(f'\n\n---\nValidation errors to fix:\n{lines}')
|
|
209
|
+
return ''.join(parts)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""File → text. Provider engines as optional extras (``docspectra[pdf-inspect]``,
|
|
2
|
+
``docspectra[anydoc]``, ``docspectra[mineru]``), magic-byte type routing,
|
|
3
|
+
and file-level decline chains: ``DOCSPECTRA_PARSER_CONFIG`` orders each type's
|
|
4
|
+
chain and a provider whose gate fires on a specific file declines, letting the
|
|
5
|
+
next engine take it."""
|
|
6
|
+
|
|
7
|
+
from docspectra.parser.contracts import (Attempt, Declined, FileInput,
|
|
8
|
+
ImageAsset, MAX_ASSETS, MAX_ASSET_BYTES,
|
|
9
|
+
ParseResult)
|
|
10
|
+
from docspectra.parser.core import (
|
|
11
|
+
ParseDeclinedError,
|
|
12
|
+
Provider,
|
|
13
|
+
configure,
|
|
14
|
+
parse,
|
|
15
|
+
parse_path,
|
|
16
|
+
provider,
|
|
17
|
+
register,
|
|
18
|
+
)
|
|
19
|
+
from docspectra.parser.sniff import sniff, validate_declared
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
'Attempt',
|
|
23
|
+
'Declined',
|
|
24
|
+
'FileInput',
|
|
25
|
+
'ImageAsset',
|
|
26
|
+
'MAX_ASSETS',
|
|
27
|
+
'MAX_ASSET_BYTES',
|
|
28
|
+
'ParseDeclinedError',
|
|
29
|
+
'ParseResult',
|
|
30
|
+
'Provider',
|
|
31
|
+
'configure',
|
|
32
|
+
'parse',
|
|
33
|
+
'parse_path',
|
|
34
|
+
'provider',
|
|
35
|
+
'register',
|
|
36
|
+
'sniff',
|
|
37
|
+
'validate_declared',
|
|
38
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Parser contracts: result shapes and control-flow values shared by core
|
|
2
|
+
and providers."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
FileInput = bytes | str | Path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class ImageAsset:
|
|
15
|
+
"""One embedded image carried faithfully, uninterpreted.
|
|
16
|
+
|
|
17
|
+
``name`` matches the ```` reference in ParseResult.text;
|
|
18
|
+
``source`` is engine provenance ('pdf p2', 'word/media/image1.png').
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
name: str
|
|
22
|
+
mime: str
|
|
23
|
+
data: bytes
|
|
24
|
+
source: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# payload-budget policy for ParseResult.images — every provider skips
|
|
28
|
+
# assets over these limits and records the skip count in meta
|
|
29
|
+
MAX_ASSETS = 50
|
|
30
|
+
MAX_ASSET_BYTES = 5 * 1024 * 1024
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Attempt:
|
|
35
|
+
"""One provider's outcome on a file, recorded by core on success and
|
|
36
|
+
failure alike — the eval suite reads routing through it."""
|
|
37
|
+
|
|
38
|
+
provider: str
|
|
39
|
+
outcome: Literal['success', 'declined', 'error']
|
|
40
|
+
reason: str | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Declined:
|
|
45
|
+
"""Expected control flow: this engine cannot read *this* file, or
|
|
46
|
+
read it into a known false-positive shape (a gate fired). Never an
|
|
47
|
+
exception — the chain moves to the next provider."""
|
|
48
|
+
|
|
49
|
+
reason: str
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class ParseResult:
|
|
54
|
+
"""The outcome of a successful parse. ``format`` is constrained —
|
|
55
|
+
extend the Literal, never free-form. ``meta`` is provider diagnostics
|
|
56
|
+
(gate hits, page counts, task ids, timings). ``provider`` and
|
|
57
|
+
``attempts`` are stamped by core, not the provider."""
|
|
58
|
+
|
|
59
|
+
text: str
|
|
60
|
+
provider: str = '' # engine that accepted the file (core-stamped)
|
|
61
|
+
format: Literal['markdown'] = 'markdown'
|
|
62
|
+
images: list[ImageAsset] = field(default_factory=list)
|
|
63
|
+
attempts: list[Attempt] = field(default_factory=list)
|
|
64
|
+
meta: dict = field(default_factory=dict)
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Provider registry, decline chains, and the async parse entry.
|
|
2
|
+
|
|
3
|
+
Core carries zero parsing dependencies: built-in providers are in-tree
|
|
4
|
+
modules under ``docspectra.providers``, try-imported at init — an absent
|
|
5
|
+
extra is skipped silently (an installed-but-broken provider is recorded
|
|
6
|
+
and surfaced in config errors instead). Engines declare themselves with
|
|
7
|
+
:func:`provider`; chains are ordered by ``DOCSPECTRA_PARSER_CONFIG`` (a
|
|
8
|
+
JSON env var mapping a document type to an ordered provider list),
|
|
9
|
+
falling back to the derived default chain — the canonical order of
|
|
10
|
+
installed providers that declared the type. Declines are return values,
|
|
11
|
+
hard errors fall through; chain exhaustion raises
|
|
12
|
+
:class:`ParseDeclinedError` with every attempt aggregated."""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import importlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Awaitable, Callable, Sequence
|
|
22
|
+
|
|
23
|
+
from docspectra.parser.contracts import Attempt, Declined, ParseResult
|
|
24
|
+
from docspectra.parser.sniff import sniff, validate_declared
|
|
25
|
+
|
|
26
|
+
_BUILTIN_MODULES = ('txt', 'pdf_inspect', 'anydoc', 'mineru')
|
|
27
|
+
_BUILTIN_NAMES = {m.replace('_', '-') for m in _BUILTIN_MODULES}
|
|
28
|
+
_CONFIG_ENV = 'DOCSPECTRA_PARSER_CONFIG'
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Provider:
|
|
33
|
+
"""One engine: the async parse entry plus the types it declares."""
|
|
34
|
+
|
|
35
|
+
name: str
|
|
36
|
+
types: tuple[str, ...]
|
|
37
|
+
parse: Callable[..., Awaitable[ParseResult | Declined]]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ParseDeclinedError(Exception):
|
|
41
|
+
"""Every provider in the chain declined or errored on this file.
|
|
42
|
+
|
|
43
|
+
``attempts`` aggregates every outcome. A hard error raised through
|
|
44
|
+
the chain is chained as the cause.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, attempts: list[Attempt]):
|
|
48
|
+
self.attempts = attempts
|
|
49
|
+
detail = '; '.join(
|
|
50
|
+
f"{a.provider}: {a.outcome}" + (f' ({a.reason})' if a.reason else '')
|
|
51
|
+
for a in attempts
|
|
52
|
+
)
|
|
53
|
+
super().__init__(f'every provider declined: {detail}')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_providers: dict[str, Provider] = {} # keyed by normalized (hyphen) engine name
|
|
57
|
+
_builtins_loaded = False
|
|
58
|
+
_config: dict | None = None # effective chain config; resolved from env on first use
|
|
59
|
+
_import_errors: dict[str, str] = {} # provider module → failure, when installed-but-broken
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _norm(name: str) -> str:
|
|
63
|
+
return name.replace('_', '-')
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def register(name: str, types: Sequence[str], fn) -> None:
|
|
67
|
+
"""Programmatic registration, same surface as the decorator. Names are
|
|
68
|
+
normalized (hyphen↔underscore) and registration is last-write-wins."""
|
|
69
|
+
_providers[_norm(name)] = Provider(_norm(name), tuple(types), fn)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def provider(name: str, types: Sequence[str]):
|
|
73
|
+
"""Declare an engine around an async ``parse(data, *, name=None)
|
|
74
|
+
-> ParseResult | Declined``."""
|
|
75
|
+
|
|
76
|
+
def deco(fn):
|
|
77
|
+
register(name, types, fn)
|
|
78
|
+
return fn
|
|
79
|
+
|
|
80
|
+
return deco
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def ensure_builtins() -> None:
|
|
84
|
+
global _builtins_loaded
|
|
85
|
+
if _builtins_loaded:
|
|
86
|
+
return
|
|
87
|
+
_builtins_loaded = True
|
|
88
|
+
for module in _BUILTIN_MODULES:
|
|
89
|
+
try:
|
|
90
|
+
importlib.import_module(f'docspectra.providers.{module}')
|
|
91
|
+
except ImportError as e:
|
|
92
|
+
if not isinstance(e, ModuleNotFoundError) \
|
|
93
|
+
or (e.name or '').startswith('docspectra'):
|
|
94
|
+
_import_errors[_norm(module)] = f'{type(e).__name__}: {e}'
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def configure(config: dict | None = None) -> None:
|
|
98
|
+
"""Pin chains to an explicit ``{type: [engine, ...]}`` mapping, or
|
|
99
|
+
reload them from DOCSPECTRA_PARSER_CONFIG (None, validated here —
|
|
100
|
+
the fail-fast hook a host calls at startup)."""
|
|
101
|
+
global _config
|
|
102
|
+
ensure_builtins()
|
|
103
|
+
_config = _validate(config) if config is not None else _env_config()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
async def parse(data: bytes, *, name: str | None = None) -> ParseResult:
|
|
107
|
+
"""Parse one file (bytes are the contract) into a ParseResult,
|
|
108
|
+
walking the type's decline chain. Raises ValueError on unroutable
|
|
109
|
+
input, on a declared binary extension whose content fails magic
|
|
110
|
+
verification, or when no provider is installed; ParseDeclinedError
|
|
111
|
+
when every provider declines or errors."""
|
|
112
|
+
ensure_builtins()
|
|
113
|
+
validate_declared(data, name) # declared-binary gate, before any provider
|
|
114
|
+
doc_type = sniff(data, name)
|
|
115
|
+
if doc_type is None:
|
|
116
|
+
raise ValueError(f'cannot determine document type of {name or "input"}')
|
|
117
|
+
chain = _chain_for(doc_type)
|
|
118
|
+
if not chain:
|
|
119
|
+
raise ValueError(f'no provider installed for document type {doc_type!r}')
|
|
120
|
+
attempts: list[Attempt] = []
|
|
121
|
+
cause = None
|
|
122
|
+
for provider_name in chain:
|
|
123
|
+
p = _providers[provider_name]
|
|
124
|
+
try:
|
|
125
|
+
result = await p.parse(data, name=name)
|
|
126
|
+
except Exception as e:
|
|
127
|
+
attempts.append(Attempt(p.name, 'error', f'{type(e).__name__}: {e}'))
|
|
128
|
+
cause = cause or e
|
|
129
|
+
continue
|
|
130
|
+
if isinstance(result, Declined):
|
|
131
|
+
attempts.append(Attempt(p.name, 'declined', result.reason))
|
|
132
|
+
continue
|
|
133
|
+
result.provider = p.name
|
|
134
|
+
result.attempts = [*attempts, Attempt(p.name, 'success')]
|
|
135
|
+
return result
|
|
136
|
+
raise ParseDeclinedError(attempts) from cause
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def parse_path(path: str | Path) -> ParseResult:
|
|
140
|
+
"""Convenience wrapper: read the file and parse it, name from the path."""
|
|
141
|
+
path = Path(path)
|
|
142
|
+
return await parse(path.read_bytes(), name=path.name)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _chain_for(doc_type: str) -> list[str]:
|
|
146
|
+
config = _effective_config()
|
|
147
|
+
if doc_type in config:
|
|
148
|
+
return list(config[doc_type])
|
|
149
|
+
# derived default chain: canonical order of installed providers that
|
|
150
|
+
# declared the type — extends automatically as built-ins are added
|
|
151
|
+
return [p.name for p in _providers.values() if doc_type in p.types]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _effective_config() -> dict:
|
|
155
|
+
global _config
|
|
156
|
+
if _config is None:
|
|
157
|
+
_config = _env_config()
|
|
158
|
+
return _config
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _env_config() -> dict:
|
|
162
|
+
raw = os.environ.get(_CONFIG_ENV)
|
|
163
|
+
if not raw:
|
|
164
|
+
return {}
|
|
165
|
+
try:
|
|
166
|
+
config = json.loads(raw)
|
|
167
|
+
except json.JSONDecodeError as e:
|
|
168
|
+
raise ValueError(f'{_CONFIG_ENV} is not valid JSON: {e}') from e
|
|
169
|
+
if not isinstance(config, dict):
|
|
170
|
+
raise ValueError(f'{_CONFIG_ENV} must be a JSON object of type → engine list')
|
|
171
|
+
return _validate(config)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _validate(config: dict) -> dict:
|
|
175
|
+
validated = {}
|
|
176
|
+
for doc_type, names in config.items():
|
|
177
|
+
if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
|
|
178
|
+
raise ValueError(f'{_CONFIG_ENV}[{doc_type!r}] must be a list of engine names')
|
|
179
|
+
normed = []
|
|
180
|
+
for name in names:
|
|
181
|
+
key = _norm(name)
|
|
182
|
+
if key not in _providers:
|
|
183
|
+
if key in _BUILTIN_NAMES:
|
|
184
|
+
detail = f' ({_import_errors[key]})' if key in _import_errors else ''
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f'provider {name!r} is configured but not installed{detail} — '
|
|
187
|
+
f'install the matching extra (docspectra[{key}])')
|
|
188
|
+
raise ValueError(
|
|
189
|
+
f'unknown parser provider {name!r}; known: {sorted(_providers)}')
|
|
190
|
+
normed.append(key)
|
|
191
|
+
validated[doc_type] = normed
|
|
192
|
+
return validated
|