docspectra 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. docspectra-0.1.0/.gitignore +16 -0
  2. docspectra-0.1.0/.python-version +1 -0
  3. docspectra-0.1.0/LICENSE +21 -0
  4. docspectra-0.1.0/PKG-INFO +72 -0
  5. docspectra-0.1.0/README.md +33 -0
  6. docspectra-0.1.0/docspectra/__init__.py +8 -0
  7. docspectra-0.1.0/docspectra/agents/__init__.py +209 -0
  8. docspectra-0.1.0/docspectra/parser/__init__.py +38 -0
  9. docspectra-0.1.0/docspectra/parser/contracts.py +64 -0
  10. docspectra-0.1.0/docspectra/parser/core.py +192 -0
  11. docspectra-0.1.0/docspectra/parser/sniff.py +97 -0
  12. docspectra-0.1.0/docspectra/providers/__init__.py +42 -0
  13. docspectra-0.1.0/docspectra/providers/_pdfscan.py +122 -0
  14. docspectra-0.1.0/docspectra/providers/anydoc.py +65 -0
  15. docspectra-0.1.0/docspectra/providers/mineru.py +213 -0
  16. docspectra-0.1.0/docspectra/providers/pdf_inspect.py +21 -0
  17. docspectra-0.1.0/docspectra/providers/txt.py +15 -0
  18. docspectra-0.1.0/docspectra/service/__init__.py +5 -0
  19. docspectra-0.1.0/docspectra/service/parse.py +60 -0
  20. docspectra-0.1.0/pyproject.toml +72 -0
  21. docspectra-0.1.0/pytest.ini +2 -0
  22. docspectra-0.1.0/tests/__init__.py +0 -0
  23. docspectra-0.1.0/tests/docx_builder.py +155 -0
  24. docspectra-0.1.0/tests/fake_providers.py +32 -0
  25. docspectra-0.1.0/tests/fixtures/legacy.doc +0 -0
  26. docspectra-0.1.0/tests/helpers.py +19 -0
  27. docspectra-0.1.0/tests/pdf_builder.py +76 -0
  28. docspectra-0.1.0/tests/test_agents.py +140 -0
  29. docspectra-0.1.0/tests/test_anydoc.py +78 -0
  30. docspectra-0.1.0/tests/test_boundary.py +36 -0
  31. docspectra-0.1.0/tests/test_mineru.py +199 -0
  32. docspectra-0.1.0/tests/test_parser.py +172 -0
  33. docspectra-0.1.0/tests/test_pdf_inspect.py +89 -0
  34. docspectra-0.1.0/tests/test_pdfscan.py +40 -0
  35. docspectra-0.1.0/tests/test_service.py +54 -0
  36. docspectra-0.1.0/tests/test_smoke.py +13 -0
  37. docspectra-0.1.0/tests/test_sniff.py +104 -0
  38. docspectra-0.1.0/tests/test_traced.py +75 -0
@@ -0,0 +1,16 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # Workspace Local
13
+ .env
14
+ .claude
15
+ .DS_Store
16
+
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Flow Jiang
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,72 @@
1
+ Metadata-Version: 2.5
2
+ Name: docspectra
3
+ Version: 0.1.0
4
+ Summary: DocSpectra: schema-driven document parsing — any file in, spectral JSON out
5
+ Project-URL: Homepage, https://github.com/flowjzh/docspectra
6
+ Project-URL: Repository, https://github.com/flowjzh/docspectra.git
7
+ Project-URL: Issues, https://github.com/flowjzh/docspectra/issues
8
+ Author-email: Flow Jiang <flowjzh@gmail.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: documents,extraction,json-schema,llm,parsing
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.10
24
+ Requires-Dist: jsonschema>=4.23
25
+ Requires-Dist: opentelemetry-api>=1.40.0
26
+ Requires-Dist: xtremeparse>=0.1.0
27
+ Provides-Extra: anydoc
28
+ Requires-Dist: firecrawl-anydoc; extra == 'anydoc'
29
+ Requires-Dist: pypdf; extra == 'anydoc'
30
+ Provides-Extra: mineru
31
+ Requires-Dist: httpx>=0.27; extra == 'mineru'
32
+ Requires-Dist: xtremeflow>=0.4.4; extra == 'mineru'
33
+ Provides-Extra: pdf-inspect
34
+ Requires-Dist: pdf-inspector; extra == 'pdf-inspect'
35
+ Requires-Dist: pypdf; extra == 'pdf-inspect'
36
+ Provides-Extra: pydantic-ai
37
+ Requires-Dist: pydantic-ai-slim[openai]>=2.30.0; extra == 'pydantic-ai'
38
+ Description-Content-Type: text/markdown
39
+
40
+ # DocSpectra
41
+
42
+ <img width="600" alt="DocSpectra" src="https://github.com/user-attachments/assets/1d858fb3-67c5-41f6-b0db-3fb6ab39fe4a" />
43
+
44
+ > **Any file in. Spectral JSON out. Fast.**
45
+
46
+ DocSpectra is a schema-driven document-parsing library: hand it a file in
47
+ any supported format and a JSON Schema, get back schema-conforming JSON —
48
+ extracted by extreme-concurrency LLM specialists that chunk, route, fan
49
+ out and self-correct (powered by
50
+ [xtremeparse](https://github.com/flowjzh/xtremeparse)).
51
+
52
+ ```
53
+ file + JSON schema ──► docspectra.parse ──► JSON
54
+ ```
55
+
56
+ ## Layout
57
+
58
+ - `docspectra/parser` — file → text (provider engines as optional extras:
59
+ `pdf-inspect`, `anydoc`, `mineru`; magic-byte type routing;
60
+ file-level decline chains — a provider whose gate fires on a specific
61
+ file declines and the next engine takes it)
62
+ - `docspectra/agents` — the only PydanticAI boundary (AgentRunner adapter)
63
+ - `docspectra/service` — `parse`: file + JSON schema → JSON
64
+
65
+ DocSpectra is deliberately generic: JSON Schema in and out, no document
66
+ vendors, no HTTP server, no domain tools. Applications assemble it with
67
+ their own schema sources and renderers (e.g. bridging
68
+ [DocXCast](https://github.com/flowjzh/docxcast) templates).
69
+
70
+ ## Status
71
+
72
+ Parser layer implemented and tested.
@@ -0,0 +1,33 @@
1
+ # DocSpectra
2
+
3
+ <img width="600" alt="DocSpectra" src="https://github.com/user-attachments/assets/1d858fb3-67c5-41f6-b0db-3fb6ab39fe4a" />
4
+
5
+ > **Any file in. Spectral JSON out. Fast.**
6
+
7
+ DocSpectra is a schema-driven document-parsing library: hand it a file in
8
+ any supported format and a JSON Schema, get back schema-conforming JSON —
9
+ extracted by extreme-concurrency LLM specialists that chunk, route, fan
10
+ out and self-correct (powered by
11
+ [xtremeparse](https://github.com/flowjzh/xtremeparse)).
12
+
13
+ ```
14
+ file + JSON schema ──► docspectra.parse ──► JSON
15
+ ```
16
+
17
+ ## Layout
18
+
19
+ - `docspectra/parser` — file → text (provider engines as optional extras:
20
+ `pdf-inspect`, `anydoc`, `mineru`; magic-byte type routing;
21
+ file-level decline chains — a provider whose gate fires on a specific
22
+ file declines and the next engine takes it)
23
+ - `docspectra/agents` — the only PydanticAI boundary (AgentRunner adapter)
24
+ - `docspectra/service` — `parse`: file + JSON schema → JSON
25
+
26
+ DocSpectra is deliberately generic: JSON Schema in and out, no document
27
+ vendors, no HTTP server, no domain tools. Applications assemble it with
28
+ their own schema sources and renderers (e.g. bridging
29
+ [DocXCast](https://github.com/flowjzh/docxcast) templates).
30
+
31
+ ## Status
32
+
33
+ Parser layer implemented and tested.
@@ -0,0 +1,8 @@
1
+ """DocSpectra: schema-driven document parsing — any file in, spectral JSON out."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version('docspectra')
7
+ except PackageNotFoundError:
8
+ __version__ = '0.1.0'
@@ -0,0 +1,209 @@
1
+ """The only PydanticAI boundary: xtremeparse's AgentRunner adapted here.
2
+ Swapping agent frameworks means rewriting this module and nothing else."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import re
8
+ from typing import Sequence
9
+
10
+ import httpx
11
+ from pydantic_ai import Agent, NativeOutput
12
+ from pydantic_ai.models import Model
13
+ from pydantic_ai.output import StructuredDict
14
+
15
+ from xtremeparse.contracts import AgentResult, Issue, JSONSchema
16
+ from xtremeparse.scheduling import report_tokens
17
+
18
+
19
+ # qwen commercial/numeric models think by default and dashscope rejects
20
+ # tool_choice='required' (structured output) while thinking — same quirk
21
+ # argus works around; generalized to any qwen3.x numeric series
22
+ _THINKING_OFF = re.compile(
23
+ r'^qwen-(plus|flash|turbo)(-.*)?$|'
24
+ r'^qwen3(\.\d+)?-(omni-flash|vl-plus|flash|plus|\d+b(-a\d+b)?)(-.*)?$')
25
+
26
+
27
+ class PydanticAIRunner:
28
+ """Adapts the AgentRunner protocol to PydanticAI.
29
+
30
+ The shared ``content`` prefix rides its own system message (Agent
31
+ ``instructions``) so provider prefix caching sees identical bytes
32
+ across every call of one extraction; the unit card, scope and
33
+ feedback trail in the user message — a history round already
34
+ carries the card and scope, so the caller passes ``''`` for them
35
+ and only the per-round feedback is fresh. Structured output goes
36
+ through
37
+ the provider's native JSON-schema mode — tool-mode definitions
38
+ serialize ahead of the system message and displace the shared
39
+ prefix, killing cache reuse across specialists. Array-shaped result
40
+ schemas are wrapped in an ``items`` object (native output requires
41
+ an object schema) and unwrapped on the way out. StructuredDict
42
+ guarantees only a str-keyed dict — deep conformance is the injected
43
+ validator's job. Sampling defaults to temperature 0 (extraction is
44
+ deterministic-intent); ``model_settings`` overrides any field.
45
+ Actual token usage is reported to the dispatching scheduler via
46
+ xtremeparse's ``report_tokens``. ``tools`` is accepted (frozen
47
+ protocol) but not implemented yet."""
48
+
49
+ def __init__(self, model: Model, *, model_settings: dict | None = None,
50
+ strict_output: bool = True):
51
+ self.model = model
52
+ # deterministic-intent extraction: sampling heat off by default
53
+ self.model_settings = {'temperature': 0, **(model_settings or {})}
54
+ # strict keeps the schema in a server-side grammar; without it some
55
+ # providers inject the schema into the prompt behind the payload and
56
+ # break the cacheable prefix — hosts whose strict grammar rejects
57
+ # the schema shape turn it off here
58
+ self.strict_output = strict_output
59
+
60
+ async def run(self, *, instructions: str, result_schema: JSONSchema, content: str,
61
+ scope: str = None, tools: list = None, history: list = None,
62
+ feedback: Sequence[Issue] = None) -> AgentResult:
63
+ if tools:
64
+ raise NotImplementedError('tool-calling is deferred; see design.md §5.1')
65
+ free = result_schema.get('type') == 'string' # free-text mode: lib parses
66
+ wrapped = not free and result_schema.get('type') == 'array'
67
+ schema = _wrap(result_schema) if wrapped else result_schema
68
+ output_type = str if free else NativeOutput(StructuredDict(schema),
69
+ strict=self.strict_output)
70
+ agent = Agent(self.model, output_type=output_type, instructions=content)
71
+ result = await agent.run(_tail(instructions, scope, feedback),
72
+ message_history=history,
73
+ model_settings=self._model_settings())
74
+ await report_tokens(result.usage.input_tokens + result.usage.output_tokens)
75
+ data = result.output.get('items') if wrapped else result.output
76
+ return AgentResult(data=data, history=result.all_messages())
77
+
78
+ def _model_settings(self):
79
+ settings = dict(self.model_settings)
80
+ if _THINKING_OFF.match(getattr(self.model, 'model_name', '')):
81
+ settings['extra_body'] = {'enable_thinking': False}
82
+ return settings
83
+
84
+
85
+ def enable_tracing() -> None:
86
+ """Turn on the framework's native OTel span emission for every agent
87
+ (process-wide). Every adapter in this boundary provides this hook so
88
+ hosts enable tracing without naming the framework."""
89
+ Agent.instrument_all(True)
90
+
91
+
92
+ def _json_body(body: dict) -> bytes:
93
+ """SDK serialization style — ensure_ascii would double CJK bytes."""
94
+ return json.dumps(body, ensure_ascii=False, separators=(',', ':')).encode()
95
+
96
+
97
+ def _stripped_headers(headers: httpx.Headers) -> httpx.Headers:
98
+ """Headers for a rewritten body: the length/encoding headers are stale."""
99
+ headers = headers.copy()
100
+ headers.pop('content-length', None)
101
+ headers.pop('content-encoding', None)
102
+ return headers
103
+
104
+
105
+ def marked_request(request: httpx.Request) -> httpx.Request:
106
+ """Rewrite a chat-completions request so its leading string-content
107
+ system message becomes a cache-marked content block (dashscope's
108
+ ``cache_control`` explicit-cache extension). The marker must close
109
+ the shared payload at a message boundary — marking mid-message does
110
+ not match. Non-matching shapes pass through untouched."""
111
+ body = json.loads(request.read())
112
+ messages = body.get('messages')
113
+ if (messages and messages[0].get('role') == 'system'
114
+ and isinstance(messages[0].get('content'), str)):
115
+ messages[0]['content'] = [{'type': 'text', 'text': messages[0]['content'],
116
+ 'cache_control': {'type': 'ephemeral'}}]
117
+ return httpx.Request(request.method, request.url,
118
+ headers=_stripped_headers(request.headers),
119
+ content=_json_body(body))
120
+ return request
121
+
122
+
123
+ def _unthinking(request: httpx.Request) -> httpx.Request:
124
+ """Drops ``enable_thinking: false`` into a chat-completions body."""
125
+ body = json.loads(request.read())
126
+ return httpx.Request(request.method, request.url,
127
+ headers=_stripped_headers(request.headers),
128
+ content=_json_body(body | {'enable_thinking': False}))
129
+
130
+
131
+ def _rename_cache_write(response: httpx.Response) -> httpx.Response:
132
+ """Relabel dashscope's cache-write usage field to the OpenAI name
133
+ pydantic-ai reads (``prompt_tokens_details.cache_write_tokens``);
134
+ without the relabel the count never reaches the usage object or
135
+ the span."""
136
+ body = response.content
137
+ if b'"cache_creation_input_tokens"' not in body:
138
+ return response
139
+ return httpx.Response(response.status_code,
140
+ headers=_stripped_headers(response.headers),
141
+ content=body.replace(b'"cache_creation_input_tokens"',
142
+ b'"cache_write_tokens"'))
143
+
144
+
145
+ class _WireTransport(httpx.AsyncHTTPTransport):
146
+ """Applies provider wire transforms — requests before they leave,
147
+ chat-completion responses as they return; the openai client beneath
148
+ is untouched."""
149
+
150
+ def __init__(self, transforms: list):
151
+ super().__init__() # the base owns the connection pool
152
+ self.transforms = transforms
153
+
154
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
155
+ if request.url.path.endswith('/chat/completions'):
156
+ for transform in self.transforms:
157
+ request = transform(request)
158
+ response = await super().handle_async_request(request)
159
+ if 'application/json' in response.headers.get('content-type', ''):
160
+ await response.aread() # a rewritable body must be read first
161
+ return _rename_cache_write(response)
162
+ return response
163
+ return await super().handle_async_request(request)
164
+
165
+
166
+ def openai_model(base_url: str, api_key: str, model: str, *, explicit_cache: bool = False,
167
+ thinking_off: bool = False,
168
+ http_client: httpx.AsyncClient | None = None):
169
+ """A pydantic-ai Model for any OpenAI-compatible endpoint — the
170
+ provider mechanism lives inside the agents boundary; hosts pass
171
+ only deployment config. Requires the ``docspectra[pydantic-ai]`` extra.
172
+ ``explicit_cache`` marks the leading system message as a provider
173
+ cache block (dashscope explicit cache): paired with a runner that
174
+ puts the shared payload in Agent instructions, the call that runs
175
+ first pays cache creation once and every call behind it hits; the
176
+ wire transport relabels dashscope's non-standard cache-write usage
177
+ field so the count also lands on the span. Providers that reject
178
+ the extension must leave it off.
179
+ ``thinking_off`` drops ``enable_thinking: false`` into every request
180
+ — for consumers that build their own pydantic-ai agents outside the
181
+ runner's model_settings (pydantic-evals LLMJudge: tools +
182
+ tool_choice='required', which dashscope rejects while thinking).
183
+ A host ``http_client`` (proxies, connection tuning) is used
184
+ verbatim."""
185
+ try:
186
+ from pydantic_ai.models.openai import OpenAIChatModel
187
+ from pydantic_ai.providers.openai import OpenAIProvider
188
+ except ImportError as e:
189
+ raise ImportError('install the pydantic-ai extra: docspectra[pydantic-ai]') from e
190
+ if transforms := ([marked_request] if explicit_cache else []) \
191
+ + ([_unthinking] if thinking_off else []):
192
+ http_client = httpx.AsyncClient(transport=_WireTransport(transforms))
193
+ return OpenAIChatModel(model, provider=OpenAIProvider(
194
+ base_url=base_url, api_key=api_key, http_client=http_client))
195
+
196
+
197
+ def _wrap(schema: JSONSchema) -> JSONSchema:
198
+ return {'type': 'object', 'properties': {'items': schema},
199
+ 'required': ['items'], 'additionalProperties': False}
200
+
201
+
202
+ def _tail(instructions: str, scope, feedback) -> str:
203
+ parts = [instructions] if instructions else []
204
+ if scope is not None:
205
+ parts.append(f'\n\n---\nAssigned material:\n{scope}')
206
+ if feedback:
207
+ lines = '\n'.join(f'- {i.path}: {i.message}' for i in feedback)
208
+ parts.append(f'\n\n---\nValidation errors to fix:\n{lines}')
209
+ return ''.join(parts)
@@ -0,0 +1,38 @@
1
+ """File → text. Provider engines as optional extras (``docspectra[pdf-inspect]``,
2
+ ``docspectra[anydoc]``, ``docspectra[mineru]``), magic-byte type routing,
3
+ and file-level decline chains: ``DOCSPECTRA_PARSER_CONFIG`` orders each type's
4
+ chain and a provider whose gate fires on a specific file declines, letting the
5
+ next engine take it."""
6
+
7
+ from docspectra.parser.contracts import (Attempt, Declined, FileInput,
8
+ ImageAsset, MAX_ASSETS, MAX_ASSET_BYTES,
9
+ ParseResult)
10
+ from docspectra.parser.core import (
11
+ ParseDeclinedError,
12
+ Provider,
13
+ configure,
14
+ parse,
15
+ parse_path,
16
+ provider,
17
+ register,
18
+ )
19
+ from docspectra.parser.sniff import sniff, validate_declared
20
+
21
+ __all__ = [
22
+ 'Attempt',
23
+ 'Declined',
24
+ 'FileInput',
25
+ 'ImageAsset',
26
+ 'MAX_ASSETS',
27
+ 'MAX_ASSET_BYTES',
28
+ 'ParseDeclinedError',
29
+ 'ParseResult',
30
+ 'Provider',
31
+ 'configure',
32
+ 'parse',
33
+ 'parse_path',
34
+ 'provider',
35
+ 'register',
36
+ 'sniff',
37
+ 'validate_declared',
38
+ ]
@@ -0,0 +1,64 @@
1
+ """Parser contracts: result shapes and control-flow values shared by core
2
+ and providers."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from dataclasses import dataclass, field
7
+ from pathlib import Path
8
+ from typing import Literal
9
+
10
+ FileInput = bytes | str | Path
11
+
12
+
13
+ @dataclass
14
+ class ImageAsset:
15
+ """One embedded image carried faithfully, uninterpreted.
16
+
17
+ ``name`` matches the ``![](...)`` reference in ParseResult.text;
18
+ ``source`` is engine provenance ('pdf p2', 'word/media/image1.png').
19
+ """
20
+
21
+ name: str
22
+ mime: str
23
+ data: bytes
24
+ source: str
25
+
26
+
27
+ # payload-budget policy for ParseResult.images — every provider skips
28
+ # assets over these limits and records the skip count in meta
29
+ MAX_ASSETS = 50
30
+ MAX_ASSET_BYTES = 5 * 1024 * 1024
31
+
32
+
33
+ @dataclass
34
+ class Attempt:
35
+ """One provider's outcome on a file, recorded by core on success and
36
+ failure alike — the eval suite reads routing through it."""
37
+
38
+ provider: str
39
+ outcome: Literal['success', 'declined', 'error']
40
+ reason: str | None = None
41
+
42
+
43
+ @dataclass
44
+ class Declined:
45
+ """Expected control flow: this engine cannot read *this* file, or
46
+ read it into a known false-positive shape (a gate fired). Never an
47
+ exception — the chain moves to the next provider."""
48
+
49
+ reason: str
50
+
51
+
52
+ @dataclass
53
+ class ParseResult:
54
+ """The outcome of a successful parse. ``format`` is constrained —
55
+ extend the Literal, never free-form. ``meta`` is provider diagnostics
56
+ (gate hits, page counts, task ids, timings). ``provider`` and
57
+ ``attempts`` are stamped by core, not the provider."""
58
+
59
+ text: str
60
+ provider: str = '' # engine that accepted the file (core-stamped)
61
+ format: Literal['markdown'] = 'markdown'
62
+ images: list[ImageAsset] = field(default_factory=list)
63
+ attempts: list[Attempt] = field(default_factory=list)
64
+ meta: dict = field(default_factory=dict)
@@ -0,0 +1,192 @@
1
+ """Provider registry, decline chains, and the async parse entry.
2
+
3
+ Core carries zero parsing dependencies: built-in providers are in-tree
4
+ modules under ``docspectra.providers``, try-imported at init — an absent
5
+ extra is skipped silently (an installed-but-broken provider is recorded
6
+ and surfaced in config errors instead). Engines declare themselves with
7
+ :func:`provider`; chains are ordered by ``DOCSPECTRA_PARSER_CONFIG`` (a
8
+ JSON env var mapping a document type to an ordered provider list),
9
+ falling back to the derived default chain — the canonical order of
10
+ installed providers that declared the type. Declines are return values,
11
+ hard errors fall through; chain exhaustion raises
12
+ :class:`ParseDeclinedError` with every attempt aggregated."""
13
+
14
+ from __future__ import annotations
15
+
16
+ import importlib
17
+ import json
18
+ import os
19
+ from dataclasses import dataclass
20
+ from pathlib import Path
21
+ from typing import Awaitable, Callable, Sequence
22
+
23
+ from docspectra.parser.contracts import Attempt, Declined, ParseResult
24
+ from docspectra.parser.sniff import sniff, validate_declared
25
+
26
+ _BUILTIN_MODULES = ('txt', 'pdf_inspect', 'anydoc', 'mineru')
27
+ _BUILTIN_NAMES = {m.replace('_', '-') for m in _BUILTIN_MODULES}
28
+ _CONFIG_ENV = 'DOCSPECTRA_PARSER_CONFIG'
29
+
30
+
31
+ @dataclass
32
+ class Provider:
33
+ """One engine: the async parse entry plus the types it declares."""
34
+
35
+ name: str
36
+ types: tuple[str, ...]
37
+ parse: Callable[..., Awaitable[ParseResult | Declined]]
38
+
39
+
40
+ class ParseDeclinedError(Exception):
41
+ """Every provider in the chain declined or errored on this file.
42
+
43
+ ``attempts`` aggregates every outcome. A hard error raised through
44
+ the chain is chained as the cause.
45
+ """
46
+
47
+ def __init__(self, attempts: list[Attempt]):
48
+ self.attempts = attempts
49
+ detail = '; '.join(
50
+ f"{a.provider}: {a.outcome}" + (f' ({a.reason})' if a.reason else '')
51
+ for a in attempts
52
+ )
53
+ super().__init__(f'every provider declined: {detail}')
54
+
55
+
56
+ _providers: dict[str, Provider] = {} # keyed by normalized (hyphen) engine name
57
+ _builtins_loaded = False
58
+ _config: dict | None = None # effective chain config; resolved from env on first use
59
+ _import_errors: dict[str, str] = {} # provider module → failure, when installed-but-broken
60
+
61
+
62
+ def _norm(name: str) -> str:
63
+ return name.replace('_', '-')
64
+
65
+
66
+ def register(name: str, types: Sequence[str], fn) -> None:
67
+ """Programmatic registration, same surface as the decorator. Names are
68
+ normalized (hyphen↔underscore) and registration is last-write-wins."""
69
+ _providers[_norm(name)] = Provider(_norm(name), tuple(types), fn)
70
+
71
+
72
+ def provider(name: str, types: Sequence[str]):
73
+ """Declare an engine around an async ``parse(data, *, name=None)
74
+ -> ParseResult | Declined``."""
75
+
76
+ def deco(fn):
77
+ register(name, types, fn)
78
+ return fn
79
+
80
+ return deco
81
+
82
+
83
+ def ensure_builtins() -> None:
84
+ global _builtins_loaded
85
+ if _builtins_loaded:
86
+ return
87
+ _builtins_loaded = True
88
+ for module in _BUILTIN_MODULES:
89
+ try:
90
+ importlib.import_module(f'docspectra.providers.{module}')
91
+ except ImportError as e:
92
+ if not isinstance(e, ModuleNotFoundError) \
93
+ or (e.name or '').startswith('docspectra'):
94
+ _import_errors[_norm(module)] = f'{type(e).__name__}: {e}'
95
+
96
+
97
+ def configure(config: dict | None = None) -> None:
98
+ """Pin chains to an explicit ``{type: [engine, ...]}`` mapping, or
99
+ reload them from DOCSPECTRA_PARSER_CONFIG (None, validated here —
100
+ the fail-fast hook a host calls at startup)."""
101
+ global _config
102
+ ensure_builtins()
103
+ _config = _validate(config) if config is not None else _env_config()
104
+
105
+
106
+ async def parse(data: bytes, *, name: str | None = None) -> ParseResult:
107
+ """Parse one file (bytes are the contract) into a ParseResult,
108
+ walking the type's decline chain. Raises ValueError on unroutable
109
+ input, on a declared binary extension whose content fails magic
110
+ verification, or when no provider is installed; ParseDeclinedError
111
+ when every provider declines or errors."""
112
+ ensure_builtins()
113
+ validate_declared(data, name) # declared-binary gate, before any provider
114
+ doc_type = sniff(data, name)
115
+ if doc_type is None:
116
+ raise ValueError(f'cannot determine document type of {name or "input"}')
117
+ chain = _chain_for(doc_type)
118
+ if not chain:
119
+ raise ValueError(f'no provider installed for document type {doc_type!r}')
120
+ attempts: list[Attempt] = []
121
+ cause = None
122
+ for provider_name in chain:
123
+ p = _providers[provider_name]
124
+ try:
125
+ result = await p.parse(data, name=name)
126
+ except Exception as e:
127
+ attempts.append(Attempt(p.name, 'error', f'{type(e).__name__}: {e}'))
128
+ cause = cause or e
129
+ continue
130
+ if isinstance(result, Declined):
131
+ attempts.append(Attempt(p.name, 'declined', result.reason))
132
+ continue
133
+ result.provider = p.name
134
+ result.attempts = [*attempts, Attempt(p.name, 'success')]
135
+ return result
136
+ raise ParseDeclinedError(attempts) from cause
137
+
138
+
139
+ async def parse_path(path: str | Path) -> ParseResult:
140
+ """Convenience wrapper: read the file and parse it, name from the path."""
141
+ path = Path(path)
142
+ return await parse(path.read_bytes(), name=path.name)
143
+
144
+
145
+ def _chain_for(doc_type: str) -> list[str]:
146
+ config = _effective_config()
147
+ if doc_type in config:
148
+ return list(config[doc_type])
149
+ # derived default chain: canonical order of installed providers that
150
+ # declared the type — extends automatically as built-ins are added
151
+ return [p.name for p in _providers.values() if doc_type in p.types]
152
+
153
+
154
+ def _effective_config() -> dict:
155
+ global _config
156
+ if _config is None:
157
+ _config = _env_config()
158
+ return _config
159
+
160
+
161
+ def _env_config() -> dict:
162
+ raw = os.environ.get(_CONFIG_ENV)
163
+ if not raw:
164
+ return {}
165
+ try:
166
+ config = json.loads(raw)
167
+ except json.JSONDecodeError as e:
168
+ raise ValueError(f'{_CONFIG_ENV} is not valid JSON: {e}') from e
169
+ if not isinstance(config, dict):
170
+ raise ValueError(f'{_CONFIG_ENV} must be a JSON object of type → engine list')
171
+ return _validate(config)
172
+
173
+
174
+ def _validate(config: dict) -> dict:
175
+ validated = {}
176
+ for doc_type, names in config.items():
177
+ if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
178
+ raise ValueError(f'{_CONFIG_ENV}[{doc_type!r}] must be a list of engine names')
179
+ normed = []
180
+ for name in names:
181
+ key = _norm(name)
182
+ if key not in _providers:
183
+ if key in _BUILTIN_NAMES:
184
+ detail = f' ({_import_errors[key]})' if key in _import_errors else ''
185
+ raise ValueError(
186
+ f'provider {name!r} is configured but not installed{detail} — '
187
+ f'install the matching extra (docspectra[{key}])')
188
+ raise ValueError(
189
+ f'unknown parser provider {name!r}; known: {sorted(_providers)}')
190
+ normed.append(key)
191
+ validated[doc_type] = normed
192
+ return validated