openextract 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openextract-2.2.0 → openextract-2.3.0}/.github/ISSUE_TEMPLATE/bug_report.yml +1 -1
- {openextract-2.2.0 → openextract-2.3.0}/.gitignore +8 -12
- {openextract-2.2.0 → openextract-2.3.0}/CHANGELOG.md +14 -1
- {openextract-2.2.0 → openextract-2.3.0}/PKG-INFO +32 -4
- {openextract-2.2.0 → openextract-2.3.0}/README.md +31 -3
- {openextract-2.2.0 → openextract-2.3.0}/pyproject.toml +1 -2
- {openextract-2.2.0 → openextract-2.3.0}/scripts/check_package.py +8 -7
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_content.py +9 -5
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_decisions.py +88 -61
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_diagnostics.py +69 -38
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_extract.py +22 -14
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_openrouter_extractions.py +27 -16
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_providers.py +150 -133
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_schemas.py +8 -4
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/_transport.py +45 -33
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/models.py +73 -41
- {openextract-2.2.0 → openextract-2.3.0}/tests/helpers.py +8 -3
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_decisions.py +40 -36
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_extract.py +1 -5
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_gemini_documents.py +59 -9
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_models.py +1 -2
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_openrouter_extractions.py +7 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_provider_errors.py +1 -2
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_providers.py +58 -54
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_sources.py +1 -6
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_transport.py +1 -2
- {openextract-2.2.0 → openextract-2.3.0}/uv.lock +1 -1
- {openextract-2.2.0 → openextract-2.3.0}/.gitattributes +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/CODEOWNERS +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/workflows/ci.yml +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/workflows/deploy-docs.yml +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.github/workflows/release.yml +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/.python-version +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/CONTRIBUTING.md +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/LICENSE +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/SECURITY.md +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/docs/index.html +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/examples/basic.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/examples/fixtures/invoice.txt +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/__init__.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/exceptions.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/src/openextract/py.typed +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/__init__.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/conftest.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/README.md +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice-multipage-scanned-degraded.pdf +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice-multipage-scanned.pdf +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice-multipage.json +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice-multipage.pdf +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice-single-scanned.pdf +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice.jpg +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice.json +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice.pdf +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/invoice.png +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/fixtures/invoices/manifest.json +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_anthropic_json.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_chat_json.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_examples.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_live.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_mistral_ocr.py +0 -0
- {openextract-2.2.0 → openextract-2.3.0}/tests/test_package.py +0 -0
|
@@ -1,21 +1,17 @@
|
|
|
1
|
-
# Python
|
|
1
|
+
# Python
|
|
2
2
|
__pycache__/
|
|
3
3
|
*.py[oc]
|
|
4
|
+
*.egg-info
|
|
4
5
|
build/
|
|
5
6
|
dist/
|
|
6
|
-
wheels/
|
|
7
|
-
*.egg-info
|
|
8
|
-
|
|
9
|
-
# Virtual environments
|
|
10
7
|
.venv
|
|
11
|
-
.extractbench/
|
|
12
8
|
|
|
13
|
-
#
|
|
9
|
+
# Test output
|
|
10
|
+
.coverage
|
|
11
|
+
coverage.xml
|
|
12
|
+
htmlcov/
|
|
13
|
+
|
|
14
|
+
# Local files
|
|
14
15
|
.env
|
|
15
16
|
tests/test.pdf
|
|
16
17
|
tests/test.png
|
|
17
|
-
|
|
18
|
-
# Coverage
|
|
19
|
-
.coverage
|
|
20
|
-
htmlcov/
|
|
21
|
-
coverage.xml
|
|
@@ -8,6 +8,17 @@ timing when that is known.
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [2.3.0] - 2026-11-18
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- Gemini spreadsheet selection through `Model(settings={"spreadsheetMetadata": ...})`
|
|
15
|
+
for XLSX, XLS, and ODS inputs, without new dependencies or extraction arguments.
|
|
16
|
+
- Value equality, immutability, and a readable `repr` on `OpenRouterField`.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
- Simplified adapter boundaries, retry loops, header resolution, and error formatting
|
|
20
|
+
internally, with no change to the public extraction API.
|
|
21
|
+
|
|
11
22
|
## [2.2.0] - 2026-11-16
|
|
12
23
|
|
|
13
24
|
### Added
|
|
@@ -605,7 +616,9 @@ timing when that is known.
|
|
|
605
616
|
## [0.1.1] - 2025-09-10
|
|
606
617
|
- Merge pull request #12 from Mellow-Artificial-Intelligence/new-release.
|
|
607
618
|
|
|
608
|
-
[Unreleased]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v2.
|
|
619
|
+
[Unreleased]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v2.3.0...HEAD
|
|
620
|
+
[2.3.0]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v2.2.0...v2.3.0
|
|
621
|
+
[2.2.0]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v2.1.0...v2.2.0
|
|
609
622
|
[2.1.0]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v2.0.0...v2.1.0
|
|
610
623
|
[2.0.0]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v1.2.0...v2.0.0
|
|
611
624
|
[1.2.0]: https://github.com/Mellow-Artificial-Intelligence/openextract/compare/v1.1.1...v1.2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: openextract
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: Extract structured data from documents, images, audio, and video using LLMs
|
|
5
5
|
Project-URL: Homepage, https://github.com/Mellow-Artificial-Intelligence/openextract
|
|
6
6
|
Project-URL: Documentation, https://mellow-artificial-intelligence.github.io/openextract/
|
|
@@ -143,8 +143,9 @@ model that rejects an encoded input reports it as `ModelError`.
|
|
|
143
143
|
for every declared field, including those with defaults, and rejects open-ended
|
|
144
144
|
dictionaries. Use a tool-calling model with `Model(..., api="chat")` for those.
|
|
145
145
|
Tagged unions are adapted automatically.
|
|
146
|
-
- **Gemini**
|
|
147
|
-
|
|
146
|
+
- **Gemini** supports [spreadsheet selection](#gemini-spreadsheet-selection).
|
|
147
|
+
Office processing varies by format and selected content layers; see Google's
|
|
148
|
+
[document guide](https://aistudio.google.com/docs/document-processing).
|
|
148
149
|
- **Bedrock** needs `AWS_REGION` (or `AWS_DEFAULT_REGION`) unless you set `base_url`,
|
|
149
150
|
and authenticates with a [bearer token](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-use.html).
|
|
150
151
|
SigV4 signing is not built in: pass your own authenticated `client` and
|
|
@@ -180,7 +181,7 @@ invoice = extract(Invoice, model, Path("invoice.pdf"))
|
|
|
180
181
|
| `api` | `responses`, `chat`, `anthropic`, `gemini`, `ollama`, `cohere`, `bedrock`, `mistral-ocr`, `openai-decisions`, `gemini-decisions`, `openrouter-decisions`, or `openrouter-extractions`. |
|
|
181
182
|
| `base_url` | Includes the API version, such as `https://host/v1`. Built-in providers also read `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, `GOOGLE_BASE_URL`, `OPENROUTER_BASE_URL`, `GROQ_BASE_URL`, `MISTRAL_BASE_URL`, `XAI_BASE_URL`, `CEREBRAS_BASE_URL`, `HF_BASE_URL`, `COHERE_BASE_URL`, `BEDROCK_BASE_URL`, and `OLLAMA_BASE_URL`. |
|
|
182
183
|
| `api_key`, `headers` | Credentials belong here, never in the URL. Explicit values win over the environment, and your headers win over generated ones. Omit `api_key` for keyless endpoints. |
|
|
183
|
-
| `settings` | Generation parameters in the provider's own names. They go to the top level of the request, except Gemini (`generationConfig`), Ollama sampling (`settings["options"]`), and Bedrock inference (`settings["inferenceConfig"]`). Anthropic's `max_tokens` defaults to 4096. Request structure and schema fields cannot be replaced, streaming is not supported, and Decisions accept no settings. |
|
|
184
|
+
| `settings` | Generation parameters in the provider's own names. They go to the top level of the request, except Gemini (`generationConfig`, with `spreadsheetMetadata` on the input part), Ollama sampling (`settings["options"]`), and Bedrock inference (`settings["inferenceConfig"]`). Anthropic's `max_tokens` defaults to 4096. Request structure and schema fields cannot be replaced, streaming is not supported, and Decisions accept no settings. |
|
|
184
185
|
| `timeout` | Seconds allowed for each model request. Defaults to 60. |
|
|
185
186
|
| `max_retries` | Retries for transport failures and HTTP 408, 409, 429, 500, 502, 503, and 504, with exponential backoff or `Retry-After` capped at 30 seconds. Defaults to 2. Other errors and redirects fail immediately. |
|
|
186
187
|
| `validation_retries` | Retries after output that fails your schema, with feedback that omits raw values. Defaults to 1. Each attempt has its own HTTP retry budget. Refusals, truncated output, and malformed responses fail immediately. |
|
|
@@ -208,6 +209,33 @@ descriptions and are still enforced by your schema. Open-ended dictionaries,
|
|
|
208
209
|
recursive references, tuples, and `allOf` raise `ValueError` before any request:
|
|
209
210
|
use tool mode for those.
|
|
210
211
|
|
|
212
|
+
## Gemini spreadsheet selection
|
|
213
|
+
|
|
214
|
+
For XLSX, XLS, and ODS inputs on Gemini 3 and 4+ models, `spreadsheetMetadata`
|
|
215
|
+
selects a worksheet, cell range, and content layers without a local parser:
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
model = Model(
|
|
219
|
+
"google:gemini-4-hydrogen",
|
|
220
|
+
settings={
|
|
221
|
+
"spreadsheetMetadata": {
|
|
222
|
+
"sheetIndex": 2,
|
|
223
|
+
"visibleCells": "A1:H50",
|
|
224
|
+
"contentLayers": ["DATA", "FORMULAS"],
|
|
225
|
+
},
|
|
226
|
+
},
|
|
227
|
+
)
|
|
228
|
+
invoice = extract(Invoice, model, Path("financial_model.xlsx"))
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
`spreadsheetMetadata` is attached to the file part while other settings still go to
|
|
232
|
+
`generationConfig`. `sheetIndex` is 1-based (omit it to include every sheet),
|
|
233
|
+
`visibleCells` takes A1 notation, and `contentLayers` accepts `"DATA"`, `"FORMULAS"`,
|
|
234
|
+
and `"VISUAL"` (`"DATA"` and `"FORMULAS"` by default). Other input types raise
|
|
235
|
+
`ValueError` locally; Google validates selector values, model support, and full-file
|
|
236
|
+
upload limits. See Google's
|
|
237
|
+
[spreadsheet options](https://aistudio.google.com/docs/document-processing?generate-content=true#spreadsheet-options).
|
|
238
|
+
|
|
211
239
|
## Mistral document annotations
|
|
212
240
|
|
|
213
241
|
`Model(api="mistral-ocr")` sends a PDF, including scans, to Mistral's
|
|
@@ -118,8 +118,9 @@ model that rejects an encoded input reports it as `ModelError`.
|
|
|
118
118
|
for every declared field, including those with defaults, and rejects open-ended
|
|
119
119
|
dictionaries. Use a tool-calling model with `Model(..., api="chat")` for those.
|
|
120
120
|
Tagged unions are adapted automatically.
|
|
121
|
-
- **Gemini**
|
|
122
|
-
|
|
121
|
+
- **Gemini** supports [spreadsheet selection](#gemini-spreadsheet-selection).
|
|
122
|
+
Office processing varies by format and selected content layers; see Google's
|
|
123
|
+
[document guide](https://aistudio.google.com/docs/document-processing).
|
|
123
124
|
- **Bedrock** needs `AWS_REGION` (or `AWS_DEFAULT_REGION`) unless you set `base_url`,
|
|
124
125
|
and authenticates with a [bearer token](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-use.html).
|
|
125
126
|
SigV4 signing is not built in: pass your own authenticated `client` and
|
|
@@ -155,7 +156,7 @@ invoice = extract(Invoice, model, Path("invoice.pdf"))
|
|
|
155
156
|
| `api` | `responses`, `chat`, `anthropic`, `gemini`, `ollama`, `cohere`, `bedrock`, `mistral-ocr`, `openai-decisions`, `gemini-decisions`, `openrouter-decisions`, or `openrouter-extractions`. |
|
|
156
157
|
| `base_url` | Includes the API version, such as `https://host/v1`. Built-in providers also read `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, `GOOGLE_BASE_URL`, `OPENROUTER_BASE_URL`, `GROQ_BASE_URL`, `MISTRAL_BASE_URL`, `XAI_BASE_URL`, `CEREBRAS_BASE_URL`, `HF_BASE_URL`, `COHERE_BASE_URL`, `BEDROCK_BASE_URL`, and `OLLAMA_BASE_URL`. |
|
|
157
158
|
| `api_key`, `headers` | Credentials belong here, never in the URL. Explicit values win over the environment, and your headers win over generated ones. Omit `api_key` for keyless endpoints. |
|
|
158
|
-
| `settings` | Generation parameters in the provider's own names. They go to the top level of the request, except Gemini (`generationConfig`), Ollama sampling (`settings["options"]`), and Bedrock inference (`settings["inferenceConfig"]`). Anthropic's `max_tokens` defaults to 4096. Request structure and schema fields cannot be replaced, streaming is not supported, and Decisions accept no settings. |
|
|
159
|
+
| `settings` | Generation parameters in the provider's own names. They go to the top level of the request, except Gemini (`generationConfig`, with `spreadsheetMetadata` on the input part), Ollama sampling (`settings["options"]`), and Bedrock inference (`settings["inferenceConfig"]`). Anthropic's `max_tokens` defaults to 4096. Request structure and schema fields cannot be replaced, streaming is not supported, and Decisions accept no settings. |
|
|
159
160
|
| `timeout` | Seconds allowed for each model request. Defaults to 60. |
|
|
160
161
|
| `max_retries` | Retries for transport failures and HTTP 408, 409, 429, 500, 502, 503, and 504, with exponential backoff or `Retry-After` capped at 30 seconds. Defaults to 2. Other errors and redirects fail immediately. |
|
|
161
162
|
| `validation_retries` | Retries after output that fails your schema, with feedback that omits raw values. Defaults to 1. Each attempt has its own HTTP retry budget. Refusals, truncated output, and malformed responses fail immediately. |
|
|
@@ -183,6 +184,33 @@ descriptions and are still enforced by your schema. Open-ended dictionaries,
|
|
|
183
184
|
recursive references, tuples, and `allOf` raise `ValueError` before any request:
|
|
184
185
|
use tool mode for those.
|
|
185
186
|
|
|
187
|
+
## Gemini spreadsheet selection
|
|
188
|
+
|
|
189
|
+
For XLSX, XLS, and ODS inputs on Gemini 3 and 4+ models, `spreadsheetMetadata`
|
|
190
|
+
selects a worksheet, cell range, and content layers without a local parser:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
model = Model(
|
|
194
|
+
"google:gemini-4-hydrogen",
|
|
195
|
+
settings={
|
|
196
|
+
"spreadsheetMetadata": {
|
|
197
|
+
"sheetIndex": 2,
|
|
198
|
+
"visibleCells": "A1:H50",
|
|
199
|
+
"contentLayers": ["DATA", "FORMULAS"],
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
)
|
|
203
|
+
invoice = extract(Invoice, model, Path("financial_model.xlsx"))
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
`spreadsheetMetadata` is attached to the file part while other settings still go to
|
|
207
|
+
`generationConfig`. `sheetIndex` is 1-based (omit it to include every sheet),
|
|
208
|
+
`visibleCells` takes A1 notation, and `contentLayers` accepts `"DATA"`, `"FORMULAS"`,
|
|
209
|
+
and `"VISUAL"` (`"DATA"` and `"FORMULAS"` by default). Other input types raise
|
|
210
|
+
`ValueError` locally; Google validates selector values, model support, and full-file
|
|
211
|
+
upload limits. See Google's
|
|
212
|
+
[spreadsheet options](https://aistudio.google.com/docs/document-processing?generate-content=true#spreadsheet-options).
|
|
213
|
+
|
|
186
214
|
## Mistral document annotations
|
|
187
215
|
|
|
188
216
|
`Model(api="mistral-ocr")` sends a PDF, including scans, to Mistral's
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "openextract"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.3.0"
|
|
4
4
|
description = "Extract structured data from documents, images, audio, and video using LLMs"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -46,7 +46,6 @@ dev = [
|
|
|
46
46
|
"ty>=0.0.35",
|
|
47
47
|
]
|
|
48
48
|
|
|
49
|
-
[tool.ty]
|
|
50
49
|
# Type-check library code only; tests intentionally use invalid types for error paths.
|
|
51
50
|
[tool.ty.src]
|
|
52
51
|
include = ["src/openextract"]
|
|
@@ -37,13 +37,14 @@ INSTALLER_FILES = {
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def installed_bytes(dist: importlib.metadata.Distribution) -> int:
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
40
|
+
total = 0
|
|
41
|
+
for file in dist.files or []:
|
|
42
|
+
if file.suffix == ".pyc" or file.name in INSTALLER_FILES:
|
|
43
|
+
continue
|
|
44
|
+
located = Path(dist.locate_file(file))
|
|
45
|
+
if located.is_file():
|
|
46
|
+
total += located.stat().st_size
|
|
47
|
+
return total
|
|
47
48
|
|
|
48
49
|
|
|
49
50
|
def normalize(name: str) -> str:
|
|
@@ -7,6 +7,7 @@ from collections.abc import Generator
|
|
|
7
7
|
from contextlib import contextmanager
|
|
8
8
|
from dataclasses import dataclass
|
|
9
9
|
from pathlib import Path
|
|
10
|
+
from typing import TypeGuard
|
|
10
11
|
|
|
11
12
|
import httpx
|
|
12
13
|
|
|
@@ -84,7 +85,7 @@ def load(source: Source, media_type: str | None) -> Content:
|
|
|
84
85
|
"""Read text, a URL, bytes, or a path. An explicit ``media_type`` always wins."""
|
|
85
86
|
if _is_url(source):
|
|
86
87
|
with _fetch_errors():
|
|
87
|
-
response = httpx.get(
|
|
88
|
+
response = httpx.get(source, follow_redirects=True, timeout=_URL_TIMEOUT_SECONDS)
|
|
88
89
|
return _downloaded(response, media_type)
|
|
89
90
|
if isinstance(source, bytes):
|
|
90
91
|
return _content(source, media_type)
|
|
@@ -100,10 +101,10 @@ async def load_async(source: Source, media_type: str | None) -> Content:
|
|
|
100
101
|
return load(source, media_type)
|
|
101
102
|
with _fetch_errors():
|
|
102
103
|
async with httpx.AsyncClient(follow_redirects=True, timeout=_URL_TIMEOUT_SECONDS) as client:
|
|
103
|
-
return _downloaded(await client.get(
|
|
104
|
+
return _downloaded(await client.get(source), media_type)
|
|
104
105
|
|
|
105
106
|
|
|
106
|
-
def _is_url(source: Source) ->
|
|
107
|
+
def _is_url(source: Source) -> TypeGuard[str]:
|
|
107
108
|
return isinstance(source, str) and source.startswith(("http://", "https://"))
|
|
108
109
|
|
|
109
110
|
|
|
@@ -122,13 +123,16 @@ def _content(
|
|
|
122
123
|
data: bytes, media_type: str | None, *, name: str = "", header: str | None = None
|
|
123
124
|
) -> Content:
|
|
124
125
|
"""Prefer the explicit type, then the file extension, the header, and the leading bytes."""
|
|
125
|
-
|
|
126
|
-
resolved = media_type or extension or header or _sniff(data)
|
|
126
|
+
resolved = media_type or _type_from_name(name) or header or _sniff(data)
|
|
127
127
|
if resolved is None:
|
|
128
128
|
raise ValueError("Could not infer the media type; pass media_type explicitly.")
|
|
129
129
|
return Content(data, resolved)
|
|
130
130
|
|
|
131
131
|
|
|
132
|
+
def _type_from_name(name: str) -> str | None:
|
|
133
|
+
return _DOCUMENT_TYPES.get(Path(name).suffix.lower()) or mimetypes.guess_type(name)[0]
|
|
134
|
+
|
|
135
|
+
|
|
132
136
|
def _sniff(data: bytes) -> str | None:
|
|
133
137
|
"""Guess a media type from the content, falling back to plain text for UTF-8."""
|
|
134
138
|
for signature, media_type in _SIGNATURES.items():
|
|
@@ -7,14 +7,12 @@ answer format; OpenAI uses lists and its own answer types.
|
|
|
7
7
|
import re
|
|
8
8
|
import typing
|
|
9
9
|
from enum import Enum
|
|
10
|
-
from typing import Any, Literal
|
|
10
|
+
from typing import Any, Literal, TypeGuard
|
|
11
11
|
|
|
12
|
-
from pydantic import BaseModel
|
|
12
|
+
from pydantic import BaseModel
|
|
13
13
|
from pydantic.fields import FieldInfo
|
|
14
14
|
|
|
15
15
|
from ._content import PROMPT, Content
|
|
16
|
-
from ._diagnostics import invalid_response_message, validation_message
|
|
17
|
-
from .exceptions import SchemaValidationError
|
|
18
16
|
from .models import ResolvedModel
|
|
19
17
|
|
|
20
18
|
_YES_THRESHOLD = 0.5
|
|
@@ -33,42 +31,38 @@ def request(
|
|
|
33
31
|
|
|
34
32
|
|
|
35
33
|
def parse[T: BaseModel](schema: type[T], model: ResolvedModel, payload: dict[str, Any]) -> T:
|
|
36
|
-
"""Validate one answer per field.
|
|
34
|
+
"""Validate one answer per schema field."""
|
|
37
35
|
is_openai = model.api == "openai-decisions"
|
|
38
|
-
yes_type, probability = ("predicate", "probability") if is_openai else ("noul", "noul")
|
|
39
36
|
fields = schema.model_fields
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
except ValidationError as exc:
|
|
52
|
-
raise SchemaValidationError(validation_message(exc, 1)) from exc
|
|
53
|
-
except (LookupError, TypeError, ValueError, AttributeError) as exc:
|
|
54
|
-
message = invalid_response_message(exc, context="decision answers")
|
|
55
|
-
raise SchemaValidationError(message) from exc
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def _openrouter(
|
|
37
|
+
answers = _answer_map(payload["answers"], is_openai=is_openai)
|
|
38
|
+
if answers.keys() != fields.keys():
|
|
39
|
+
raise ValueError("Expected exactly one answer for every schema field.")
|
|
40
|
+
values = {
|
|
41
|
+
name: _parse_answer(name, fields[name], answer, is_openai=is_openai)
|
|
42
|
+
for name, answer in answers.items()
|
|
43
|
+
}
|
|
44
|
+
return schema.model_validate(values, by_name=True)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _build_openrouter(
|
|
59
48
|
schema: type[BaseModel], content: Content, instructions: str | None
|
|
60
49
|
) -> dict[str, Any]:
|
|
61
50
|
_require_text_or_image(content)
|
|
62
|
-
|
|
63
|
-
|
|
51
|
+
document: str | dict[str, Any] = (
|
|
52
|
+
content.text
|
|
53
|
+
if content.is_text
|
|
54
|
+
else {"type": "image_url", "image_url": {"url": content.data_uri}}
|
|
55
|
+
)
|
|
64
56
|
return {
|
|
65
|
-
"state": [*
|
|
66
|
-
"questions":
|
|
57
|
+
"state": [*_guidance(instructions), document],
|
|
58
|
+
"questions": _named_questions(schema),
|
|
67
59
|
}
|
|
68
60
|
|
|
69
61
|
|
|
70
|
-
def
|
|
71
|
-
|
|
62
|
+
def _build_gemini(
|
|
63
|
+
schema: type[BaseModel], content: Content, instructions: str | None
|
|
64
|
+
) -> dict[str, Any]:
|
|
65
|
+
questions = _named_questions(schema)
|
|
72
66
|
_check_gemini_limits(questions)
|
|
73
67
|
if content.is_text:
|
|
74
68
|
document = {"type": "text", "text": content.text}
|
|
@@ -79,27 +73,22 @@ def _gemini(schema: type[BaseModel], content: Content, instructions: str | None)
|
|
|
79
73
|
raise ValueError(
|
|
80
74
|
f"Gemini Decisions supports text, images and PDF, not {content.media_type}."
|
|
81
75
|
)
|
|
82
|
-
|
|
83
|
-
texts = [{"type": "text", "text": text + "\n"} for text in guidance]
|
|
76
|
+
texts = [{"type": "text", "text": text + "\n"} for text in _guidance(instructions)]
|
|
84
77
|
return {"input": [*texts, document], "questions": questions}
|
|
85
78
|
|
|
86
79
|
|
|
87
|
-
def
|
|
80
|
+
def _build_openai(
|
|
81
|
+
schema: type[BaseModel], content: Content, instructions: str | None
|
|
82
|
+
) -> dict[str, Any]:
|
|
88
83
|
if not 1 <= len(schema.model_fields) <= 400:
|
|
89
84
|
raise ValueError("OpenAI Decisions requires between 1 and 400 schema fields.")
|
|
90
|
-
questions = []
|
|
91
|
-
for name, field in schema.model_fields.items():
|
|
92
|
-
question = {"name": name, "instructions": _instructions(name, field), "type": "predicate"}
|
|
93
|
-
if field.annotation is not bool:
|
|
94
|
-
options = _options(name, field.annotation)
|
|
95
|
-
question["type"] = "choice"
|
|
96
|
-
question["choices"] = [{"value": key, "description": key} for key in options]
|
|
97
|
-
questions.append(question)
|
|
85
|
+
questions = [_openai_question(name, field) for name, field in schema.model_fields.items()]
|
|
98
86
|
_require_text_or_image(content)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
87
|
+
document = (
|
|
88
|
+
{"type": "input_text", "text": content.text}
|
|
89
|
+
if content.is_text
|
|
90
|
+
else {"type": "input_image", "image_url": content.data_uri}
|
|
91
|
+
)
|
|
103
92
|
return {
|
|
104
93
|
"input": [
|
|
105
94
|
{"role": "developer", "content": instructions or PROMPT},
|
|
@@ -124,15 +113,32 @@ def _check_gemini_limits(questions: dict[str, dict[str, Any]]) -> None:
|
|
|
124
113
|
raise ValueError("Invalid Gemini Decisions option key.")
|
|
125
114
|
|
|
126
115
|
|
|
127
|
-
def
|
|
116
|
+
def _named_questions(schema: type[BaseModel]) -> dict[str, dict[str, Any]]:
|
|
117
|
+
return {name: _named_question(name, field) for name, field in schema.model_fields.items()}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _guidance(instructions: str | None) -> list[str]:
|
|
121
|
+
return [PROMPT, instructions] if instructions else [PROMPT]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _named_question(name: str, field: FieldInfo) -> dict[str, Any]:
|
|
128
125
|
"""Phrase a field in the named format OpenRouter and Gemini share."""
|
|
126
|
+
instructions = _field_instructions(name, field)
|
|
129
127
|
if field.annotation is bool:
|
|
130
|
-
return {"type": "noul", "instructions":
|
|
128
|
+
return {"type": "noul", "instructions": instructions}
|
|
131
129
|
criteria = {key: key for key in _options(name, field.annotation)}
|
|
132
|
-
return {"type": "choice", "instructions":
|
|
130
|
+
return {"type": "choice", "instructions": instructions, "criteria": criteria}
|
|
131
|
+
|
|
133
132
|
|
|
133
|
+
def _openai_question(name: str, field: FieldInfo) -> dict[str, Any]:
|
|
134
|
+
instructions = _field_instructions(name, field)
|
|
135
|
+
if field.annotation is bool:
|
|
136
|
+
return {"name": name, "type": "predicate", "instructions": instructions}
|
|
137
|
+
choices = [{"value": key, "description": key} for key in _options(name, field.annotation)]
|
|
138
|
+
return {"name": name, "type": "choice", "instructions": instructions, "choices": choices}
|
|
134
139
|
|
|
135
|
-
|
|
140
|
+
|
|
141
|
+
def _field_instructions(name: str, field: FieldInfo) -> str:
|
|
136
142
|
return field.description or name.replace("_", " ")
|
|
137
143
|
|
|
138
144
|
|
|
@@ -148,11 +154,13 @@ def _options(name: str, annotation: object) -> dict[str, object]:
|
|
|
148
154
|
|
|
149
155
|
|
|
150
156
|
def _is_gemini_option(key: str) -> bool:
|
|
151
|
-
"""Google allows 1–128 trimmed characters in at most 512 bytes
|
|
152
|
-
|
|
153
|
-
|
|
157
|
+
"""Google allows 1–128 trimmed printable characters in at most 512 UTF-8 bytes."""
|
|
158
|
+
return (
|
|
159
|
+
1 <= len(key) <= 128
|
|
160
|
+
and len(key.encode()) <= 512
|
|
161
|
+
and key == key.strip()
|
|
162
|
+
and key.isprintable()
|
|
154
163
|
)
|
|
155
|
-
return 1 <= len(key) <= 128 and len(key.encode()) <= 512 and key == key.strip() and not control
|
|
156
164
|
|
|
157
165
|
|
|
158
166
|
def _require_text_or_image(content: Content) -> None:
|
|
@@ -160,15 +168,30 @@ def _require_text_or_image(content: Content) -> None:
|
|
|
160
168
|
raise ValueError(f"Decisions models read text and images, not {content.media_type}.")
|
|
161
169
|
|
|
162
170
|
|
|
163
|
-
def
|
|
164
|
-
|
|
171
|
+
def _answer_map(raw_answers: Any, *, is_openai: bool) -> dict[str, dict[str, Any]]:
|
|
172
|
+
"""Normalize OpenAI's answer list or OpenRouter/Gemini's answer map into a field map."""
|
|
173
|
+
if not is_openai:
|
|
174
|
+
if not isinstance(raw_answers, dict):
|
|
175
|
+
raise TypeError("Expected an object of decision answers.")
|
|
176
|
+
return raw_answers
|
|
177
|
+
if not isinstance(raw_answers, list):
|
|
178
|
+
raise TypeError("Expected a list of decision answers.")
|
|
179
|
+
answers = {item["name"]: item for item in raw_answers}
|
|
180
|
+
if len(answers) != len(raw_answers):
|
|
181
|
+
raise ValueError("Expected at most one answer per schema field.")
|
|
182
|
+
return answers
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _parse_answer(
|
|
186
|
+
name: str, field: FieldInfo, answer: dict[str, Any], *, is_openai: bool
|
|
165
187
|
) -> object:
|
|
166
188
|
"""Turn an answer into the field's value; yes/no is true from a probability of one half."""
|
|
167
189
|
if answer["type"] == "refusal":
|
|
168
190
|
raise ValueError(f"The model refused to answer {name!r}.")
|
|
169
191
|
if field.annotation is bool:
|
|
170
|
-
|
|
171
|
-
|
|
192
|
+
yes_type, prob_key = ("predicate", "probability") if is_openai else ("noul", "noul")
|
|
193
|
+
chance = answer[prob_key]
|
|
194
|
+
if not (answer["type"] == yes_type and _is_probability(chance)):
|
|
172
195
|
raise ValueError(
|
|
173
196
|
f"Expected a yes/no answer with a numeric probability from 0 to 1 for {name!r}."
|
|
174
197
|
)
|
|
@@ -181,8 +204,12 @@ def _value(
|
|
|
181
204
|
return options[answer["choice"]]
|
|
182
205
|
|
|
183
206
|
|
|
207
|
+
def _is_probability(value: object) -> TypeGuard[int | float]:
|
|
208
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool) and 0 <= value <= 1
|
|
209
|
+
|
|
210
|
+
|
|
184
211
|
_BUILDERS = {
|
|
185
|
-
"openai-decisions":
|
|
186
|
-
"gemini-decisions":
|
|
187
|
-
"openrouter-decisions":
|
|
212
|
+
"openai-decisions": _build_openai,
|
|
213
|
+
"gemini-decisions": _build_gemini,
|
|
214
|
+
"openrouter-decisions": _build_openrouter,
|
|
188
215
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from collections.abc import Iterable
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any, NamedTuple
|
|
6
6
|
|
|
7
7
|
import httpx
|
|
8
8
|
from pydantic import ValidationError
|
|
@@ -85,55 +85,81 @@ def request_error(error: httpx.RequestError, attempts: int) -> ModelError:
|
|
|
85
85
|
)
|
|
86
86
|
|
|
87
87
|
|
|
88
|
+
class _ErrorDetails(NamedTuple):
|
|
89
|
+
provider: str | None
|
|
90
|
+
code: str | int | None
|
|
91
|
+
finish_reason: str | None
|
|
92
|
+
native_finish_reason: str | None
|
|
93
|
+
message: str | None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def has_error_envelope(payload: dict[str, Any]) -> bool:
|
|
97
|
+
"""Whether a 2xx response body carries a provider or generation error."""
|
|
98
|
+
if payload.get("error") is not None:
|
|
99
|
+
return True
|
|
100
|
+
choice = _first_choice(payload)
|
|
101
|
+
return choice.get("error") is not None or choice.get("finish_reason") == "error"
|
|
102
|
+
|
|
103
|
+
|
|
88
104
|
def provider_error(
|
|
89
105
|
payload: dict[str, Any], status_code: int, secrets: Iterable[str] = ()
|
|
90
|
-
) -> ModelError
|
|
91
|
-
"""
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
return None
|
|
102
|
-
|
|
103
|
-
fields = _object(error)
|
|
104
|
-
metadata = _object(fields.get("metadata"))
|
|
105
|
-
provider = _text(metadata.get("provider_name")) or _text(payload.get("provider"))
|
|
106
|
-
code = fields.get("code", fields.get("type"))
|
|
107
|
-
if type(code) not in (str, int):
|
|
108
|
-
code = None
|
|
109
|
-
message = _text(fields.get("message")) or _text(error) or _text(payload.get("message"))
|
|
110
|
-
if upstream := _upstream_message(metadata.get("raw")):
|
|
111
|
-
message = f"{message}: {upstream}" if message else upstream
|
|
106
|
+
) -> ModelError:
|
|
107
|
+
"""Format an HTTP failure or an error envelope with redacted credentials."""
|
|
108
|
+
raw = _extract_error_details(payload)
|
|
109
|
+
next_step = _next_step(status_code, raw.code, raw.native_finish_reason)
|
|
110
|
+
ordered_secrets = sorted({secret for secret in secrets if secret}, key=len, reverse=True)
|
|
111
|
+
|
|
112
|
+
provider = _redact(raw.provider, ordered_secrets)
|
|
113
|
+
code = _redact(raw.code, ordered_secrets) if isinstance(raw.code, str) else raw.code
|
|
114
|
+
finish_reason = _redact(raw.finish_reason, ordered_secrets)
|
|
115
|
+
native_finish_reason = _redact(raw.native_finish_reason, ordered_secrets)
|
|
116
|
+
details = _redact(raw.message, ordered_secrets)
|
|
112
117
|
|
|
113
|
-
secrets = sorted({secret for secret in secrets if secret}, key=len, reverse=True)
|
|
114
118
|
facts = {
|
|
115
119
|
"Provider": provider,
|
|
116
120
|
"Code": code,
|
|
117
|
-
"Finish reason":
|
|
118
|
-
"Native finish reason":
|
|
119
|
-
"Details":
|
|
121
|
+
"Finish reason": finish_reason,
|
|
122
|
+
"Native finish reason": native_finish_reason,
|
|
123
|
+
"Details": details,
|
|
120
124
|
}
|
|
125
|
+
succeeded = 200 <= status_code < 300
|
|
126
|
+
heading = (
|
|
127
|
+
"Model provider failed." if succeeded else f"Model provider returned HTTP {status_code}."
|
|
128
|
+
)
|
|
121
129
|
lines = [
|
|
122
|
-
|
|
123
|
-
*(
|
|
124
|
-
|
|
125
|
-
for label, fact in facts.items()
|
|
126
|
-
if fact is not None
|
|
127
|
-
),
|
|
128
|
-
"Next step: " + _next_step(status_code, code, native),
|
|
130
|
+
heading,
|
|
131
|
+
*(f"{label}: {value}" for label, value in facts.items() if value is not None),
|
|
132
|
+
f"Next step: {next_step}",
|
|
129
133
|
]
|
|
130
134
|
return ModelError(
|
|
131
135
|
"\n".join(lines),
|
|
132
136
|
status_code=status_code,
|
|
133
|
-
code=
|
|
134
|
-
provider=
|
|
135
|
-
finish_reason=
|
|
136
|
-
native_finish_reason=
|
|
137
|
+
code=code,
|
|
138
|
+
provider=provider,
|
|
139
|
+
finish_reason=finish_reason,
|
|
140
|
+
native_finish_reason=native_finish_reason,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _extract_error_details(payload: dict[str, Any]) -> _ErrorDetails:
|
|
145
|
+
choice = _first_choice(payload)
|
|
146
|
+
error = payload.get("error")
|
|
147
|
+
if error is None:
|
|
148
|
+
error = choice.get("error")
|
|
149
|
+
fields = _object(error)
|
|
150
|
+
metadata = _object(fields.get("metadata"))
|
|
151
|
+
provider = _text(metadata.get("provider_name")) or _text(payload.get("provider"))
|
|
152
|
+
raw_code = fields.get("code", fields.get("type"))
|
|
153
|
+
code = raw_code if isinstance(raw_code, (str, int)) and not isinstance(raw_code, bool) else None
|
|
154
|
+
message = _text(fields.get("message")) or _text(error) or _text(payload.get("message"))
|
|
155
|
+
if upstream := _upstream_message(metadata.get("raw")):
|
|
156
|
+
message = f"{message}: {upstream}" if message else upstream
|
|
157
|
+
return _ErrorDetails(
|
|
158
|
+
provider=provider,
|
|
159
|
+
code=code,
|
|
160
|
+
finish_reason=_text(choice.get("finish_reason")),
|
|
161
|
+
native_finish_reason=_text(choice.get("native_finish_reason")),
|
|
162
|
+
message=message,
|
|
137
163
|
)
|
|
138
164
|
|
|
139
165
|
|
|
@@ -187,6 +213,11 @@ def _count(number: int, noun: str) -> str:
|
|
|
187
213
|
return f"{number} {noun}" if number == 1 else f"{number} {noun}s"
|
|
188
214
|
|
|
189
215
|
|
|
216
|
+
def _first_choice(payload: dict[str, Any]) -> dict[str, Any]:
|
|
217
|
+
choices = payload.get("choices")
|
|
218
|
+
return _object(choices[0]) if isinstance(choices, list) and choices else {}
|
|
219
|
+
|
|
220
|
+
|
|
190
221
|
def _object(value: object) -> dict[str, Any]:
|
|
191
222
|
return value if isinstance(value, dict) else {}
|
|
192
223
|
|