structx 0.4.10__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {structx-0.4.10/structx.egg-info → structx-0.5.0}/PKG-INFO +53 -95
- {structx-0.4.10 → structx-0.5.0}/README.md +47 -63
- {structx-0.4.10 → structx-0.5.0}/pyproject.toml +36 -34
- {structx-0.4.10 → structx-0.5.0}/structx/__init__.py +1 -1
- {structx-0.4.10 → structx-0.5.0}/structx/core/config.py +3 -4
- {structx-0.4.10 → structx-0.5.0}/structx/core/models.py +5 -7
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/engines/extraction_engine.py +1 -1
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/extractor.py +21 -4
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/processors/__init__.py +2 -1
- structx-0.5.0/structx/extraction/processors/content_analyzer.py +164 -0
- structx-0.5.0/structx/extraction/processors/data_processor.py +175 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/processors/model_operations.py +1 -1
- structx-0.5.0/structx/utils/file_reader.py +206 -0
- {structx-0.4.10 → structx-0.5.0/structx.egg-info}/PKG-INFO +53 -95
- {structx-0.4.10 → structx-0.5.0}/structx.egg-info/SOURCES.txt +10 -2
- structx-0.5.0/structx.egg-info/requires.txt +31 -0
- structx-0.5.0/tests/test_config_and_model_utils.py +66 -0
- structx-0.5.0/tests/test_content_analyzer.py +80 -0
- structx-0.5.0/tests/test_data_processor.py +129 -0
- structx-0.5.0/tests/test_document_conversion_integration.py +52 -0
- structx-0.5.0/tests/test_extractor_async.py +109 -0
- structx-0.5.0/tests/test_file_reader.py +141 -0
- structx-0.5.0/tests/test_result_manager_and_helpers.py +78 -0
- structx-0.4.10/structx/extraction/processors/data_content.py +0 -367
- structx-0.4.10/structx/utils/file_reader.py +0 -298
- structx-0.4.10/structx.egg-info/requires.txt +0 -62
- {structx-0.4.10 → structx-0.5.0}/LICENSE +0 -0
- {structx-0.4.10 → structx-0.5.0}/setup.cfg +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/core/__init__.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/core/exceptions.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/__init__.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/core/__init__.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/core/llm_core.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/core/model_utils.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/engines/__init__.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/generator.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/extraction/result_manager.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/__init__.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/constants.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/helpers.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/prompts.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/types.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx/utils/usage.py +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx.egg-info/dependency_links.txt +0 -0
- {structx-0.4.10 → structx-0.5.0}/structx.egg-info/top_level.txt +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: structx
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
|
-
Author-email: blacksuan19 <py@
|
|
5
|
+
Author-email: blacksuan19 <py@aolabs.dev>
|
|
6
6
|
Classifier: Development Status :: 5 - Production/Stable
|
|
7
7
|
Classifier: Framework :: AsyncIO
|
|
8
8
|
Classifier: Framework :: Pydantic
|
|
@@ -43,42 +43,16 @@ Provides-Extra: dev
|
|
|
43
43
|
Requires-Dist: black; extra == "dev"
|
|
44
44
|
Requires-Dist: bumpver; extra == "dev"
|
|
45
45
|
Requires-Dist: pip-tools; extra == "dev"
|
|
46
|
+
Requires-Dist: pytest; extra == "dev"
|
|
46
47
|
Requires-Dist: wheel; extra == "dev"
|
|
47
|
-
Provides-Extra: pdf
|
|
48
|
-
Requires-Dist: pypdf2; extra == "pdf"
|
|
49
|
-
Requires-Dist: instructor[multimodal]; extra == "pdf"
|
|
50
|
-
Requires-Dist: weasyprint; extra == "pdf"
|
|
51
|
-
Requires-Dist: markdown; extra == "pdf"
|
|
52
|
-
Provides-Extra: docx
|
|
53
|
-
Requires-Dist: python-docx; extra == "docx"
|
|
54
|
-
Requires-Dist: docling; extra == "docx"
|
|
55
|
-
Requires-Dist: markdown; extra == "docx"
|
|
56
|
-
Requires-Dist: weasyprint; extra == "docx"
|
|
57
48
|
Provides-Extra: docs
|
|
58
|
-
Requires-Dist:
|
|
59
|
-
Requires-Dist:
|
|
60
|
-
Requires-Dist:
|
|
61
|
-
Requires-Dist: docling; extra == "docs"
|
|
49
|
+
Requires-Dist: docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]; extra == "docs"
|
|
50
|
+
Requires-Dist: torch; extra == "docs"
|
|
51
|
+
Requires-Dist: torchvision; extra == "docs"
|
|
62
52
|
Requires-Dist: weasyprint; extra == "docs"
|
|
63
|
-
Requires-Dist: markdown; extra == "docs"
|
|
64
|
-
Provides-Extra: unstructured
|
|
65
|
-
Requires-Dist: unstructured[all-docs]; extra == "unstructured"
|
|
66
|
-
Provides-Extra: docling
|
|
67
|
-
Requires-Dist: docling; extra == "docling"
|
|
68
|
-
Requires-Dist: markdown; extra == "docling"
|
|
69
|
-
Requires-Dist: weasyprint; extra == "docling"
|
|
70
|
-
Provides-Extra: advanced
|
|
71
|
-
Requires-Dist: pypdf2; extra == "advanced"
|
|
72
|
-
Requires-Dist: python-docx; extra == "advanced"
|
|
73
|
-
Requires-Dist: instructor[multimodal]; extra == "advanced"
|
|
74
|
-
Requires-Dist: docling; extra == "advanced"
|
|
75
|
-
Requires-Dist: weasyprint; extra == "advanced"
|
|
76
|
-
Requires-Dist: markdown; extra == "advanced"
|
|
77
|
-
Requires-Dist: unstructured[all-docs]; extra == "advanced"
|
|
78
53
|
Provides-Extra: mkdocs
|
|
79
54
|
Requires-Dist: mkdocs; extra == "mkdocs"
|
|
80
55
|
Requires-Dist: mkdocs-material; extra == "mkdocs"
|
|
81
|
-
Requires-Dist: mkdocstrings[markdown]; extra == "mkdocs"
|
|
82
56
|
Requires-Dist: mkdocstrings[python]; extra == "mkdocs"
|
|
83
57
|
Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == "mkdocs"
|
|
84
58
|
Requires-Dist: mkdocs-material[imaging]; extra == "mkdocs"
|
|
@@ -90,14 +64,14 @@ Dynamic: license-file
|
|
|
90
64
|
Advanced structured data extraction from any document using LLMs with multimodal
|
|
91
65
|
support.
|
|
92
66
|
|
|
93
|
-
[](https://structx.
|
|
94
|
-
[](https://structx.aolabs.dev "Documentation")
|
|
68
|
+
[](https://pypi.org/project/structx "Package")
|
|
95
69
|
[](# "Build with GitHub Actions")
|
|
96
70
|
|
|
97
|
-
`structx` is a powerful Python library for extracting structured data from
|
|
98
|
-
|
|
99
|
-
multimodal PDF
|
|
100
|
-
|
|
71
|
+
`structx` is a powerful Python library for extracting structured data from text,
|
|
72
|
+
tables, and documents using Large Language Models (LLMs). With the optional
|
|
73
|
+
`docs` extra, it provides a multimodal PDF pipeline that passes PDFs directly to
|
|
74
|
+
vision-capable models and converts other document formats to PDF first.
|
|
101
75
|
|
|
102
76
|
## 🔔 Package rename notice (PyPI)
|
|
103
77
|
|
|
@@ -105,7 +79,7 @@ The PyPI distribution has been renamed from `structx-llm` to `structx`
|
|
|
105
79
|
(September 2025).
|
|
106
80
|
|
|
107
81
|
- Imports are unchanged: continue using `import structx`
|
|
108
|
-
-
|
|
82
|
+
- Document processing now lives in the optional `docs` extra
|
|
109
83
|
- Please update your environments and requirement files to use the new name
|
|
110
84
|
|
|
111
85
|
Upgrade commands:
|
|
@@ -116,20 +90,20 @@ pip install -U structx
|
|
|
116
90
|
```
|
|
117
91
|
|
|
118
92
|
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
119
|
-
with `structx`.
|
|
93
|
+
with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
120
94
|
|
|
121
95
|
## ✨ Key Features
|
|
122
96
|
|
|
123
97
|
### 🎯 **Advanced Document Processing**
|
|
124
98
|
|
|
125
|
-
- **� Multimodal PDF Pipeline**:
|
|
126
|
-
|
|
99
|
+
- **� Multimodal PDF Pipeline**: Passes PDFs directly to vision-capable models
|
|
100
|
+
and converts supported non-PDF documents to PDF
|
|
127
101
|
- **🖼️ Vision-Enabled Extraction**: Native instructor multimodal support for
|
|
128
102
|
PDFs and images
|
|
129
103
|
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
130
104
|
results
|
|
131
|
-
- **📊
|
|
132
|
-
Markdown, and more
|
|
105
|
+
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet in the base install,
|
|
106
|
+
with PDF, DOCX, TXT, Markdown, and more via `structx[docs]`
|
|
133
107
|
|
|
134
108
|
### 🚀 **Intelligent Data Extraction**
|
|
135
109
|
|
|
@@ -153,41 +127,21 @@ with `structx`.
|
|
|
153
127
|
## Installation
|
|
154
128
|
|
|
155
129
|
```bash
|
|
156
|
-
# Core package with basic extraction capabilities
|
|
157
130
|
pip install structx
|
|
158
131
|
```
|
|
159
132
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
For the best experience with all document types including advanced multimodal
|
|
163
|
-
PDF processing:
|
|
133
|
+
For document and multimodal PDF support:
|
|
164
134
|
|
|
165
135
|
```bash
|
|
166
|
-
|
|
167
|
-
pip install structx[docs]
|
|
168
|
-
|
|
169
|
-
# Individual components
|
|
170
|
-
pip install structx[pdf] # PDF processing with multimodal support
|
|
171
|
-
pip install structx[docx] # Advanced DOCX conversion via docling
|
|
136
|
+
pip install "structx[docs]"
|
|
172
137
|
```
|
|
173
138
|
|
|
174
|
-
### 🔧 What
|
|
175
|
-
|
|
176
|
-
- **`[docs]`**: Complete multimodal document processing pipeline
|
|
177
|
-
- PDF conversion from any document type
|
|
178
|
-
- Instructor multimodal vision support
|
|
179
|
-
- Advanced DOCX processing via docling
|
|
180
|
-
- Enhanced extraction quality
|
|
181
|
-
- **`[pdf]`**: PDF-specific processing
|
|
139
|
+
### 🔧 What The Package Provides
|
|
182
140
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
- **`[docx]`**: Advanced DOCX support
|
|
188
|
-
- Document conversion via docling
|
|
189
|
-
- Structure preservation
|
|
190
|
-
- Markdown-based processing pipeline
|
|
141
|
+
- Structured readers for CSV, Excel, JSON, Parquet, and Feather
|
|
142
|
+
- Instructor multimodal vision support
|
|
143
|
+
- Optional Docling document parsing with CPU-only PyTorch resolution for uv on Linux
|
|
144
|
+
- Optional WeasyPrint PDF rendering for non-PDF document formats
|
|
191
145
|
|
|
192
146
|
## Quick Start
|
|
193
147
|
|
|
@@ -218,16 +172,19 @@ print(result.data[0].model_dump_json(indent=2))
|
|
|
218
172
|
|
|
219
173
|
### 📄 Document Processing with Multimodal Support
|
|
220
174
|
|
|
175
|
+
Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
|
|
176
|
+
be passed directly through the multimodal path.
|
|
177
|
+
|
|
221
178
|
```python
|
|
222
|
-
# Process a PDF invoice
|
|
179
|
+
# Process a PDF invoice through the multimodal pipeline
|
|
223
180
|
result = extractor.extract(
|
|
224
|
-
data="scripts/example_input/S0305SampleInvoice.pdf",
|
|
181
|
+
data="scripts/example_input/S0305SampleInvoice.pdf",
|
|
225
182
|
query="extract the invoice number, total amount, and line items"
|
|
226
183
|
)
|
|
227
184
|
|
|
228
185
|
# Convert a DOCX contract and process with multimodal support
|
|
229
186
|
result = extractor.extract(
|
|
230
|
-
data="scripts/example_input/free-consultancy-agreement.docx",
|
|
187
|
+
data="scripts/example_input/free-consultancy-agreement.docx",
|
|
231
188
|
query="extract parties, effective date, and payment terms"
|
|
232
189
|
)
|
|
233
190
|
```
|
|
@@ -252,28 +209,28 @@ traditional text-based extraction:
|
|
|
252
209
|
complex layouts
|
|
253
210
|
- **🔄 No Chunking Issues**: Eliminates problems with information split across
|
|
254
211
|
chunks
|
|
255
|
-
- **📊 Universal Format**:
|
|
256
|
-
conversion
|
|
212
|
+
- **📊 Universal Format**: Existing PDFs are passed through directly; supported
|
|
213
|
+
non-PDF documents become processable through PDF conversion
|
|
257
214
|
- **🖼️ Visual Understanding**: Handles documents with visual elements,
|
|
258
215
|
formatting, and structure
|
|
259
216
|
|
|
260
217
|
## 📚 Documentation
|
|
261
218
|
|
|
262
219
|
For comprehensive documentation, examples, and guides, visit our
|
|
263
|
-
[documentation site](https://structx.
|
|
220
|
+
[documentation site](https://structx.aolabs.dev).
|
|
264
221
|
|
|
265
|
-
- [Getting Started](https://structx.
|
|
266
|
-
- [Basic Extraction](https://structx.
|
|
267
|
-
- [Unstructured Text Processing](https://structx.
|
|
268
|
-
- [Async Operations](https://structx.
|
|
269
|
-
- [Multiple Queries](https://structx.
|
|
270
|
-
- [Custom Models](https://structx.
|
|
271
|
-
- [Token Usage Tracking](https://structx.
|
|
272
|
-
- [API Reference](https://structx.
|
|
222
|
+
- [Getting Started](https://structx.aolabs.dev/getting-started)
|
|
223
|
+
- [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
|
|
224
|
+
- [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
|
|
225
|
+
- [Async Operations](https://structx.aolabs.dev/guides/async-operations)
|
|
226
|
+
- [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
|
|
227
|
+
- [Custom Models](https://structx.aolabs.dev/guides/custom-models)
|
|
228
|
+
- [Token Usage Tracking](https://structx.aolabs.dev/guides/token-tracking)
|
|
229
|
+
- [API Reference](https://structx.aolabs.dev/api/extractor)
|
|
273
230
|
|
|
274
231
|
## Examples
|
|
275
232
|
|
|
276
|
-
Check out our [example gallery](https://structx.
|
|
233
|
+
Check out our [example gallery](https://structx.aolabs.dev/examples) for
|
|
277
234
|
real-world use cases,
|
|
278
235
|
|
|
279
236
|
## 📁 Supported File Formats
|
|
@@ -290,21 +247,22 @@ real-world use cases,
|
|
|
290
247
|
|
|
291
248
|
| Format | Extensions | Processing Method | Quality |
|
|
292
249
|
| -------- | --------------------------------------------- | ------------------------------------- | ---------- |
|
|
293
|
-
| **PDF** | `.pdf` |
|
|
294
|
-
| **Word** | `.docx`, `.doc` | Docling →
|
|
295
|
-
| **
|
|
250
|
+
| **PDF** | `.pdf` | PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
251
|
+
| **Word** | `.docx`, `.doc` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
252
|
+
| **PowerPoint** | `.pptx`, `.ppt` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
253
|
+
| **Text** | `.txt`, `.md`, `.py`, `.log`, `.xml`, `.html` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
296
254
|
|
|
297
|
-
### 🔄 Processing
|
|
255
|
+
### 🔄 Processing Pipeline
|
|
298
256
|
|
|
299
|
-
- **
|
|
300
|
-
- **
|
|
301
|
-
|
|
302
|
-
- **
|
|
257
|
+
- **PDF passthrough**: Existing PDFs are sent directly to multimodal extraction
|
|
258
|
+
- **Docling parsing**: Reads non-PDF document-like inputs into a structured document model
|
|
259
|
+
- **WeasyPrint rendering**: Converts Docling HTML to a temporary PDF for non-PDF inputs
|
|
260
|
+
- **Multimodal extraction**: Sends the rendered PDF to instructor's multimodal API
|
|
303
261
|
|
|
304
262
|
## Contributing
|
|
305
263
|
|
|
306
264
|
Contributions are welcome! Please read our
|
|
307
|
-
[Contributing Guidelines](https://structx.
|
|
265
|
+
[Contributing Guidelines](https://structx.aolabs.dev/contributing) for
|
|
308
266
|
details.
|
|
309
267
|
|
|
310
268
|
## License
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
Advanced structured data extraction from any document using LLMs with multimodal
|
|
4
4
|
support.
|
|
5
5
|
|
|
6
|
-
[](https://structx.
|
|
7
|
-
[](https://structx.aolabs.dev "Documentation")
|
|
7
|
+
[](https://pypi.org/project/structx "Package")
|
|
8
8
|
[](# "Build with GitHub Actions")
|
|
9
9
|
|
|
10
|
-
`structx` is a powerful Python library for extracting structured data from
|
|
11
|
-
|
|
12
|
-
multimodal PDF
|
|
13
|
-
|
|
10
|
+
`structx` is a powerful Python library for extracting structured data from text,
|
|
11
|
+
tables, and documents using Large Language Models (LLMs). With the optional
|
|
12
|
+
`docs` extra, it provides a multimodal PDF pipeline that passes PDFs directly to
|
|
13
|
+
vision-capable models and converts other document formats to PDF first.
|
|
14
14
|
|
|
15
15
|
## 🔔 Package rename notice (PyPI)
|
|
16
16
|
|
|
@@ -18,7 +18,7 @@ The PyPI distribution has been renamed from `structx-llm` to `structx`
|
|
|
18
18
|
(September 2025).
|
|
19
19
|
|
|
20
20
|
- Imports are unchanged: continue using `import structx`
|
|
21
|
-
-
|
|
21
|
+
- Document processing now lives in the optional `docs` extra
|
|
22
22
|
- Please update your environments and requirement files to use the new name
|
|
23
23
|
|
|
24
24
|
Upgrade commands:
|
|
@@ -29,20 +29,20 @@ pip install -U structx
|
|
|
29
29
|
```
|
|
30
30
|
|
|
31
31
|
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
32
|
-
with `structx`.
|
|
32
|
+
with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
33
33
|
|
|
34
34
|
## ✨ Key Features
|
|
35
35
|
|
|
36
36
|
### 🎯 **Advanced Document Processing**
|
|
37
37
|
|
|
38
|
-
- **� Multimodal PDF Pipeline**:
|
|
39
|
-
|
|
38
|
+
- **� Multimodal PDF Pipeline**: Passes PDFs directly to vision-capable models
|
|
39
|
+
and converts supported non-PDF documents to PDF
|
|
40
40
|
- **🖼️ Vision-Enabled Extraction**: Native instructor multimodal support for
|
|
41
41
|
PDFs and images
|
|
42
42
|
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
43
43
|
results
|
|
44
|
-
- **📊
|
|
45
|
-
Markdown, and more
|
|
44
|
+
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet in the base install,
|
|
45
|
+
with PDF, DOCX, TXT, Markdown, and more via `structx[docs]`
|
|
46
46
|
|
|
47
47
|
### 🚀 **Intelligent Data Extraction**
|
|
48
48
|
|
|
@@ -66,41 +66,21 @@ with `structx`.
|
|
|
66
66
|
## Installation
|
|
67
67
|
|
|
68
68
|
```bash
|
|
69
|
-
# Core package with basic extraction capabilities
|
|
70
69
|
pip install structx
|
|
71
70
|
```
|
|
72
71
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
For the best experience with all document types including advanced multimodal
|
|
76
|
-
PDF processing:
|
|
72
|
+
For document and multimodal PDF support:
|
|
77
73
|
|
|
78
74
|
```bash
|
|
79
|
-
|
|
80
|
-
pip install structx[docs]
|
|
81
|
-
|
|
82
|
-
# Individual components
|
|
83
|
-
pip install structx[pdf] # PDF processing with multimodal support
|
|
84
|
-
pip install structx[docx] # Advanced DOCX conversion via docling
|
|
75
|
+
pip install "structx[docs]"
|
|
85
76
|
```
|
|
86
77
|
|
|
87
|
-
### 🔧 What
|
|
88
|
-
|
|
89
|
-
- **`[docs]`**: Complete multimodal document processing pipeline
|
|
90
|
-
- PDF conversion from any document type
|
|
91
|
-
- Instructor multimodal vision support
|
|
92
|
-
- Advanced DOCX processing via docling
|
|
93
|
-
- Enhanced extraction quality
|
|
94
|
-
- **`[pdf]`**: PDF-specific processing
|
|
78
|
+
### 🔧 What The Package Provides
|
|
95
79
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
- **`[docx]`**: Advanced DOCX support
|
|
101
|
-
- Document conversion via docling
|
|
102
|
-
- Structure preservation
|
|
103
|
-
- Markdown-based processing pipeline
|
|
80
|
+
- Structured readers for CSV, Excel, JSON, Parquet, and Feather
|
|
81
|
+
- Instructor multimodal vision support
|
|
82
|
+
- Optional Docling document parsing with CPU-only PyTorch resolution for uv on Linux
|
|
83
|
+
- Optional WeasyPrint PDF rendering for non-PDF document formats
|
|
104
84
|
|
|
105
85
|
## Quick Start
|
|
106
86
|
|
|
@@ -131,16 +111,19 @@ print(result.data[0].model_dump_json(indent=2))
|
|
|
131
111
|
|
|
132
112
|
### 📄 Document Processing with Multimodal Support
|
|
133
113
|
|
|
114
|
+
Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
|
|
115
|
+
be passed directly through the multimodal path.
|
|
116
|
+
|
|
134
117
|
```python
|
|
135
|
-
# Process a PDF invoice
|
|
118
|
+
# Process a PDF invoice through the multimodal pipeline
|
|
136
119
|
result = extractor.extract(
|
|
137
|
-
data="scripts/example_input/S0305SampleInvoice.pdf",
|
|
120
|
+
data="scripts/example_input/S0305SampleInvoice.pdf",
|
|
138
121
|
query="extract the invoice number, total amount, and line items"
|
|
139
122
|
)
|
|
140
123
|
|
|
141
124
|
# Convert a DOCX contract and process with multimodal support
|
|
142
125
|
result = extractor.extract(
|
|
143
|
-
data="scripts/example_input/free-consultancy-agreement.docx",
|
|
126
|
+
data="scripts/example_input/free-consultancy-agreement.docx",
|
|
144
127
|
query="extract parties, effective date, and payment terms"
|
|
145
128
|
)
|
|
146
129
|
```
|
|
@@ -165,28 +148,28 @@ traditional text-based extraction:
|
|
|
165
148
|
complex layouts
|
|
166
149
|
- **🔄 No Chunking Issues**: Eliminates problems with information split across
|
|
167
150
|
chunks
|
|
168
|
-
- **📊 Universal Format**:
|
|
169
|
-
conversion
|
|
151
|
+
- **📊 Universal Format**: Existing PDFs are passed through directly; supported
|
|
152
|
+
non-PDF documents become processable through PDF conversion
|
|
170
153
|
- **🖼️ Visual Understanding**: Handles documents with visual elements,
|
|
171
154
|
formatting, and structure
|
|
172
155
|
|
|
173
156
|
## 📚 Documentation
|
|
174
157
|
|
|
175
158
|
For comprehensive documentation, examples, and guides, visit our
|
|
176
|
-
[documentation site](https://structx.
|
|
159
|
+
[documentation site](https://structx.aolabs.dev).
|
|
177
160
|
|
|
178
|
-
- [Getting Started](https://structx.
|
|
179
|
-
- [Basic Extraction](https://structx.
|
|
180
|
-
- [Unstructured Text Processing](https://structx.
|
|
181
|
-
- [Async Operations](https://structx.
|
|
182
|
-
- [Multiple Queries](https://structx.
|
|
183
|
-
- [Custom Models](https://structx.
|
|
184
|
-
- [Token Usage Tracking](https://structx.
|
|
185
|
-
- [API Reference](https://structx.
|
|
161
|
+
- [Getting Started](https://structx.aolabs.dev/getting-started)
|
|
162
|
+
- [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
|
|
163
|
+
- [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
|
|
164
|
+
- [Async Operations](https://structx.aolabs.dev/guides/async-operations)
|
|
165
|
+
- [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
|
|
166
|
+
- [Custom Models](https://structx.aolabs.dev/guides/custom-models)
|
|
167
|
+
- [Token Usage Tracking](https://structx.aolabs.dev/guides/token-tracking)
|
|
168
|
+
- [API Reference](https://structx.aolabs.dev/api/extractor)
|
|
186
169
|
|
|
187
170
|
## Examples
|
|
188
171
|
|
|
189
|
-
Check out our [example gallery](https://structx.
|
|
172
|
+
Check out our [example gallery](https://structx.aolabs.dev/examples) for
|
|
190
173
|
real-world use cases,
|
|
191
174
|
|
|
192
175
|
## 📁 Supported File Formats
|
|
@@ -203,21 +186,22 @@ real-world use cases,
|
|
|
203
186
|
|
|
204
187
|
| Format | Extensions | Processing Method | Quality |
|
|
205
188
|
| -------- | --------------------------------------------- | ------------------------------------- | ---------- |
|
|
206
|
-
| **PDF** | `.pdf` |
|
|
207
|
-
| **Word** | `.docx`, `.doc` | Docling →
|
|
208
|
-
| **
|
|
189
|
+
| **PDF** | `.pdf` | PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
190
|
+
| **Word** | `.docx`, `.doc` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
191
|
+
| **PowerPoint** | `.pptx`, `.ppt` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
192
|
+
| **Text** | `.txt`, `.md`, `.py`, `.log`, `.xml`, `.html` | Docling → HTML → PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
209
193
|
|
|
210
|
-
### 🔄 Processing
|
|
194
|
+
### 🔄 Processing Pipeline
|
|
211
195
|
|
|
212
|
-
- **
|
|
213
|
-
- **
|
|
214
|
-
|
|
215
|
-
- **
|
|
196
|
+
- **PDF passthrough**: Existing PDFs are sent directly to multimodal extraction
|
|
197
|
+
- **Docling parsing**: Reads non-PDF document-like inputs into a structured document model
|
|
198
|
+
- **WeasyPrint rendering**: Converts Docling HTML to a temporary PDF for non-PDF inputs
|
|
199
|
+
- **Multimodal extraction**: Sends the rendered PDF to instructor's multimodal API
|
|
216
200
|
|
|
217
201
|
## Contributing
|
|
218
202
|
|
|
219
203
|
Contributions are welcome! Please read our
|
|
220
|
-
[Contributing Guidelines](https://structx.
|
|
204
|
+
[Contributing Guidelines](https://structx.aolabs.dev/contributing) for
|
|
221
205
|
details.
|
|
222
206
|
|
|
223
207
|
## License
|
|
@@ -4,9 +4,9 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "structx"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.5.0"
|
|
8
8
|
description = "Structured data extraction from text using LLMs and dynamic model generation"
|
|
9
|
-
authors = [{ name = "blacksuan19", email = "py@
|
|
9
|
+
authors = [{ name = "blacksuan19", email = "py@aolabs.dev" }]
|
|
10
10
|
license-files = ["LICENSE"]
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
requires-python = ">=3.10"
|
|
@@ -50,51 +50,44 @@ dependencies = [
|
|
|
50
50
|
]
|
|
51
51
|
|
|
52
52
|
[project.optional-dependencies]
|
|
53
|
-
dev = ["black", "bumpver", "pip-tools", "wheel"]
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
]
|
|
60
|
-
docx = [
|
|
61
|
-
"python-docx", # Basic DOCX support
|
|
62
|
-
"docling", # Advanced DOCX conversion via docling
|
|
63
|
-
"markdown", # Markdown processing
|
|
64
|
-
"weasyprint", # PDF generation
|
|
65
|
-
]
|
|
53
|
+
dev = ["black", "bumpver", "pip-tools", "pytest", "wheel"]
|
|
54
|
+
|
|
55
|
+
# Optional document processing support. Kept out of the base install because
|
|
56
|
+
# Docling's local model pipeline requires PyTorch.
|
|
57
|
+
# The format-pdf extra is still needed because DocumentConverter imports its
|
|
58
|
+
# PDF backend dependencies even when existing PDFs are passed through directly.
|
|
66
59
|
docs = [
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"docling", # Advanced document conversion
|
|
71
|
-
"weasyprint", # PDF generation from HTML/markdown
|
|
72
|
-
"markdown", # Markdown processing and extensions
|
|
73
|
-
]
|
|
74
|
-
# Legacy support - kept for backward compatibility
|
|
75
|
-
unstructured = ["unstructured[all-docs]"]
|
|
76
|
-
docling = ["docling", "markdown", "weasyprint"]
|
|
77
|
-
advanced = [
|
|
78
|
-
"pypdf2",
|
|
79
|
-
"python-docx",
|
|
80
|
-
"instructor[multimodal]",
|
|
81
|
-
"docling",
|
|
60
|
+
"docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]",
|
|
61
|
+
"torch",
|
|
62
|
+
"torchvision",
|
|
82
63
|
"weasyprint",
|
|
83
|
-
"markdown",
|
|
84
|
-
"unstructured[all-docs]",
|
|
85
64
|
]
|
|
65
|
+
|
|
66
|
+
# Documentation site tooling.
|
|
86
67
|
mkdocs = [
|
|
87
68
|
"mkdocs",
|
|
88
69
|
"mkdocs-material",
|
|
89
|
-
"mkdocstrings[markdown]",
|
|
90
70
|
"mkdocstrings[python]",
|
|
91
71
|
"mkdocs-git-revision-date-localized-plugin",
|
|
92
72
|
"mkdocs-material[imaging]",
|
|
93
73
|
"mkdocs-mermaid2-plugin",
|
|
94
74
|
]
|
|
95
75
|
|
|
76
|
+
[[tool.uv.index]]
|
|
77
|
+
name = "pytorch-cpu"
|
|
78
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
79
|
+
explicit = true
|
|
80
|
+
|
|
81
|
+
[tool.uv.sources]
|
|
82
|
+
torch = [
|
|
83
|
+
{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" },
|
|
84
|
+
]
|
|
85
|
+
torchvision = [
|
|
86
|
+
{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" },
|
|
87
|
+
]
|
|
88
|
+
|
|
96
89
|
[tool.bumpver]
|
|
97
|
-
current_version = "0.
|
|
90
|
+
current_version = "0.5.0"
|
|
98
91
|
version_pattern = "MAJOR.MINOR.PATCH"
|
|
99
92
|
commit_message = "Bump version {old_version} -> {new_version}"
|
|
100
93
|
commit = true
|
|
@@ -110,3 +103,12 @@ push = true
|
|
|
110
103
|
[tool.setuptools.packages.find]
|
|
111
104
|
# Only package the library code under structx; exclude MkDocs overrides and others
|
|
112
105
|
include = ["structx*"]
|
|
106
|
+
|
|
107
|
+
[tool.pytest.ini_options]
|
|
108
|
+
filterwarnings = [
|
|
109
|
+
"ignore:This field is deprecated.*:DeprecationWarning:docling\\.pipeline\\.standard_pdf_pipeline",
|
|
110
|
+
"ignore:torch\\.ao\\.quantization is deprecated.*:DeprecationWarning:torch\\.ao\\.quantization\\.quantize",
|
|
111
|
+
"ignore:torch\\.quantize_per_tensor.*:UserWarning:torch\\.ao\\.nn\\.quantized\\.dynamic\\.modules\\.rnn",
|
|
112
|
+
]
|
|
113
|
+
pythonpath = ["."]
|
|
114
|
+
testpaths = ["tests"]
|
|
@@ -2,7 +2,7 @@ from pathlib import Path
|
|
|
2
2
|
from typing import Annotated, Any, Dict, Optional, Union
|
|
3
3
|
|
|
4
4
|
from omegaconf import OmegaConf
|
|
5
|
-
from pydantic import BaseModel, Field
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
6
|
|
|
7
7
|
from structx.utils.types import DictStrAny
|
|
8
8
|
|
|
@@ -10,6 +10,8 @@ from structx.utils.types import DictStrAny
|
|
|
10
10
|
class StepConfig(BaseModel):
|
|
11
11
|
"""Configuration for an individual extraction step"""
|
|
12
12
|
|
|
13
|
+
model_config = ConfigDict(validate_assignment=True)
|
|
14
|
+
|
|
13
15
|
temperature: Optional[
|
|
14
16
|
Annotated[
|
|
15
17
|
float, Field(ge=0.0, le=1.0, description="Sampling temperature for LLM")
|
|
@@ -45,9 +47,6 @@ class StepConfig(BaseModel):
|
|
|
45
47
|
# Merge with defaults, preferring current values
|
|
46
48
|
return {**defaults, **current}
|
|
47
49
|
|
|
48
|
-
class Config:
|
|
49
|
-
validate_assignment = True
|
|
50
|
-
|
|
51
50
|
|
|
52
51
|
class ExtractionConfig:
|
|
53
52
|
"""Configuration management for structx using OmegaConf and Pydantic"""
|
|
@@ -2,7 +2,7 @@ from dataclasses import dataclass
|
|
|
2
2
|
from typing import Any, Dict, Generic, List, Optional, Type, Union
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
|
-
from pydantic import BaseModel, Field
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
6
|
|
|
7
7
|
from structx.utils.types import T
|
|
8
8
|
from structx.utils.usage import ExtractorUsage, UsageSummary
|
|
@@ -11,6 +11,8 @@ from structx.utils.usage import ExtractorUsage, UsageSummary
|
|
|
11
11
|
class ModelField(BaseModel):
|
|
12
12
|
"""Definition of a field in the extraction model"""
|
|
13
13
|
|
|
14
|
+
model_config = ConfigDict(validate_assignment=True)
|
|
15
|
+
|
|
14
16
|
name: str = Field(description="Name of the field")
|
|
15
17
|
type: str = Field(description="Type of the field")
|
|
16
18
|
description: str = Field(description="Description of what this field represents")
|
|
@@ -21,9 +23,6 @@ class ModelField(BaseModel):
|
|
|
21
23
|
default=None, description="Fields for nested models"
|
|
22
24
|
)
|
|
23
25
|
|
|
24
|
-
class Config:
|
|
25
|
-
validate_assignment = True
|
|
26
|
-
|
|
27
26
|
|
|
28
27
|
class QueryRefinement(BaseModel):
|
|
29
28
|
"""Refined query with structural information"""
|
|
@@ -40,6 +39,8 @@ class QueryRefinement(BaseModel):
|
|
|
40
39
|
class ExtractionGuide(BaseModel):
|
|
41
40
|
"""Guide for structured extraction"""
|
|
42
41
|
|
|
42
|
+
model_config = ConfigDict(extra="allow")
|
|
43
|
+
|
|
43
44
|
target_columns: List[str] = Field(description="Columns to analyze")
|
|
44
45
|
|
|
45
46
|
structural_patterns: Optional[Dict[str, str]] = Field(
|
|
@@ -52,9 +53,6 @@ class ExtractionGuide(BaseModel):
|
|
|
52
53
|
description="Principles for data organization"
|
|
53
54
|
)
|
|
54
55
|
|
|
55
|
-
class Config:
|
|
56
|
-
extra = "allow" # Allow extra fields to be flexible with LLM responses
|
|
57
|
-
|
|
58
56
|
|
|
59
57
|
class ExtractionRequest(BaseModel):
|
|
60
58
|
"""Request for model generation"""
|
|
@@ -123,7 +123,7 @@ class ExtractionEngine:
|
|
|
123
123
|
except ImportError:
|
|
124
124
|
raise ImportError(
|
|
125
125
|
"instructor multimodal support is required for PDF processing. "
|
|
126
|
-
"Install
|
|
126
|
+
"Install structx with: pip install structx"
|
|
127
127
|
)
|
|
128
128
|
|
|
129
129
|
# For multimodal PDF, we need a single wrapper model, not a container with multiple items
|