structx 0.4.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- structx-0.4.8/LICENSE +21 -0
- structx-0.4.8/PKG-INFO +313 -0
- structx-0.4.8/README.md +226 -0
- structx-0.4.8/pyproject.toml +112 -0
- structx-0.4.8/setup.cfg +4 -0
- structx-0.4.8/structx/__init__.py +24 -0
- structx-0.4.8/structx/core/__init__.py +0 -0
- structx-0.4.8/structx/core/config.py +127 -0
- structx-0.4.8/structx/core/exceptions.py +34 -0
- structx-0.4.8/structx/core/models.py +147 -0
- structx-0.4.8/structx/extraction/__init__.py +38 -0
- structx-0.4.8/structx/extraction/core/__init__.py +6 -0
- structx-0.4.8/structx/extraction/core/llm_core.py +206 -0
- structx-0.4.8/structx/extraction/core/model_utils.py +143 -0
- structx-0.4.8/structx/extraction/engines/__init__.py +5 -0
- structx-0.4.8/structx/extraction/engines/extraction_engine.py +221 -0
- structx-0.4.8/structx/extraction/extractor.py +540 -0
- structx-0.4.8/structx/extraction/generator.py +232 -0
- structx-0.4.8/structx/extraction/processors/__init__.py +6 -0
- structx-0.4.8/structx/extraction/processors/data_content.py +367 -0
- structx-0.4.8/structx/extraction/processors/model_operations.py +233 -0
- structx-0.4.8/structx/extraction/result_manager.py +125 -0
- structx-0.4.8/structx/utils/__init__.py +0 -0
- structx-0.4.8/structx/utils/constants.py +36 -0
- structx-0.4.8/structx/utils/file_reader.py +298 -0
- structx-0.4.8/structx/utils/helpers.py +233 -0
- structx-0.4.8/structx/utils/prompts.py +183 -0
- structx-0.4.8/structx/utils/types.py +12 -0
- structx-0.4.8/structx/utils/usage.py +327 -0
- structx-0.4.8/structx.egg-info/PKG-INFO +313 -0
- structx-0.4.8/structx.egg-info/SOURCES.txt +32 -0
- structx-0.4.8/structx.egg-info/dependency_links.txt +1 -0
- structx-0.4.8/structx.egg-info/requires.txt +62 -0
- structx-0.4.8/structx.egg-info/top_level.txt +1 -0
structx-0.4.8/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Abubakar Omer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
structx-0.4.8/PKG-INFO
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: structx
|
|
3
|
+
Version: 0.4.8
|
|
4
|
+
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
|
+
Author-email: blacksuan19 <py@blacksuan19.dev>
|
|
6
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
7
|
+
Classifier: Framework :: AsyncIO
|
|
8
|
+
Classifier: Framework :: Pydantic
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Database
|
|
17
|
+
Classifier: Topic :: Documentation
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
19
|
+
Classifier: Topic :: Office/Business
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
25
|
+
Classifier: Topic :: Text Processing :: Markup
|
|
26
|
+
Classifier: Topic :: Text Processing
|
|
27
|
+
Classifier: Topic :: Utilities
|
|
28
|
+
Classifier: Typing :: Typed
|
|
29
|
+
Requires-Python: >=3.10
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Requires-Dist: instructor
|
|
33
|
+
Requires-Dist: litellm
|
|
34
|
+
Requires-Dist: loguru
|
|
35
|
+
Requires-Dist: omegaconf
|
|
36
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
37
|
+
Requires-Dist: pandas
|
|
38
|
+
Requires-Dist: pydantic
|
|
39
|
+
Requires-Dist: tabulate>=0.9.0
|
|
40
|
+
Requires-Dist: tenacity
|
|
41
|
+
Requires-Dist: tqdm
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: black; extra == "dev"
|
|
44
|
+
Requires-Dist: bumpver; extra == "dev"
|
|
45
|
+
Requires-Dist: pip-tools; extra == "dev"
|
|
46
|
+
Requires-Dist: wheel; extra == "dev"
|
|
47
|
+
Provides-Extra: pdf
|
|
48
|
+
Requires-Dist: pypdf2; extra == "pdf"
|
|
49
|
+
Requires-Dist: instructor[multimodal]; extra == "pdf"
|
|
50
|
+
Requires-Dist: weasyprint; extra == "pdf"
|
|
51
|
+
Requires-Dist: markdown; extra == "pdf"
|
|
52
|
+
Provides-Extra: docx
|
|
53
|
+
Requires-Dist: python-docx; extra == "docx"
|
|
54
|
+
Requires-Dist: docling; extra == "docx"
|
|
55
|
+
Requires-Dist: markdown; extra == "docx"
|
|
56
|
+
Requires-Dist: weasyprint; extra == "docx"
|
|
57
|
+
Provides-Extra: docs
|
|
58
|
+
Requires-Dist: pypdf2; extra == "docs"
|
|
59
|
+
Requires-Dist: python-docx; extra == "docs"
|
|
60
|
+
Requires-Dist: instructor[multimodal]; extra == "docs"
|
|
61
|
+
Requires-Dist: docling; extra == "docs"
|
|
62
|
+
Requires-Dist: weasyprint; extra == "docs"
|
|
63
|
+
Requires-Dist: markdown; extra == "docs"
|
|
64
|
+
Provides-Extra: unstructured
|
|
65
|
+
Requires-Dist: unstructured[all-docs]; extra == "unstructured"
|
|
66
|
+
Provides-Extra: docling
|
|
67
|
+
Requires-Dist: docling; extra == "docling"
|
|
68
|
+
Requires-Dist: markdown; extra == "docling"
|
|
69
|
+
Requires-Dist: weasyprint; extra == "docling"
|
|
70
|
+
Provides-Extra: advanced
|
|
71
|
+
Requires-Dist: pypdf2; extra == "advanced"
|
|
72
|
+
Requires-Dist: python-docx; extra == "advanced"
|
|
73
|
+
Requires-Dist: instructor[multimodal]; extra == "advanced"
|
|
74
|
+
Requires-Dist: docling; extra == "advanced"
|
|
75
|
+
Requires-Dist: weasyprint; extra == "advanced"
|
|
76
|
+
Requires-Dist: markdown; extra == "advanced"
|
|
77
|
+
Requires-Dist: unstructured[all-docs]; extra == "advanced"
|
|
78
|
+
Provides-Extra: mkdocs
|
|
79
|
+
Requires-Dist: mkdocs; extra == "mkdocs"
|
|
80
|
+
Requires-Dist: mkdocs-material; extra == "mkdocs"
|
|
81
|
+
Requires-Dist: mkdocstrings[markdown]; extra == "mkdocs"
|
|
82
|
+
Requires-Dist: mkdocstrings[python]; extra == "mkdocs"
|
|
83
|
+
Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == "mkdocs"
|
|
84
|
+
Requires-Dist: mkdocs-material[imaging]; extra == "mkdocs"
|
|
85
|
+
Requires-Dist: mkdocs-mermaid2-plugin; extra == "mkdocs"
|
|
86
|
+
Dynamic: license-file
|
|
87
|
+
|
|
88
|
+
# structx
|
|
89
|
+
|
|
90
|
+
Advanced structured data extraction from any document using LLMs with multimodal
|
|
91
|
+
support.
|
|
92
|
+
|
|
93
|
+
[](https://structx.blacksuan19.dev "Documentation")
|
|
94
|
+
[](https://pypi.org/project/structx "Package")
|
|
95
|
+
[](# "Build with GitHub Actions")
|
|
96
|
+
|
|
97
|
+
`structx` is a powerful Python library for extracting structured data from any
|
|
98
|
+
document or text using Large Language Models (LLMs). It features an innovative
|
|
99
|
+
multimodal PDF processing pipeline that converts any document to PDF and uses
|
|
100
|
+
instructor's vision capabilities for superior extraction quality.
|
|
101
|
+
|
|
102
|
+
## 🔔 Package rename notice (PyPI)
|
|
103
|
+
|
|
104
|
+
The PyPI distribution has been renamed from `structx-llm` to `structx`
|
|
105
|
+
(September 2025).
|
|
106
|
+
|
|
107
|
+
- Imports are unchanged: continue using `import structx`
|
|
108
|
+
- Extras are unchanged: `structx[docs]`, `structx[pdf]`, `structx[docx]`
|
|
109
|
+
- Please update your environments and requirement files to use the new name
|
|
110
|
+
|
|
111
|
+
Upgrade commands:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
pip uninstall -y structx-llm
|
|
115
|
+
pip install -U structx
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
119
|
+
with `structx`.
|
|
120
|
+
|
|
121
|
+
## ✨ Key Features
|
|
122
|
+
|
|
123
|
+
### 🎯 **Advanced Document Processing**
|
|
124
|
+
|
|
125
|
+
- **� Multimodal PDF Pipeline**: Converts any document (TXT, DOCX, etc.) to PDF
|
|
126
|
+
for optimal extraction
|
|
127
|
+
- **🖼️ Vision-Enabled Extraction**: Native instructor multimodal support for
|
|
128
|
+
PDFs and images
|
|
129
|
+
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
130
|
+
results
|
|
131
|
+
- **📊 Universal File Support**: CSV, Excel, JSON, Parquet, PDF, DOCX, TXT,
|
|
132
|
+
Markdown, and more
|
|
133
|
+
|
|
134
|
+
### 🚀 **Intelligent Data Extraction**
|
|
135
|
+
|
|
136
|
+
- **🔄 Dynamic Model Generation**: Create type-safe Pydantic models from natural
|
|
137
|
+
language queries
|
|
138
|
+
- **🎯 Automatic Schema Inference**: Intelligent schema generation and
|
|
139
|
+
refinement
|
|
140
|
+
- **📊 Complex Data Structures**: Support for nested and hierarchical data
|
|
141
|
+
- **🔄 Natural Language Refinement**: Improve models with conversational
|
|
142
|
+
instructions
|
|
143
|
+
|
|
144
|
+
### ⚡ **Performance & Reliability**
|
|
145
|
+
|
|
146
|
+
- **🚀 High-Performance Processing**: Multi-threaded and async operations
|
|
147
|
+
- **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
|
|
148
|
+
backoff
|
|
149
|
+
- **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
|
|
150
|
+
- **� Flexible Configuration**: Configurable extraction using OmegaConf
|
|
151
|
+
- **🔌 Multiple LLM Providers**: Support through litellm integration
|
|
152
|
+
|
|
153
|
+
## Installation
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
# Core package with basic extraction capabilities
|
|
157
|
+
pip install structx
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### 📄 Enhanced Document Processing (Recommended)
|
|
161
|
+
|
|
162
|
+
For the best experience with all document types including advanced multimodal
|
|
163
|
+
PDF processing:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
# Complete document processing support
|
|
167
|
+
pip install structx[docs]
|
|
168
|
+
|
|
169
|
+
# Individual components
|
|
170
|
+
pip install structx[pdf] # PDF processing with multimodal support
|
|
171
|
+
pip install structx[docx] # Advanced DOCX conversion via docling
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### 🔧 What Each Extra Provides
|
|
175
|
+
|
|
176
|
+
- **`[docs]`**: Complete multimodal document processing pipeline
|
|
177
|
+
- PDF conversion from any document type
|
|
178
|
+
- Instructor multimodal vision support
|
|
179
|
+
- Advanced DOCX processing via docling
|
|
180
|
+
- Enhanced extraction quality
|
|
181
|
+
- **`[pdf]`**: PDF-specific processing
|
|
182
|
+
|
|
183
|
+
- Multimodal PDF support via instructor
|
|
184
|
+
- PDF generation capabilities
|
|
185
|
+
- Basic PDF text extraction fallback
|
|
186
|
+
|
|
187
|
+
- **`[docx]`**: Advanced DOCX support
|
|
188
|
+
- Document conversion via docling
|
|
189
|
+
- Structure preservation
|
|
190
|
+
- Markdown-based processing pipeline
|
|
191
|
+
|
|
192
|
+
## Quick Start
|
|
193
|
+
|
|
194
|
+
### Basic Text Extraction
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
from structx import Extractor
|
|
198
|
+
|
|
199
|
+
# Initialize extractor
|
|
200
|
+
extractor = Extractor.from_litellm(
|
|
201
|
+
model="gpt-4o",
|
|
202
|
+
api_key="your-api-key",
|
|
203
|
+
max_retries=3, # Automatically retry on transient errors
|
|
204
|
+
min_wait=1, # Start with 1 second wait
|
|
205
|
+
max_wait=10 # Maximum 10 seconds between retries
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Extract from text
|
|
209
|
+
result = extractor.extract(
|
|
210
|
+
data="System check on 2024-01-15 detected high CPU usage (92%) on server-01.",
|
|
211
|
+
query="extract incident date and details"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
# Access results
|
|
215
|
+
print(f"Extracted {result.success_count} items")
|
|
216
|
+
print(result.data[0].model_dump_json(indent=2))
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### 📄 Document Processing with Multimodal Support
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
# Process a PDF invoice directly with vision capabilities
|
|
223
|
+
result = extractor.extract(
|
|
224
|
+
data="scripts/example_input/S0305SampleInvoice.pdf", # Direct multimodal processing
|
|
225
|
+
query="extract the invoice number, total amount, and line items"
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# Convert a DOCX contract and process with multimodal support
|
|
229
|
+
result = extractor.extract(
|
|
230
|
+
data="scripts/example_input/free-consultancy-agreement.docx", # Auto-converted to PDF -> multimodal
|
|
231
|
+
query="extract parties, effective date, and payment terms"
|
|
232
|
+
)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### 📊 Token Usage Monitoring
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
# Check token usage for cost monitoring
|
|
239
|
+
usage = result.get_token_usage()
|
|
240
|
+
if usage:
|
|
241
|
+
print(f"Total tokens: {usage.total_tokens}")
|
|
242
|
+
print(f"By step: {[(s.name, s.tokens) for s in usage.steps]}")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
## 🚀 Why Multimodal PDF Processing?
|
|
246
|
+
|
|
247
|
+
The innovative multimodal approach provides significant advantages over
|
|
248
|
+
traditional text-based extraction:
|
|
249
|
+
|
|
250
|
+
- **📄 Context Preservation**: Full document layout and structure are maintained
|
|
251
|
+
- **🎯 Higher Accuracy**: Vision models can interpret tables, charts, and
|
|
252
|
+
complex layouts
|
|
253
|
+
- **🔄 No Chunking Issues**: Eliminates problems with information split across
|
|
254
|
+
chunks
|
|
255
|
+
- **📊 Universal Format**: Any document type becomes processable through PDF
|
|
256
|
+
conversion
|
|
257
|
+
- **🖼️ Visual Understanding**: Handles documents with visual elements,
|
|
258
|
+
formatting, and structure
|
|
259
|
+
|
|
260
|
+
## 📚 Documentation
|
|
261
|
+
|
|
262
|
+
For comprehensive documentation, examples, and guides, visit our
|
|
263
|
+
[documentation site](https://structx.blacksuan19.dev).
|
|
264
|
+
|
|
265
|
+
- [Getting Started](https://structx.blacksuan19.dev/getting-started)
|
|
266
|
+
- [Basic Extraction](https://structx.blacksuan19.dev/guides/basic-extraction)
|
|
267
|
+
- [Unstructured Text Processing](https://structx.blacksuan19.dev/guides/unstructured-text)
|
|
268
|
+
- [Async Operations](https://structx.blacksuan19.dev/guides/async-operations)
|
|
269
|
+
- [Multiple Queries](https://structx.blacksuan19.dev/guides/multiple-queries)
|
|
270
|
+
- [Custom Models](https://structx.blacksuan19.dev/guides/custom-models)
|
|
271
|
+
- [Token Usage Tracking](https://structx.blacksuan19.dev/guides/token-tracking)
|
|
272
|
+
- [API Reference](https://structx.blacksuan19.dev/api/extractor)
|
|
273
|
+
|
|
274
|
+
## Examples
|
|
275
|
+
|
|
276
|
+
Check out our [example gallery](https://structx.blacksuan19.dev/examples) for
|
|
277
|
+
real-world use cases,
|
|
278
|
+
|
|
279
|
+
## 📁 Supported File Formats
|
|
280
|
+
|
|
281
|
+
### 📊 Structured Data (Direct Processing)
|
|
282
|
+
|
|
283
|
+
- **CSV**: Comma-separated values with custom delimiters
|
|
284
|
+
- **Excel**: .xlsx/.xls with sheet selection and custom options
|
|
285
|
+
- **JSON**: JavaScript Object Notation with nested support
|
|
286
|
+
- **Parquet**: Columnar storage format for large datasets
|
|
287
|
+
- **Feather**: Fast binary format for data frames
|
|
288
|
+
|
|
289
|
+
### 📄 Unstructured Documents (Multimodal Pipeline)
|
|
290
|
+
|
|
291
|
+
| Format | Extensions | Processing Method | Quality |
|
|
292
|
+
| -------- | --------------------------------------------- | ------------------------------------- | ---------- |
|
|
293
|
+
| **PDF** | `.pdf` | Direct multimodal processing | ⭐⭐⭐⭐⭐ |
|
|
294
|
+
| **Word** | `.docx`, `.doc` | Docling → Markdown → PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
295
|
+
| **Text** | `.txt`, `.md`, `.py`, `.log`, `.xml`, `.html` | Styled PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
296
|
+
|
|
297
|
+
### 🔄 Processing Modes
|
|
298
|
+
|
|
299
|
+
- **Multimodal PDF** (default): Best quality, preserves layout and context
|
|
300
|
+
- **Simple Text**: Fallback mode with chunking for memory-constrained
|
|
301
|
+
environments
|
|
302
|
+
- **Simple PDF**: Basic PDF text extraction without vision capabilities
|
|
303
|
+
|
|
304
|
+
## Contributing
|
|
305
|
+
|
|
306
|
+
Contributions are welcome! Please read our
|
|
307
|
+
[Contributing Guidelines](https://structx.blacksuan19.dev/contributing) for
|
|
308
|
+
details.
|
|
309
|
+
|
|
310
|
+
## License
|
|
311
|
+
|
|
312
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|
313
|
+
for details.
|
structx-0.4.8/README.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# structx
|
|
2
|
+
|
|
3
|
+
Advanced structured data extraction from any document using LLMs with multimodal
|
|
4
|
+
support.
|
|
5
|
+
|
|
6
|
+
[](https://structx.blacksuan19.dev "Documentation")
|
|
7
|
+
[](https://pypi.org/project/structx "Package")
|
|
8
|
+
[](# "Build with GitHub Actions")
|
|
9
|
+
|
|
10
|
+
`structx` is a powerful Python library for extracting structured data from any
|
|
11
|
+
document or text using Large Language Models (LLMs). It features an innovative
|
|
12
|
+
multimodal PDF processing pipeline that converts any document to PDF and uses
|
|
13
|
+
instructor's vision capabilities for superior extraction quality.
|
|
14
|
+
|
|
15
|
+
## 🔔 Package rename notice (PyPI)
|
|
16
|
+
|
|
17
|
+
The PyPI distribution has been renamed from `structx-llm` to `structx`
|
|
18
|
+
(September 2025).
|
|
19
|
+
|
|
20
|
+
- Imports are unchanged: continue using `import structx`
|
|
21
|
+
- Extras are unchanged: `structx[docs]`, `structx[pdf]`, `structx[docx]`
|
|
22
|
+
- Please update your environments and requirement files to use the new name
|
|
23
|
+
|
|
24
|
+
Upgrade commands:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip uninstall -y structx-llm
|
|
28
|
+
pip install -U structx
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
32
|
+
with `structx`.
|
|
33
|
+
|
|
34
|
+
## ✨ Key Features
|
|
35
|
+
|
|
36
|
+
### 🎯 **Advanced Document Processing**
|
|
37
|
+
|
|
38
|
+
- **� Multimodal PDF Pipeline**: Converts any document (TXT, DOCX, etc.) to PDF
|
|
39
|
+
for optimal extraction
|
|
40
|
+
- **🖼️ Vision-Enabled Extraction**: Native instructor multimodal support for
|
|
41
|
+
PDFs and images
|
|
42
|
+
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
43
|
+
results
|
|
44
|
+
- **📊 Universal File Support**: CSV, Excel, JSON, Parquet, PDF, DOCX, TXT,
|
|
45
|
+
Markdown, and more
|
|
46
|
+
|
|
47
|
+
### 🚀 **Intelligent Data Extraction**
|
|
48
|
+
|
|
49
|
+
- **🔄 Dynamic Model Generation**: Create type-safe Pydantic models from natural
|
|
50
|
+
language queries
|
|
51
|
+
- **🎯 Automatic Schema Inference**: Intelligent schema generation and
|
|
52
|
+
refinement
|
|
53
|
+
- **📊 Complex Data Structures**: Support for nested and hierarchical data
|
|
54
|
+
- **🔄 Natural Language Refinement**: Improve models with conversational
|
|
55
|
+
instructions
|
|
56
|
+
|
|
57
|
+
### ⚡ **Performance & Reliability**
|
|
58
|
+
|
|
59
|
+
- **🚀 High-Performance Processing**: Multi-threaded and async operations
|
|
60
|
+
- **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
|
|
61
|
+
backoff
|
|
62
|
+
- **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
|
|
63
|
+
- **� Flexible Configuration**: Configurable extraction using OmegaConf
|
|
64
|
+
- **🔌 Multiple LLM Providers**: Support through litellm integration
|
|
65
|
+
|
|
66
|
+
## Installation
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# Core package with basic extraction capabilities
|
|
70
|
+
pip install structx
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### 📄 Enhanced Document Processing (Recommended)
|
|
74
|
+
|
|
75
|
+
For the best experience with all document types including advanced multimodal
|
|
76
|
+
PDF processing:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# Complete document processing support
|
|
80
|
+
pip install structx[docs]
|
|
81
|
+
|
|
82
|
+
# Individual components
|
|
83
|
+
pip install structx[pdf] # PDF processing with multimodal support
|
|
84
|
+
pip install structx[docx] # Advanced DOCX conversion via docling
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### 🔧 What Each Extra Provides
|
|
88
|
+
|
|
89
|
+
- **`[docs]`**: Complete multimodal document processing pipeline
|
|
90
|
+
- PDF conversion from any document type
|
|
91
|
+
- Instructor multimodal vision support
|
|
92
|
+
- Advanced DOCX processing via docling
|
|
93
|
+
- Enhanced extraction quality
|
|
94
|
+
- **`[pdf]`**: PDF-specific processing
|
|
95
|
+
|
|
96
|
+
- Multimodal PDF support via instructor
|
|
97
|
+
- PDF generation capabilities
|
|
98
|
+
- Basic PDF text extraction fallback
|
|
99
|
+
|
|
100
|
+
- **`[docx]`**: Advanced DOCX support
|
|
101
|
+
- Document conversion via docling
|
|
102
|
+
- Structure preservation
|
|
103
|
+
- Markdown-based processing pipeline
|
|
104
|
+
|
|
105
|
+
## Quick Start
|
|
106
|
+
|
|
107
|
+
### Basic Text Extraction
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from structx import Extractor
|
|
111
|
+
|
|
112
|
+
# Initialize extractor
|
|
113
|
+
extractor = Extractor.from_litellm(
|
|
114
|
+
model="gpt-4o",
|
|
115
|
+
api_key="your-api-key",
|
|
116
|
+
max_retries=3, # Automatically retry on transient errors
|
|
117
|
+
min_wait=1, # Start with 1 second wait
|
|
118
|
+
max_wait=10 # Maximum 10 seconds between retries
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Extract from text
|
|
122
|
+
result = extractor.extract(
|
|
123
|
+
data="System check on 2024-01-15 detected high CPU usage (92%) on server-01.",
|
|
124
|
+
query="extract incident date and details"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Access results
|
|
128
|
+
print(f"Extracted {result.success_count} items")
|
|
129
|
+
print(result.data[0].model_dump_json(indent=2))
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### 📄 Document Processing with Multimodal Support
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
# Process a PDF invoice directly with vision capabilities
|
|
136
|
+
result = extractor.extract(
|
|
137
|
+
data="scripts/example_input/S0305SampleInvoice.pdf", # Direct multimodal processing
|
|
138
|
+
query="extract the invoice number, total amount, and line items"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Convert a DOCX contract and process with multimodal support
|
|
142
|
+
result = extractor.extract(
|
|
143
|
+
data="scripts/example_input/free-consultancy-agreement.docx", # Auto-converted to PDF -> multimodal
|
|
144
|
+
query="extract parties, effective date, and payment terms"
|
|
145
|
+
)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### 📊 Token Usage Monitoring
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
# Check token usage for cost monitoring
|
|
152
|
+
usage = result.get_token_usage()
|
|
153
|
+
if usage:
|
|
154
|
+
print(f"Total tokens: {usage.total_tokens}")
|
|
155
|
+
print(f"By step: {[(s.name, s.tokens) for s in usage.steps]}")
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## 🚀 Why Multimodal PDF Processing?
|
|
159
|
+
|
|
160
|
+
The innovative multimodal approach provides significant advantages over
|
|
161
|
+
traditional text-based extraction:
|
|
162
|
+
|
|
163
|
+
- **📄 Context Preservation**: Full document layout and structure are maintained
|
|
164
|
+
- **🎯 Higher Accuracy**: Vision models can interpret tables, charts, and
|
|
165
|
+
complex layouts
|
|
166
|
+
- **🔄 No Chunking Issues**: Eliminates problems with information split across
|
|
167
|
+
chunks
|
|
168
|
+
- **📊 Universal Format**: Any document type becomes processable through PDF
|
|
169
|
+
conversion
|
|
170
|
+
- **🖼️ Visual Understanding**: Handles documents with visual elements,
|
|
171
|
+
formatting, and structure
|
|
172
|
+
|
|
173
|
+
## 📚 Documentation
|
|
174
|
+
|
|
175
|
+
For comprehensive documentation, examples, and guides, visit our
|
|
176
|
+
[documentation site](https://structx.blacksuan19.dev).
|
|
177
|
+
|
|
178
|
+
- [Getting Started](https://structx.blacksuan19.dev/getting-started)
|
|
179
|
+
- [Basic Extraction](https://structx.blacksuan19.dev/guides/basic-extraction)
|
|
180
|
+
- [Unstructured Text Processing](https://structx.blacksuan19.dev/guides/unstructured-text)
|
|
181
|
+
- [Async Operations](https://structx.blacksuan19.dev/guides/async-operations)
|
|
182
|
+
- [Multiple Queries](https://structx.blacksuan19.dev/guides/multiple-queries)
|
|
183
|
+
- [Custom Models](https://structx.blacksuan19.dev/guides/custom-models)
|
|
184
|
+
- [Token Usage Tracking](https://structx.blacksuan19.dev/guides/token-tracking)
|
|
185
|
+
- [API Reference](https://structx.blacksuan19.dev/api/extractor)
|
|
186
|
+
|
|
187
|
+
## Examples
|
|
188
|
+
|
|
189
|
+
Check out our [example gallery](https://structx.blacksuan19.dev/examples) for
|
|
190
|
+
real-world use cases,
|
|
191
|
+
|
|
192
|
+
## 📁 Supported File Formats
|
|
193
|
+
|
|
194
|
+
### 📊 Structured Data (Direct Processing)
|
|
195
|
+
|
|
196
|
+
- **CSV**: Comma-separated values with custom delimiters
|
|
197
|
+
- **Excel**: .xlsx/.xls with sheet selection and custom options
|
|
198
|
+
- **JSON**: JavaScript Object Notation with nested support
|
|
199
|
+
- **Parquet**: Columnar storage format for large datasets
|
|
200
|
+
- **Feather**: Fast binary format for data frames
|
|
201
|
+
|
|
202
|
+
### 📄 Unstructured Documents (Multimodal Pipeline)
|
|
203
|
+
|
|
204
|
+
| Format | Extensions | Processing Method | Quality |
|
|
205
|
+
| -------- | --------------------------------------------- | ------------------------------------- | ---------- |
|
|
206
|
+
| **PDF** | `.pdf` | Direct multimodal processing | ⭐⭐⭐⭐⭐ |
|
|
207
|
+
| **Word** | `.docx`, `.doc` | Docling → Markdown → PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
208
|
+
| **Text** | `.txt`, `.md`, `.py`, `.log`, `.xml`, `.html` | Styled PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
209
|
+
|
|
210
|
+
### 🔄 Processing Modes
|
|
211
|
+
|
|
212
|
+
- **Multimodal PDF** (default): Best quality, preserves layout and context
|
|
213
|
+
- **Simple Text**: Fallback mode with chunking for memory-constrained
|
|
214
|
+
environments
|
|
215
|
+
- **Simple PDF**: Basic PDF text extraction without vision capabilities
|
|
216
|
+
|
|
217
|
+
## Contributing
|
|
218
|
+
|
|
219
|
+
Contributions are welcome! Please read our
|
|
220
|
+
[Contributing Guidelines](https://structx.blacksuan19.dev/contributing) for
|
|
221
|
+
details.
|
|
222
|
+
|
|
223
|
+
## License
|
|
224
|
+
|
|
225
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|
226
|
+
for details.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel", "pip>=21.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "structx"
|
|
7
|
+
version = "0.4.8"
|
|
8
|
+
description = "Structured data extraction from text using LLMs and dynamic model generation"
|
|
9
|
+
authors = [{ name = "blacksuan19", email = "py@blacksuan19.dev" }]
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 5 - Production/Stable",
|
|
15
|
+
"Framework :: AsyncIO",
|
|
16
|
+
"Framework :: Pydantic",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.9",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Topic :: Database",
|
|
25
|
+
"Topic :: Documentation",
|
|
26
|
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
27
|
+
"Topic :: Office/Business",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
29
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
30
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
31
|
+
"Topic :: Software Development :: Libraries",
|
|
32
|
+
"Topic :: Text Processing :: Linguistic",
|
|
33
|
+
"Topic :: Text Processing :: Markup",
|
|
34
|
+
"Topic :: Text Processing",
|
|
35
|
+
"Topic :: Utilities",
|
|
36
|
+
"Typing :: Typed",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
dependencies = [
|
|
40
|
+
"instructor",
|
|
41
|
+
"litellm",
|
|
42
|
+
"loguru",
|
|
43
|
+
"omegaconf",
|
|
44
|
+
"openpyxl>=3.1.5",
|
|
45
|
+
"pandas",
|
|
46
|
+
"pydantic",
|
|
47
|
+
"tabulate>=0.9.0", # required for pandas .to_markdown()
|
|
48
|
+
"tenacity",
|
|
49
|
+
"tqdm",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[project.optional-dependencies]
|
|
53
|
+
dev = ["black", "bumpver", "pip-tools", "wheel"]
|
|
54
|
+
pdf = [
|
|
55
|
+
"pypdf2", # PDF text extraction fallback
|
|
56
|
+
"instructor[multimodal]", # Multimodal PDF processing
|
|
57
|
+
"weasyprint", # PDF generation from HTML/markdown
|
|
58
|
+
"markdown", # Markdown processing for conversion
|
|
59
|
+
]
|
|
60
|
+
docx = [
|
|
61
|
+
"python-docx", # Basic DOCX support
|
|
62
|
+
"docling", # Advanced DOCX conversion via docling
|
|
63
|
+
"markdown", # Markdown processing
|
|
64
|
+
"weasyprint", # PDF generation
|
|
65
|
+
]
|
|
66
|
+
docs = [
|
|
67
|
+
"pypdf2", # PDF text extraction
|
|
68
|
+
"python-docx", # DOCX support
|
|
69
|
+
"instructor[multimodal]", # Multimodal processing
|
|
70
|
+
"docling", # Advanced document conversion
|
|
71
|
+
"weasyprint", # PDF generation from HTML/markdown
|
|
72
|
+
"markdown", # Markdown processing and extensions
|
|
73
|
+
]
|
|
74
|
+
# Legacy support - kept for backward compatibility
|
|
75
|
+
unstructured = ["unstructured[all-docs]"]
|
|
76
|
+
docling = ["docling", "markdown", "weasyprint"]
|
|
77
|
+
advanced = [
|
|
78
|
+
"pypdf2",
|
|
79
|
+
"python-docx",
|
|
80
|
+
"instructor[multimodal]",
|
|
81
|
+
"docling",
|
|
82
|
+
"weasyprint",
|
|
83
|
+
"markdown",
|
|
84
|
+
"unstructured[all-docs]",
|
|
85
|
+
]
|
|
86
|
+
mkdocs = [
|
|
87
|
+
"mkdocs",
|
|
88
|
+
"mkdocs-material",
|
|
89
|
+
"mkdocstrings[markdown]",
|
|
90
|
+
"mkdocstrings[python]",
|
|
91
|
+
"mkdocs-git-revision-date-localized-plugin",
|
|
92
|
+
"mkdocs-material[imaging]",
|
|
93
|
+
"mkdocs-mermaid2-plugin",
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
[tool.bumpver]
|
|
97
|
+
current_version = "0.4.8"
|
|
98
|
+
version_pattern = "MAJOR.MINOR.PATCH"
|
|
99
|
+
commit_message = "Bump version {old_version} -> {new_version}"
|
|
100
|
+
commit = true
|
|
101
|
+
tag = true
|
|
102
|
+
push = true
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
[tool.bumpver.file_patterns]
|
|
106
|
+
"pyproject.toml" = ['current_version = "{version}"', 'version = "{version}"']
|
|
107
|
+
"structx/__init__.py" = ['__version__ = "{version}"']
|
|
108
|
+
"README.md" = ["PyPi-{version}"]
|
|
109
|
+
|
|
110
|
+
[tool.setuptools.packages.find]
|
|
111
|
+
# Only package the library code under structx; exclude MkDocs overrides and others
|
|
112
|
+
include = ["structx*"]
|
structx-0.4.8/setup.cfg
ADDED