structx 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {structx-0.5.0/structx.egg-info → structx-0.6.0}/PKG-INFO +42 -39
- {structx-0.5.0 → structx-0.6.0}/README.md +18 -14
- {structx-0.5.0 → structx-0.6.0}/pyproject.toml +28 -22
- {structx-0.5.0 → structx-0.6.0}/structx/__init__.py +7 -6
- structx-0.6.0/structx/core/config.py +101 -0
- structx-0.6.0/structx/core/exceptions.py +34 -0
- structx-0.6.0/structx/core/input.py +58 -0
- structx-0.6.0/structx/core/models.py +189 -0
- structx-0.6.0/structx/core/type_system.py +367 -0
- {structx-0.5.0 → structx-0.6.0}/structx/extraction/__init__.py +0 -16
- {structx-0.5.0 → structx-0.6.0}/structx/extraction/core/__init__.py +1 -2
- structx-0.6.0/structx/extraction/core/llm_core.py +249 -0
- structx-0.6.0/structx/extraction/engines/extraction_engine.py +239 -0
- structx-0.6.0/structx/extraction/extractor.py +709 -0
- structx-0.6.0/structx/extraction/generator.py +123 -0
- structx-0.6.0/structx/extraction/processors/__init__.py +8 -0
- structx-0.6.0/structx/extraction/processors/batch_processor.py +159 -0
- structx-0.6.0/structx/extraction/processors/content_analyzer.py +22 -0
- structx-0.6.0/structx/extraction/processors/input_processor.py +107 -0
- structx-0.6.0/structx/extraction/processors/model_operations.py +219 -0
- structx-0.6.0/structx/extraction/result_manager.py +94 -0
- {structx-0.5.0 → structx-0.6.0}/structx/utils/file_reader.py +76 -27
- structx-0.6.0/structx/utils/helpers.py +54 -0
- structx-0.6.0/structx/utils/prompts.py +96 -0
- structx-0.6.0/structx/utils/usage.py +162 -0
- {structx-0.5.0 → structx-0.6.0/structx.egg-info}/PKG-INFO +42 -39
- {structx-0.5.0 → structx-0.6.0}/structx.egg-info/SOURCES.txt +12 -5
- structx-0.6.0/structx.egg-info/requires.txt +31 -0
- structx-0.6.0/tests/test_config_and_model_utils.py +73 -0
- structx-0.6.0/tests/test_content_analyzer.py +33 -0
- {structx-0.5.0 → structx-0.6.0}/tests/test_document_conversion_integration.py +7 -10
- structx-0.6.0/tests/test_extraction_engine.py +35 -0
- structx-0.6.0/tests/test_extractor_async.py +87 -0
- structx-0.6.0/tests/test_extractor_flow.py +87 -0
- {structx-0.5.0 → structx-0.6.0}/tests/test_file_reader.py +57 -11
- structx-0.6.0/tests/test_input_and_batch_processors.py +163 -0
- structx-0.6.0/tests/test_live_llm.py +74 -0
- structx-0.6.0/tests/test_llm_configuration.py +286 -0
- structx-0.6.0/tests/test_result_manager_and_helpers.py +177 -0
- structx-0.6.0/tests/test_type_system_and_planning.py +290 -0
- structx-0.6.0/tests/test_usage.py +98 -0
- structx-0.5.0/structx/core/config.py +0 -126
- structx-0.5.0/structx/core/exceptions.py +0 -34
- structx-0.5.0/structx/core/models.py +0 -145
- structx-0.5.0/structx/extraction/core/llm_core.py +0 -206
- structx-0.5.0/structx/extraction/core/model_utils.py +0 -143
- structx-0.5.0/structx/extraction/engines/extraction_engine.py +0 -221
- structx-0.5.0/structx/extraction/extractor.py +0 -557
- structx-0.5.0/structx/extraction/generator.py +0 -232
- structx-0.5.0/structx/extraction/processors/__init__.py +0 -7
- structx-0.5.0/structx/extraction/processors/content_analyzer.py +0 -164
- structx-0.5.0/structx/extraction/processors/data_processor.py +0 -175
- structx-0.5.0/structx/extraction/processors/model_operations.py +0 -233
- structx-0.5.0/structx/extraction/result_manager.py +0 -125
- structx-0.5.0/structx/utils/constants.py +0 -36
- structx-0.5.0/structx/utils/helpers.py +0 -233
- structx-0.5.0/structx/utils/prompts.py +0 -183
- structx-0.5.0/structx/utils/usage.py +0 -327
- structx-0.5.0/structx.egg-info/requires.txt +0 -31
- structx-0.5.0/tests/test_config_and_model_utils.py +0 -66
- structx-0.5.0/tests/test_content_analyzer.py +0 -80
- structx-0.5.0/tests/test_data_processor.py +0 -129
- structx-0.5.0/tests/test_extractor_async.py +0 -109
- structx-0.5.0/tests/test_result_manager_and_helpers.py +0 -78
- {structx-0.5.0 → structx-0.6.0}/LICENSE +0 -0
- {structx-0.5.0 → structx-0.6.0}/setup.cfg +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx/core/__init__.py +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx/extraction/engines/__init__.py +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx/utils/__init__.py +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx/utils/types.py +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx.egg-info/dependency_links.txt +0 -0
- {structx-0.5.0 → structx-0.6.0}/structx.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: structx
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
5
|
Author-email: blacksuan19 <py@aolabs.dev>
|
|
6
6
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -11,7 +11,6 @@ Classifier: Operating System :: OS Independent
|
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
14
|
Classifier: Programming Language :: Python :: 3
|
|
16
15
|
Classifier: Topic :: Database
|
|
17
16
|
Classifier: Topic :: Documentation
|
|
@@ -29,34 +28,34 @@ Classifier: Typing :: Typed
|
|
|
29
28
|
Requires-Python: >=3.10
|
|
30
29
|
Description-Content-Type: text/markdown
|
|
31
30
|
License-File: LICENSE
|
|
32
|
-
Requires-Dist: instructor
|
|
33
|
-
Requires-Dist: litellm
|
|
34
|
-
Requires-Dist: loguru
|
|
35
|
-
Requires-Dist: omegaconf
|
|
31
|
+
Requires-Dist: instructor>=1.15.4
|
|
32
|
+
Requires-Dist: litellm>=1.91.1
|
|
33
|
+
Requires-Dist: loguru>=0.7.3
|
|
36
34
|
Requires-Dist: openpyxl>=3.1.5
|
|
37
|
-
Requires-Dist: pandas
|
|
38
|
-
Requires-Dist: pydantic
|
|
35
|
+
Requires-Dist: pandas>=2.2.3
|
|
36
|
+
Requires-Dist: pydantic>=2.11.7
|
|
37
|
+
Requires-Dist: pydantic-settings[yaml]>=2.14.2
|
|
39
38
|
Requires-Dist: tabulate>=0.9.0
|
|
40
|
-
Requires-Dist: tenacity
|
|
41
|
-
Requires-Dist: tqdm
|
|
39
|
+
Requires-Dist: tenacity>=9.1.2
|
|
40
|
+
Requires-Dist: tqdm>=4.68.4
|
|
42
41
|
Provides-Extra: dev
|
|
43
|
-
Requires-Dist: black; extra == "dev"
|
|
44
|
-
Requires-Dist: bumpver; extra == "dev"
|
|
45
|
-
Requires-Dist: pip-tools; extra == "dev"
|
|
46
|
-
Requires-Dist: pytest; extra == "dev"
|
|
47
|
-
Requires-Dist: wheel; extra == "dev"
|
|
42
|
+
Requires-Dist: black>=25.1.0; extra == "dev"
|
|
43
|
+
Requires-Dist: bumpver>=2024.1130; extra == "dev"
|
|
44
|
+
Requires-Dist: pip-tools>=7.4.1; extra == "dev"
|
|
45
|
+
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
46
|
+
Requires-Dist: wheel>=0.45.1; extra == "dev"
|
|
48
47
|
Provides-Extra: docs
|
|
49
|
-
Requires-Dist: docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]; extra == "docs"
|
|
50
|
-
Requires-Dist: torch; extra == "docs"
|
|
51
|
-
Requires-Dist: torchvision; extra == "docs"
|
|
52
|
-
Requires-Dist: weasyprint; extra == "docs"
|
|
48
|
+
Requires-Dist: docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]>=2.111.0; extra == "docs"
|
|
49
|
+
Requires-Dist: torch>=2.13.0; extra == "docs"
|
|
50
|
+
Requires-Dist: torchvision>=0.28.0; extra == "docs"
|
|
51
|
+
Requires-Dist: weasyprint>=69.0; extra == "docs"
|
|
53
52
|
Provides-Extra: mkdocs
|
|
54
|
-
Requires-Dist: mkdocs; extra == "mkdocs"
|
|
55
|
-
Requires-Dist: mkdocs-material; extra == "mkdocs"
|
|
56
|
-
Requires-Dist: mkdocstrings[python]; extra == "mkdocs"
|
|
57
|
-
Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == "mkdocs"
|
|
58
|
-
Requires-Dist: mkdocs-material[imaging]; extra == "mkdocs"
|
|
59
|
-
Requires-Dist: mkdocs-mermaid2-plugin; extra == "mkdocs"
|
|
53
|
+
Requires-Dist: mkdocs>=1.6.1; extra == "mkdocs"
|
|
54
|
+
Requires-Dist: mkdocs-material>=9.7.6; extra == "mkdocs"
|
|
55
|
+
Requires-Dist: mkdocstrings[python]>=0.29.1; extra == "mkdocs"
|
|
56
|
+
Requires-Dist: mkdocs-git-revision-date-localized-plugin>=1.4.5; extra == "mkdocs"
|
|
57
|
+
Requires-Dist: mkdocs-material[imaging]>=9.7.6; extra == "mkdocs"
|
|
58
|
+
Requires-Dist: mkdocs-mermaid2-plugin>=1.2.3; extra == "mkdocs"
|
|
60
59
|
Dynamic: license-file
|
|
61
60
|
|
|
62
61
|
# structx
|
|
@@ -65,13 +64,13 @@ Advanced structured data extraction from any document using LLMs with multimodal
|
|
|
65
64
|
support.
|
|
66
65
|
|
|
67
66
|
[](https://structx.aolabs.dev "Documentation")
|
|
68
|
-
[](https://pypi.org/project/structx "Package")
|
|
69
68
|
[](# "Build with GitHub Actions")
|
|
70
69
|
|
|
71
70
|
`structx` is a powerful Python library for extracting structured data from text,
|
|
72
|
-
tables, and documents using Large Language Models (LLMs).
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
tables, and documents using Large Language Models (LLMs). It passes existing
|
|
72
|
+
PDFs directly to vision-capable models; the optional `docs` extra converts other
|
|
73
|
+
document formats to PDF first.
|
|
75
74
|
|
|
76
75
|
## 🔔 Package rename notice (PyPI)
|
|
77
76
|
|
|
@@ -90,7 +89,7 @@ pip install -U structx
|
|
|
90
89
|
```
|
|
91
90
|
|
|
92
91
|
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
93
|
-
with `structx`.
|
|
92
|
+
with `structx`. Install `structx[docs]` for non-PDF document conversion.
|
|
94
93
|
|
|
95
94
|
## ✨ Key Features
|
|
96
95
|
|
|
@@ -102,8 +101,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
102
101
|
PDFs and images
|
|
103
102
|
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
104
103
|
results
|
|
105
|
-
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet
|
|
106
|
-
|
|
104
|
+
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet, raw text, and existing
|
|
105
|
+
PDFs in the base install; DOCX, PPTX, images, and more via `structx[docs]`
|
|
107
106
|
|
|
108
107
|
### 🚀 **Intelligent Data Extraction**
|
|
109
108
|
|
|
@@ -117,11 +116,12 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
117
116
|
|
|
118
117
|
### ⚡ **Performance & Reliability**
|
|
119
118
|
|
|
120
|
-
- **🚀 High-Performance Processing**:
|
|
119
|
+
- **🚀 High-Performance Processing**: Threaded sync and native async row requests
|
|
121
120
|
- **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
|
|
122
121
|
backoff
|
|
123
122
|
- **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
|
|
124
|
-
-
|
|
123
|
+
- **Flexible Configuration**: Model settings from arguments, YAML, environment
|
|
124
|
+
variables, dotenv files, and secrets through Pydantic Settings
|
|
125
125
|
- **🔌 Multiple LLM Providers**: Support through litellm integration
|
|
126
126
|
|
|
127
127
|
## Installation
|
|
@@ -130,7 +130,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
130
130
|
pip install structx
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
-
For
|
|
133
|
+
For converting DOCX, PowerPoint, OpenDocument, markup, image, and other
|
|
134
|
+
non-PDF document formats:
|
|
134
135
|
|
|
135
136
|
```bash
|
|
136
137
|
pip install "structx[docs]"
|
|
@@ -166,14 +167,14 @@ result = extractor.extract(
|
|
|
166
167
|
)
|
|
167
168
|
|
|
168
169
|
# Access results
|
|
169
|
-
print(f"
|
|
170
|
+
print(f"Successful rows: {result.success_count}")
|
|
170
171
|
print(result.data[0].model_dump_json(indent=2))
|
|
171
172
|
```
|
|
172
173
|
|
|
173
174
|
### 📄 Document Processing with Multimodal Support
|
|
174
175
|
|
|
175
176
|
Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
|
|
176
|
-
be passed directly through the multimodal path.
|
|
177
|
+
be passed directly through the multimodal path with the base installation.
|
|
177
178
|
|
|
178
179
|
```python
|
|
179
180
|
# Process a PDF invoice through the multimodal pipeline
|
|
@@ -193,10 +194,11 @@ result = extractor.extract(
|
|
|
193
194
|
|
|
194
195
|
```python
|
|
195
196
|
# Check token usage for cost monitoring
|
|
196
|
-
usage = result.
|
|
197
|
+
usage = result.usage
|
|
197
198
|
if usage:
|
|
198
199
|
print(f"Total tokens: {usage.total_tokens}")
|
|
199
|
-
|
|
200
|
+
for step, calls in usage.steps.items():
|
|
201
|
+
print(step.value, [call.total_tokens for call in calls])
|
|
200
202
|
```
|
|
201
203
|
|
|
202
204
|
## 🚀 Why Multimodal PDF Processing?
|
|
@@ -221,6 +223,7 @@ For comprehensive documentation, examples, and guides, visit our
|
|
|
221
223
|
|
|
222
224
|
- [Getting Started](https://structx.aolabs.dev/getting-started)
|
|
223
225
|
- [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
|
|
226
|
+
- [Working with Results](https://structx.aolabs.dev/guides/working-with-results)
|
|
224
227
|
- [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
|
|
225
228
|
- [Async Operations](https://structx.aolabs.dev/guides/async-operations)
|
|
226
229
|
- [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
|
|
@@ -4,13 +4,13 @@ Advanced structured data extraction from any document using LLMs with multimodal
|
|
|
4
4
|
support.
|
|
5
5
|
|
|
6
6
|
[](https://structx.aolabs.dev "Documentation")
|
|
7
|
-
[](https://pypi.org/project/structx "Package")
|
|
8
8
|
[](# "Build with GitHub Actions")
|
|
9
9
|
|
|
10
10
|
`structx` is a powerful Python library for extracting structured data from text,
|
|
11
|
-
tables, and documents using Large Language Models (LLMs).
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
tables, and documents using Large Language Models (LLMs). It passes existing
|
|
12
|
+
PDFs directly to vision-capable models; the optional `docs` extra converts other
|
|
13
|
+
document formats to PDF first.
|
|
14
14
|
|
|
15
15
|
## 🔔 Package rename notice (PyPI)
|
|
16
16
|
|
|
@@ -29,7 +29,7 @@ pip install -U structx
|
|
|
29
29
|
```
|
|
30
30
|
|
|
31
31
|
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
32
|
-
with `structx`.
|
|
32
|
+
with `structx`. Install `structx[docs]` for non-PDF document conversion.
|
|
33
33
|
|
|
34
34
|
## ✨ Key Features
|
|
35
35
|
|
|
@@ -41,8 +41,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
41
41
|
PDFs and images
|
|
42
42
|
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
43
43
|
results
|
|
44
|
-
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet
|
|
45
|
-
|
|
44
|
+
- **📊 Flexible File Support**: CSV, Excel, JSON, Parquet, raw text, and existing
|
|
45
|
+
PDFs in the base install; DOCX, PPTX, images, and more via `structx[docs]`
|
|
46
46
|
|
|
47
47
|
### 🚀 **Intelligent Data Extraction**
|
|
48
48
|
|
|
@@ -56,11 +56,12 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
56
56
|
|
|
57
57
|
### ⚡ **Performance & Reliability**
|
|
58
58
|
|
|
59
|
-
- **🚀 High-Performance Processing**:
|
|
59
|
+
- **🚀 High-Performance Processing**: Threaded sync and native async row requests
|
|
60
60
|
- **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
|
|
61
61
|
backoff
|
|
62
62
|
- **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
|
|
63
|
-
-
|
|
63
|
+
- **Flexible Configuration**: Model settings from arguments, YAML, environment
|
|
64
|
+
variables, dotenv files, and secrets through Pydantic Settings
|
|
64
65
|
- **🔌 Multiple LLM Providers**: Support through litellm integration
|
|
65
66
|
|
|
66
67
|
## Installation
|
|
@@ -69,7 +70,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
|
|
|
69
70
|
pip install structx
|
|
70
71
|
```
|
|
71
72
|
|
|
72
|
-
For
|
|
73
|
+
For converting DOCX, PowerPoint, OpenDocument, markup, image, and other
|
|
74
|
+
non-PDF document formats:
|
|
73
75
|
|
|
74
76
|
```bash
|
|
75
77
|
pip install "structx[docs]"
|
|
@@ -105,14 +107,14 @@ result = extractor.extract(
|
|
|
105
107
|
)
|
|
106
108
|
|
|
107
109
|
# Access results
|
|
108
|
-
print(f"
|
|
110
|
+
print(f"Successful rows: {result.success_count}")
|
|
109
111
|
print(result.data[0].model_dump_json(indent=2))
|
|
110
112
|
```
|
|
111
113
|
|
|
112
114
|
### 📄 Document Processing with Multimodal Support
|
|
113
115
|
|
|
114
116
|
Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
|
|
115
|
-
be passed directly through the multimodal path.
|
|
117
|
+
be passed directly through the multimodal path with the base installation.
|
|
116
118
|
|
|
117
119
|
```python
|
|
118
120
|
# Process a PDF invoice through the multimodal pipeline
|
|
@@ -132,10 +134,11 @@ result = extractor.extract(
|
|
|
132
134
|
|
|
133
135
|
```python
|
|
134
136
|
# Check token usage for cost monitoring
|
|
135
|
-
usage = result.
|
|
137
|
+
usage = result.usage
|
|
136
138
|
if usage:
|
|
137
139
|
print(f"Total tokens: {usage.total_tokens}")
|
|
138
|
-
|
|
140
|
+
for step, calls in usage.steps.items():
|
|
141
|
+
print(step.value, [call.total_tokens for call in calls])
|
|
139
142
|
```
|
|
140
143
|
|
|
141
144
|
## 🚀 Why Multimodal PDF Processing?
|
|
@@ -160,6 +163,7 @@ For comprehensive documentation, examples, and guides, visit our
|
|
|
160
163
|
|
|
161
164
|
- [Getting Started](https://structx.aolabs.dev/getting-started)
|
|
162
165
|
- [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
|
|
166
|
+
- [Working with Results](https://structx.aolabs.dev/guides/working-with-results)
|
|
163
167
|
- [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
|
|
164
168
|
- [Async Operations](https://structx.aolabs.dev/guides/async-operations)
|
|
165
169
|
- [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "structx"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Structured data extraction from text using LLMs and dynamic model generation"
|
|
9
9
|
authors = [{ name = "blacksuan19", email = "py@aolabs.dev" }]
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -19,7 +19,6 @@ classifiers = [
|
|
|
19
19
|
"Programming Language :: Python :: 3.10",
|
|
20
20
|
"Programming Language :: Python :: 3.11",
|
|
21
21
|
"Programming Language :: Python :: 3.12",
|
|
22
|
-
"Programming Language :: Python :: 3.9",
|
|
23
22
|
"Programming Language :: Python :: 3",
|
|
24
23
|
"Topic :: Database",
|
|
25
24
|
"Topic :: Documentation",
|
|
@@ -37,40 +36,46 @@ classifiers = [
|
|
|
37
36
|
]
|
|
38
37
|
|
|
39
38
|
dependencies = [
|
|
40
|
-
"instructor",
|
|
41
|
-
"litellm",
|
|
42
|
-
"loguru",
|
|
43
|
-
"omegaconf",
|
|
39
|
+
"instructor>=1.15.4",
|
|
40
|
+
"litellm>=1.91.1",
|
|
41
|
+
"loguru>=0.7.3",
|
|
44
42
|
"openpyxl>=3.1.5",
|
|
45
|
-
"pandas",
|
|
46
|
-
"pydantic",
|
|
43
|
+
"pandas>=2.2.3",
|
|
44
|
+
"pydantic>=2.11.7",
|
|
45
|
+
"pydantic-settings[yaml]>=2.14.2",
|
|
47
46
|
"tabulate>=0.9.0", # required for pandas .to_markdown()
|
|
48
|
-
"tenacity",
|
|
49
|
-
"tqdm",
|
|
47
|
+
"tenacity>=9.1.2",
|
|
48
|
+
"tqdm>=4.68.4",
|
|
50
49
|
]
|
|
51
50
|
|
|
52
51
|
[project.optional-dependencies]
|
|
53
|
-
dev = [
|
|
52
|
+
dev = [
|
|
53
|
+
"black>=25.1.0",
|
|
54
|
+
"bumpver>=2024.1130",
|
|
55
|
+
"pip-tools>=7.4.1",
|
|
56
|
+
"pytest>=9.1.1",
|
|
57
|
+
"wheel>=0.45.1",
|
|
58
|
+
]
|
|
54
59
|
|
|
55
60
|
# Optional document processing support. Kept out of the base install because
|
|
56
61
|
# Docling's local model pipeline requires PyTorch.
|
|
57
62
|
# The format-pdf extra is still needed because DocumentConverter imports its
|
|
58
63
|
# PDF backend dependencies even when existing PDFs are passed through directly.
|
|
59
64
|
docs = [
|
|
60
|
-
"docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]",
|
|
61
|
-
"torch",
|
|
62
|
-
"torchvision",
|
|
63
|
-
"weasyprint",
|
|
65
|
+
"docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]>=2.111.0",
|
|
66
|
+
"torch>=2.13.0",
|
|
67
|
+
"torchvision>=0.28.0",
|
|
68
|
+
"weasyprint>=69.0",
|
|
64
69
|
]
|
|
65
70
|
|
|
66
71
|
# Documentation site tooling.
|
|
67
72
|
mkdocs = [
|
|
68
|
-
"mkdocs",
|
|
69
|
-
"mkdocs-material",
|
|
70
|
-
"mkdocstrings[python]",
|
|
71
|
-
"mkdocs-git-revision-date-localized-plugin",
|
|
72
|
-
"mkdocs-material[imaging]",
|
|
73
|
-
"mkdocs-mermaid2-plugin",
|
|
73
|
+
"mkdocs>=1.6.1",
|
|
74
|
+
"mkdocs-material>=9.7.6",
|
|
75
|
+
"mkdocstrings[python]>=0.29.1",
|
|
76
|
+
"mkdocs-git-revision-date-localized-plugin>=1.4.5",
|
|
77
|
+
"mkdocs-material[imaging]>=9.7.6",
|
|
78
|
+
"mkdocs-mermaid2-plugin>=1.2.3",
|
|
74
79
|
]
|
|
75
80
|
|
|
76
81
|
[[tool.uv.index]]
|
|
@@ -87,7 +92,7 @@ torchvision = [
|
|
|
87
92
|
]
|
|
88
93
|
|
|
89
94
|
[tool.bumpver]
|
|
90
|
-
current_version = "0.
|
|
95
|
+
current_version = "0.6.0"
|
|
91
96
|
version_pattern = "MAJOR.MINOR.PATCH"
|
|
92
97
|
commit_message = "Bump version {old_version} -> {new_version}"
|
|
93
98
|
commit = true
|
|
@@ -106,6 +111,7 @@ include = ["structx*"]
|
|
|
106
111
|
|
|
107
112
|
[tool.pytest.ini_options]
|
|
108
113
|
filterwarnings = [
|
|
114
|
+
"ignore:There is no current event loop:DeprecationWarning:litellm\\._service_logger",
|
|
109
115
|
"ignore:This field is deprecated.*:DeprecationWarning:docling\\.pipeline\\.standard_pdf_pipeline",
|
|
110
116
|
"ignore:torch\\.ao\\.quantization is deprecated.*:DeprecationWarning:torch\\.ao\\.quantization\\.quantize",
|
|
111
117
|
"ignore:torch\\.quantize_per_tensor.*:UserWarning:torch\\.ao\\.nn\\.quantized\\.dynamic\\.modules\\.rnn",
|
|
@@ -4,21 +4,22 @@ structx: Structured data extraction using LLMs
|
|
|
4
4
|
|
|
5
5
|
from structx.core.config import ExtractionConfig, StepConfig
|
|
6
6
|
from structx.core.models import (
|
|
7
|
-
|
|
7
|
+
ExtractionPlan,
|
|
8
8
|
ExtractionRequest,
|
|
9
|
+
ExtractionResult,
|
|
9
10
|
ModelField,
|
|
10
|
-
|
|
11
|
+
RowResult,
|
|
11
12
|
)
|
|
12
13
|
from structx.extraction.extractor import Extractor
|
|
13
14
|
|
|
14
|
-
__version__ = "0.
|
|
15
|
+
__version__ = "0.6.0"
|
|
15
16
|
__all__ = [
|
|
16
17
|
"Extractor",
|
|
17
18
|
"ExtractionConfig",
|
|
18
19
|
"StepConfig",
|
|
19
20
|
"ModelField",
|
|
20
|
-
"
|
|
21
|
-
"QueryRefinement",
|
|
22
|
-
"ExtractionGuide",
|
|
21
|
+
"ExtractionPlan",
|
|
23
22
|
"ExtractionRequest",
|
|
23
|
+
"ExtractionResult",
|
|
24
|
+
"RowResult",
|
|
24
25
|
]
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Pydantic Settings configuration for model-backed extraction steps."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Type
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
8
|
+
from pydantic_settings import (
|
|
9
|
+
BaseSettings,
|
|
10
|
+
PydanticBaseSettingsSource,
|
|
11
|
+
SettingsConfigDict,
|
|
12
|
+
YamlConfigSettingsSource,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from structx.core.exceptions import ConfigurationError
|
|
16
|
+
from structx.utils.types import DictStrAny
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class StepConfig(BaseModel):
|
|
20
|
+
"""Provider-specific completion parameters for one extraction step."""
|
|
21
|
+
|
|
22
|
+
model_config = ConfigDict(validate_assignment=True, extra="allow")
|
|
23
|
+
|
|
24
|
+
@model_validator(mode="before")
|
|
25
|
+
@classmethod
|
|
26
|
+
def decode_scalar_values(cls, value: Any) -> Any:
|
|
27
|
+
"""Decode numeric and boolean values supplied by nested environment keys."""
|
|
28
|
+
if not isinstance(value, dict):
|
|
29
|
+
return value
|
|
30
|
+
decoded = {}
|
|
31
|
+
for key, item in value.items():
|
|
32
|
+
if isinstance(item, str):
|
|
33
|
+
try:
|
|
34
|
+
item = json.loads(item)
|
|
35
|
+
except json.JSONDecodeError:
|
|
36
|
+
pass
|
|
37
|
+
decoded[key] = item
|
|
38
|
+
return decoded
|
|
39
|
+
|
|
40
|
+
def options(self) -> DictStrAny:
|
|
41
|
+
"""Return explicitly configured provider parameters."""
|
|
42
|
+
return self.model_dump(exclude_none=True)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ExtractionConfig(BaseSettings):
|
|
46
|
+
"""Settings loaded from arguments, environment, dotenv, YAML, or secrets.
|
|
47
|
+
|
|
48
|
+
Source priority is constructor values, environment variables, ``.env``,
|
|
49
|
+
YAML, then file secrets. Environment and secret names use the ``STRUCTX_``
|
|
50
|
+
prefix.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
planning: StepConfig = Field(default_factory=StepConfig)
|
|
54
|
+
extraction: StepConfig = Field(default_factory=StepConfig)
|
|
55
|
+
|
|
56
|
+
model_config = SettingsConfigDict(
|
|
57
|
+
extra="ignore",
|
|
58
|
+
env_prefix="STRUCTX_",
|
|
59
|
+
env_nested_delimiter="__",
|
|
60
|
+
env_file=".env",
|
|
61
|
+
env_file_encoding="utf-8",
|
|
62
|
+
yaml_file=None,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def settings_customise_sources(
|
|
67
|
+
cls,
|
|
68
|
+
settings_cls: Type[BaseSettings],
|
|
69
|
+
init_settings: PydanticBaseSettingsSource,
|
|
70
|
+
env_settings: PydanticBaseSettingsSource,
|
|
71
|
+
dotenv_settings: PydanticBaseSettingsSource,
|
|
72
|
+
file_secret_settings: PydanticBaseSettingsSource,
|
|
73
|
+
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
74
|
+
"""Use conventional precedence with YAML below dotenv values."""
|
|
75
|
+
return (
|
|
76
|
+
init_settings,
|
|
77
|
+
env_settings,
|
|
78
|
+
dotenv_settings,
|
|
79
|
+
YamlConfigSettingsSource(settings_cls),
|
|
80
|
+
file_secret_settings,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def from_yaml(cls, path: str | Path, **overrides: Any) -> "ExtractionConfig":
|
|
85
|
+
"""Load settings from a YAML file with optional highest-priority overrides."""
|
|
86
|
+
yaml_path = Path(path).expanduser()
|
|
87
|
+
if not yaml_path.is_file():
|
|
88
|
+
raise ConfigurationError(f"Configuration file not found: {yaml_path}")
|
|
89
|
+
|
|
90
|
+
class YamlExtractionConfig(cls):
|
|
91
|
+
model_config = SettingsConfigDict(
|
|
92
|
+
**{**cls.model_config, "yaml_file": yaml_path}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
return YamlExtractionConfig(**overrides)
|
|
96
|
+
|
|
97
|
+
def for_step(self, step: str) -> DictStrAny:
|
|
98
|
+
"""Return provider kwargs for ``planning`` or ``extraction``."""
|
|
99
|
+
if step not in {"planning", "extraction"}:
|
|
100
|
+
raise ConfigurationError(f"Unknown extraction step: {step}")
|
|
101
|
+
return getattr(self, step).options()
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
class StructXError(Exception):
|
|
2
|
+
"""Base exception for errors raised directly by Structx."""
|
|
3
|
+
|
|
4
|
+
pass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ConfigurationError(StructXError):
|
|
8
|
+
"""Raised for invalid Structx configuration or runtime settings."""
|
|
9
|
+
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ExtractionError(StructXError):
|
|
14
|
+
"""Raised when a public extraction operation cannot complete."""
|
|
15
|
+
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ValidationError(StructXError):
|
|
20
|
+
"""Raised for Structx-specific input validation failures."""
|
|
21
|
+
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ModelGenerationError(StructXError):
|
|
26
|
+
"""Raised when an extraction schema cannot become a Pydantic model."""
|
|
27
|
+
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FileError(StructXError):
|
|
32
|
+
"""Raised for invalid paths, unsupported files, or conversion failures."""
|
|
33
|
+
|
|
34
|
+
pass
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Typed contracts for prepared extraction inputs."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Dict, List, Optional, Union
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class PdfRow:
|
|
12
|
+
"""A PDF payload associated with one source row.
|
|
13
|
+
|
|
14
|
+
Attributes:
|
|
15
|
+
pdf_path: PDF sent to Instructor's multimodal input.
|
|
16
|
+
source: Original user-supplied document path. For converted documents,
|
|
17
|
+
this differs from ``pdf_path``.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
pdf_path: Path
|
|
21
|
+
source: Path
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
RowPayload = Union[str, PdfRow]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class PreparedInput:
|
|
29
|
+
"""Normalized extraction input and its owned resources.
|
|
30
|
+
|
|
31
|
+
``InputProcessor`` creates this contract before planning or row processing.
|
|
32
|
+
Application code normally receives an ``ExtractionResult`` instead; this
|
|
33
|
+
type is primarily useful when calling ``FileReader`` directly.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
dataframe: Source rows retained for planning, provenance, and optional
|
|
37
|
+
DataFrame output.
|
|
38
|
+
pdf_rows: Positional row mappings for multimodal PDF payloads.
|
|
39
|
+
planning_sample: Text extracted once from a converted document for
|
|
40
|
+
schema planning. Existing PDFs usually leave this unset and are
|
|
41
|
+
attached directly to the planning request.
|
|
42
|
+
owned_paths: Temporary files that must be deleted after processing.
|
|
43
|
+
``InputProcessor.prepared`` handles this automatically.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
dataframe: pd.DataFrame
|
|
47
|
+
pdf_rows: Dict[int, PdfRow] = field(default_factory=dict)
|
|
48
|
+
planning_sample: Optional[str] = None
|
|
49
|
+
owned_paths: List[Path] = field(default_factory=list)
|
|
50
|
+
|
|
51
|
+
def row_payload(
|
|
52
|
+
self, position: int, row: pd.Series, target_columns: List[str]
|
|
53
|
+
) -> RowPayload:
|
|
54
|
+
"""Build the text or PDF payload for one positional input row."""
|
|
55
|
+
pdf_row = self.pdf_rows.get(position)
|
|
56
|
+
if pdf_row is not None:
|
|
57
|
+
return pdf_row
|
|
58
|
+
return row[target_columns].to_markdown()
|