structx 0.5.1__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {structx-0.5.1/structx.egg-info → structx-0.6.0}/PKG-INFO +44 -41
  2. {structx-0.5.1 → structx-0.6.0}/README.md +18 -14
  3. {structx-0.5.1 → structx-0.6.0}/pyproject.toml +28 -28
  4. {structx-0.5.1 → structx-0.6.0}/structx/__init__.py +7 -6
  5. structx-0.6.0/structx/core/config.py +101 -0
  6. structx-0.6.0/structx/core/exceptions.py +34 -0
  7. structx-0.6.0/structx/core/input.py +58 -0
  8. structx-0.6.0/structx/core/models.py +189 -0
  9. structx-0.6.0/structx/core/type_system.py +367 -0
  10. {structx-0.5.1 → structx-0.6.0}/structx/extraction/__init__.py +0 -16
  11. {structx-0.5.1 → structx-0.6.0}/structx/extraction/core/__init__.py +1 -2
  12. structx-0.6.0/structx/extraction/core/llm_core.py +249 -0
  13. structx-0.6.0/structx/extraction/engines/extraction_engine.py +239 -0
  14. structx-0.6.0/structx/extraction/extractor.py +709 -0
  15. structx-0.6.0/structx/extraction/generator.py +123 -0
  16. structx-0.6.0/structx/extraction/processors/__init__.py +8 -0
  17. structx-0.6.0/structx/extraction/processors/batch_processor.py +159 -0
  18. structx-0.6.0/structx/extraction/processors/content_analyzer.py +22 -0
  19. structx-0.6.0/structx/extraction/processors/input_processor.py +107 -0
  20. structx-0.6.0/structx/extraction/processors/model_operations.py +219 -0
  21. structx-0.6.0/structx/extraction/result_manager.py +94 -0
  22. {structx-0.5.1 → structx-0.6.0}/structx/utils/file_reader.py +76 -27
  23. structx-0.6.0/structx/utils/helpers.py +54 -0
  24. structx-0.6.0/structx/utils/prompts.py +96 -0
  25. structx-0.6.0/structx/utils/usage.py +162 -0
  26. {structx-0.5.1 → structx-0.6.0/structx.egg-info}/PKG-INFO +44 -41
  27. {structx-0.5.1 → structx-0.6.0}/structx.egg-info/SOURCES.txt +12 -5
  28. structx-0.6.0/structx.egg-info/requires.txt +31 -0
  29. structx-0.6.0/tests/test_config_and_model_utils.py +73 -0
  30. structx-0.6.0/tests/test_content_analyzer.py +33 -0
  31. {structx-0.5.1 → structx-0.6.0}/tests/test_document_conversion_integration.py +7 -10
  32. structx-0.6.0/tests/test_extraction_engine.py +35 -0
  33. structx-0.6.0/tests/test_extractor_async.py +87 -0
  34. structx-0.6.0/tests/test_extractor_flow.py +87 -0
  35. {structx-0.5.1 → structx-0.6.0}/tests/test_file_reader.py +57 -11
  36. structx-0.6.0/tests/test_input_and_batch_processors.py +163 -0
  37. structx-0.6.0/tests/test_live_llm.py +74 -0
  38. structx-0.6.0/tests/test_llm_configuration.py +286 -0
  39. structx-0.6.0/tests/test_result_manager_and_helpers.py +177 -0
  40. structx-0.6.0/tests/test_type_system_and_planning.py +290 -0
  41. structx-0.6.0/tests/test_usage.py +98 -0
  42. structx-0.5.1/structx/core/config.py +0 -126
  43. structx-0.5.1/structx/core/exceptions.py +0 -34
  44. structx-0.5.1/structx/core/models.py +0 -145
  45. structx-0.5.1/structx/extraction/core/llm_core.py +0 -206
  46. structx-0.5.1/structx/extraction/core/model_utils.py +0 -143
  47. structx-0.5.1/structx/extraction/engines/extraction_engine.py +0 -221
  48. structx-0.5.1/structx/extraction/extractor.py +0 -557
  49. structx-0.5.1/structx/extraction/generator.py +0 -232
  50. structx-0.5.1/structx/extraction/processors/__init__.py +0 -7
  51. structx-0.5.1/structx/extraction/processors/content_analyzer.py +0 -164
  52. structx-0.5.1/structx/extraction/processors/data_processor.py +0 -175
  53. structx-0.5.1/structx/extraction/processors/model_operations.py +0 -233
  54. structx-0.5.1/structx/extraction/result_manager.py +0 -125
  55. structx-0.5.1/structx/utils/constants.py +0 -36
  56. structx-0.5.1/structx/utils/helpers.py +0 -233
  57. structx-0.5.1/structx/utils/prompts.py +0 -183
  58. structx-0.5.1/structx/utils/usage.py +0 -327
  59. structx-0.5.1/structx.egg-info/requires.txt +0 -31
  60. structx-0.5.1/tests/test_config_and_model_utils.py +0 -66
  61. structx-0.5.1/tests/test_content_analyzer.py +0 -80
  62. structx-0.5.1/tests/test_data_processor.py +0 -129
  63. structx-0.5.1/tests/test_extractor_async.py +0 -109
  64. structx-0.5.1/tests/test_result_manager_and_helpers.py +0 -78
  65. {structx-0.5.1 → structx-0.6.0}/LICENSE +0 -0
  66. {structx-0.5.1 → structx-0.6.0}/setup.cfg +0 -0
  67. {structx-0.5.1 → structx-0.6.0}/structx/core/__init__.py +0 -0
  68. {structx-0.5.1 → structx-0.6.0}/structx/extraction/engines/__init__.py +0 -0
  69. {structx-0.5.1 → structx-0.6.0}/structx/utils/__init__.py +0 -0
  70. {structx-0.5.1 → structx-0.6.0}/structx/utils/types.py +0 -0
  71. {structx-0.5.1 → structx-0.6.0}/structx.egg-info/dependency_links.txt +0 -0
  72. {structx-0.5.1 → structx-0.6.0}/structx.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structx
3
- Version: 0.5.1
3
+ Version: 0.6.0
4
4
  Summary: Structured data extraction from text using LLMs and dynamic model generation
5
5
  Author-email: blacksuan19 <py@aolabs.dev>
6
6
  Classifier: Development Status :: 5 - Production/Stable
@@ -11,7 +11,6 @@ Classifier: Operating System :: OS Independent
11
11
  Classifier: Programming Language :: Python :: 3.10
12
12
  Classifier: Programming Language :: Python :: 3.11
13
13
  Classifier: Programming Language :: Python :: 3.12
14
- Classifier: Programming Language :: Python :: 3.9
15
14
  Classifier: Programming Language :: Python :: 3
16
15
  Classifier: Topic :: Database
17
16
  Classifier: Topic :: Documentation
@@ -29,34 +28,34 @@ Classifier: Typing :: Typed
29
28
  Requires-Python: >=3.10
30
29
  Description-Content-Type: text/markdown
31
30
  License-File: LICENSE
32
- Requires-Dist: instructor==1.8.0
33
- Requires-Dist: litellm==1.68.1
34
- Requires-Dist: loguru==0.7.3
35
- Requires-Dist: omegaconf==2.3.0
36
- Requires-Dist: openpyxl==3.1.5
37
- Requires-Dist: pandas==2.2.3
38
- Requires-Dist: pydantic==2.11.4
39
- Requires-Dist: tabulate==0.9.0
40
- Requires-Dist: tenacity==9.1.2
41
- Requires-Dist: tqdm==4.68.4
31
+ Requires-Dist: instructor>=1.15.4
32
+ Requires-Dist: litellm>=1.91.1
33
+ Requires-Dist: loguru>=0.7.3
34
+ Requires-Dist: openpyxl>=3.1.5
35
+ Requires-Dist: pandas>=2.2.3
36
+ Requires-Dist: pydantic>=2.11.7
37
+ Requires-Dist: pydantic-settings[yaml]>=2.14.2
38
+ Requires-Dist: tabulate>=0.9.0
39
+ Requires-Dist: tenacity>=9.1.2
40
+ Requires-Dist: tqdm>=4.68.4
42
41
  Provides-Extra: dev
43
- Requires-Dist: black==25.1.0; extra == "dev"
44
- Requires-Dist: bumpver==2024.1130; extra == "dev"
45
- Requires-Dist: pip-tools==7.4.1; extra == "dev"
46
- Requires-Dist: pytest==9.1.1; extra == "dev"
47
- Requires-Dist: wheel==0.45.1; extra == "dev"
42
+ Requires-Dist: black>=25.1.0; extra == "dev"
43
+ Requires-Dist: bumpver>=2024.1130; extra == "dev"
44
+ Requires-Dist: pip-tools>=7.4.1; extra == "dev"
45
+ Requires-Dist: pytest>=9.1.1; extra == "dev"
46
+ Requires-Dist: wheel>=0.45.1; extra == "dev"
48
47
  Provides-Extra: docs
49
- Requires-Dist: docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]==2.111.0; extra == "docs"
50
- Requires-Dist: torch==2.13.0; extra == "docs"
51
- Requires-Dist: torchvision==0.28.0; extra == "docs"
52
- Requires-Dist: weasyprint==69.0; extra == "docs"
48
+ Requires-Dist: docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]>=2.111.0; extra == "docs"
49
+ Requires-Dist: torch>=2.13.0; extra == "docs"
50
+ Requires-Dist: torchvision>=0.28.0; extra == "docs"
51
+ Requires-Dist: weasyprint>=69.0; extra == "docs"
53
52
  Provides-Extra: mkdocs
54
- Requires-Dist: mkdocs==1.6.1; extra == "mkdocs"
55
- Requires-Dist: mkdocs-material==9.7.6; extra == "mkdocs"
56
- Requires-Dist: mkdocstrings[python]==0.29.1; extra == "mkdocs"
57
- Requires-Dist: mkdocs-git-revision-date-localized-plugin==1.4.5; extra == "mkdocs"
58
- Requires-Dist: mkdocs-material[imaging]==9.7.6; extra == "mkdocs"
59
- Requires-Dist: mkdocs-mermaid2-plugin==1.2.3; extra == "mkdocs"
53
+ Requires-Dist: mkdocs>=1.6.1; extra == "mkdocs"
54
+ Requires-Dist: mkdocs-material>=9.7.6; extra == "mkdocs"
55
+ Requires-Dist: mkdocstrings[python]>=0.29.1; extra == "mkdocs"
56
+ Requires-Dist: mkdocs-git-revision-date-localized-plugin>=1.4.5; extra == "mkdocs"
57
+ Requires-Dist: mkdocs-material[imaging]>=9.7.6; extra == "mkdocs"
58
+ Requires-Dist: mkdocs-mermaid2-plugin>=1.2.3; extra == "mkdocs"
60
59
  Dynamic: license-file
61
60
 
62
61
  # structx
@@ -65,13 +64,13 @@ Advanced structured data extraction from any document using LLMs with multimodal
65
64
  support.
66
65
 
67
66
  [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg?style=for-the-badge)](https://structx.aolabs.dev "Documentation")
68
- [![PyPI](https://img.shields.io/badge/PyPi-0.5.1-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
67
+ [![PyPI](https://img.shields.io/badge/PyPi-0.6.0-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
69
68
  [![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)](# "Build with GitHub Actions")
70
69
 
71
70
  `structx` is a powerful Python library for extracting structured data from text,
72
- tables, and documents using Large Language Models (LLMs). With the optional
73
- `docs` extra, it provides a multimodal PDF pipeline that passes PDFs directly to
74
- vision-capable models and converts other document formats to PDF first.
71
+ tables, and documents using Large Language Models (LLMs). It passes existing
72
+ PDFs directly to vision-capable models; the optional `docs` extra converts other
73
+ document formats to PDF first.
75
74
 
76
75
  ## 🔔 Package rename notice (PyPI)
77
76
 
@@ -90,7 +89,7 @@ pip install -U structx
90
89
  ```
91
90
 
92
91
  If you previously pinned `structx-llm` in requirements or lock files, replace it
93
- with `structx`. For document/PDF processing, install `structx[docs]`.
92
+ with `structx`. Install `structx[docs]` for non-PDF document conversion.
94
93
 
95
94
  ## ✨ Key Features
96
95
 
@@ -102,8 +101,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
102
101
  PDFs and images
103
102
  - **🔄 Smart Format Detection**: Automatic processing mode selection for best
104
103
  results
105
- - **📊 Flexible File Support**: CSV, Excel, JSON, Parquet in the base install,
106
- with PDF, DOCX, TXT, Markdown, and more via `structx[docs]`
104
+ - **📊 Flexible File Support**: CSV, Excel, JSON, Parquet, raw text, and existing
105
+ PDFs in the base install; DOCX, PPTX, images, and more via `structx[docs]`
107
106
 
108
107
  ### 🚀 **Intelligent Data Extraction**
109
108
 
@@ -117,11 +116,12 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
117
116
 
118
117
  ### ⚡ **Performance & Reliability**
119
118
 
120
- - **🚀 High-Performance Processing**: Multi-threaded and async operations
119
+ - **🚀 High-Performance Processing**: Threaded sync and native async row requests
121
120
  - **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
122
121
  backoff
123
122
  - **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
124
- - **� Flexible Configuration**: Configurable extraction using OmegaConf
123
+ - **Flexible Configuration**: Model settings from arguments, YAML, environment
124
+ variables, dotenv files, and secrets through Pydantic Settings
125
125
  - **🔌 Multiple LLM Providers**: Support through litellm integration
126
126
 
127
127
  ## Installation
@@ -130,7 +130,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
130
130
  pip install structx
131
131
  ```
132
132
 
133
- For document and multimodal PDF support:
133
+ For converting DOCX, PowerPoint, OpenDocument, markup, image, and other
134
+ non-PDF document formats:
134
135
 
135
136
  ```bash
136
137
  pip install "structx[docs]"
@@ -166,14 +167,14 @@ result = extractor.extract(
166
167
  )
167
168
 
168
169
  # Access results
169
- print(f"Extracted {result.success_count} items")
170
+ print(f"Successful rows: {result.success_count}")
170
171
  print(result.data[0].model_dump_json(indent=2))
171
172
  ```
172
173
 
173
174
  ### 📄 Document Processing with Multimodal Support
174
175
 
175
176
  Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
176
- be passed directly through the multimodal path.
177
+ be passed directly through the multimodal path with the base installation.
177
178
 
178
179
  ```python
179
180
  # Process a PDF invoice through the multimodal pipeline
@@ -193,10 +194,11 @@ result = extractor.extract(
193
194
 
194
195
  ```python
195
196
  # Check token usage for cost monitoring
196
- usage = result.get_token_usage()
197
+ usage = result.usage
197
198
  if usage:
198
199
  print(f"Total tokens: {usage.total_tokens}")
199
- print(f"By step: {[(s.name, s.tokens) for s in usage.steps]}")
200
+ for step, calls in usage.steps.items():
201
+ print(step.value, [call.total_tokens for call in calls])
200
202
  ```
201
203
 
202
204
  ## 🚀 Why Multimodal PDF Processing?
@@ -221,6 +223,7 @@ For comprehensive documentation, examples, and guides, visit our
221
223
 
222
224
  - [Getting Started](https://structx.aolabs.dev/getting-started)
223
225
  - [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
226
+ - [Working with Results](https://structx.aolabs.dev/guides/working-with-results)
224
227
  - [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
225
228
  - [Async Operations](https://structx.aolabs.dev/guides/async-operations)
226
229
  - [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
@@ -4,13 +4,13 @@ Advanced structured data extraction from any document using LLMs with multimodal
4
4
  support.
5
5
 
6
6
  [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg?style=for-the-badge)](https://structx.aolabs.dev "Documentation")
7
- [![PyPI](https://img.shields.io/badge/PyPi-0.5.1-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
7
+ [![PyPI](https://img.shields.io/badge/PyPi-0.6.0-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
8
8
  [![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)](# "Build with GitHub Actions")
9
9
 
10
10
  `structx` is a powerful Python library for extracting structured data from text,
11
- tables, and documents using Large Language Models (LLMs). With the optional
12
- `docs` extra, it provides a multimodal PDF pipeline that passes PDFs directly to
13
- vision-capable models and converts other document formats to PDF first.
11
+ tables, and documents using Large Language Models (LLMs). It passes existing
12
+ PDFs directly to vision-capable models; the optional `docs` extra converts other
13
+ document formats to PDF first.
14
14
 
15
15
  ## 🔔 Package rename notice (PyPI)
16
16
 
@@ -29,7 +29,7 @@ pip install -U structx
29
29
  ```
30
30
 
31
31
  If you previously pinned `structx-llm` in requirements or lock files, replace it
32
- with `structx`. For document/PDF processing, install `structx[docs]`.
32
+ with `structx`. Install `structx[docs]` for non-PDF document conversion.
33
33
 
34
34
  ## ✨ Key Features
35
35
 
@@ -41,8 +41,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
41
41
  PDFs and images
42
42
  - **🔄 Smart Format Detection**: Automatic processing mode selection for best
43
43
  results
44
- - **📊 Flexible File Support**: CSV, Excel, JSON, Parquet in the base install,
45
- with PDF, DOCX, TXT, Markdown, and more via `structx[docs]`
44
+ - **📊 Flexible File Support**: CSV, Excel, JSON, Parquet, raw text, and existing
45
+ PDFs in the base install; DOCX, PPTX, images, and more via `structx[docs]`
46
46
 
47
47
  ### 🚀 **Intelligent Data Extraction**
48
48
 
@@ -56,11 +56,12 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
56
56
 
57
57
  ### ⚡ **Performance & Reliability**
58
58
 
59
- - **🚀 High-Performance Processing**: Multi-threaded and async operations
59
+ - **🚀 High-Performance Processing**: Threaded sync and native async row requests
60
60
  - **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
61
61
  backoff
62
62
  - **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
63
- - **� Flexible Configuration**: Configurable extraction using OmegaConf
63
+ - **Flexible Configuration**: Model settings from arguments, YAML, environment
64
+ variables, dotenv files, and secrets through Pydantic Settings
64
65
  - **🔌 Multiple LLM Providers**: Support through litellm integration
65
66
 
66
67
  ## Installation
@@ -69,7 +70,8 @@ with `structx`. For document/PDF processing, install `structx[docs]`.
69
70
  pip install structx
70
71
  ```
71
72
 
72
- For document and multimodal PDF support:
73
+ For converting DOCX, PowerPoint, OpenDocument, markup, image, and other
74
+ non-PDF document formats:
73
75
 
74
76
  ```bash
75
77
  pip install "structx[docs]"
@@ -105,14 +107,14 @@ result = extractor.extract(
105
107
  )
106
108
 
107
109
  # Access results
108
- print(f"Extracted {result.success_count} items")
110
+ print(f"Successful rows: {result.success_count}")
109
111
  print(result.data[0].model_dump_json(indent=2))
110
112
  ```
111
113
 
112
114
  ### 📄 Document Processing with Multimodal Support
113
115
 
114
116
  Install `structx[docs]` before using non-PDF document formats. Existing PDFs can
115
- be passed directly through the multimodal path.
117
+ be passed directly through the multimodal path with the base installation.
116
118
 
117
119
  ```python
118
120
  # Process a PDF invoice through the multimodal pipeline
@@ -132,10 +134,11 @@ result = extractor.extract(
132
134
 
133
135
  ```python
134
136
  # Check token usage for cost monitoring
135
- usage = result.get_token_usage()
137
+ usage = result.usage
136
138
  if usage:
137
139
  print(f"Total tokens: {usage.total_tokens}")
138
- print(f"By step: {[(s.name, s.tokens) for s in usage.steps]}")
140
+ for step, calls in usage.steps.items():
141
+ print(step.value, [call.total_tokens for call in calls])
139
142
  ```
140
143
 
141
144
  ## 🚀 Why Multimodal PDF Processing?
@@ -160,6 +163,7 @@ For comprehensive documentation, examples, and guides, visit our
160
163
 
161
164
  - [Getting Started](https://structx.aolabs.dev/getting-started)
162
165
  - [Basic Extraction](https://structx.aolabs.dev/guides/basic-extraction)
166
+ - [Working with Results](https://structx.aolabs.dev/guides/working-with-results)
163
167
  - [Unstructured Text Processing](https://structx.aolabs.dev/guides/unstructured-text)
164
168
  - [Async Operations](https://structx.aolabs.dev/guides/async-operations)
165
169
  - [Multiple Queries](https://structx.aolabs.dev/guides/multiple-queries)
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "structx"
7
- version = "0.5.1"
7
+ version = "0.6.0"
8
8
  description = "Structured data extraction from text using LLMs and dynamic model generation"
9
9
  authors = [{ name = "blacksuan19", email = "py@aolabs.dev" }]
10
10
  license-files = ["LICENSE"]
@@ -19,7 +19,6 @@ classifiers = [
19
19
  "Programming Language :: Python :: 3.10",
20
20
  "Programming Language :: Python :: 3.11",
21
21
  "Programming Language :: Python :: 3.12",
22
- "Programming Language :: Python :: 3.9",
23
22
  "Programming Language :: Python :: 3",
24
23
  "Topic :: Database",
25
24
  "Topic :: Documentation",
@@ -37,25 +36,25 @@ classifiers = [
37
36
  ]
38
37
 
39
38
  dependencies = [
40
- "instructor==1.8.0",
41
- "litellm==1.68.1",
42
- "loguru==0.7.3",
43
- "omegaconf==2.3.0",
44
- "openpyxl==3.1.5",
45
- "pandas==2.2.3",
46
- "pydantic==2.11.4",
47
- "tabulate==0.9.0", # required for pandas .to_markdown()
48
- "tenacity==9.1.2",
49
- "tqdm==4.68.4",
39
+ "instructor>=1.15.4",
40
+ "litellm>=1.91.1",
41
+ "loguru>=0.7.3",
42
+ "openpyxl>=3.1.5",
43
+ "pandas>=2.2.3",
44
+ "pydantic>=2.11.7",
45
+ "pydantic-settings[yaml]>=2.14.2",
46
+ "tabulate>=0.9.0", # required for pandas .to_markdown()
47
+ "tenacity>=9.1.2",
48
+ "tqdm>=4.68.4",
50
49
  ]
51
50
 
52
51
  [project.optional-dependencies]
53
52
  dev = [
54
- "black==25.1.0",
55
- "bumpver==2024.1130",
56
- "pip-tools==7.4.1",
57
- "pytest==9.1.1",
58
- "wheel==0.45.1",
53
+ "black>=25.1.0",
54
+ "bumpver>=2024.1130",
55
+ "pip-tools>=7.4.1",
56
+ "pytest>=9.1.1",
57
+ "wheel>=0.45.1",
59
58
  ]
60
59
 
61
60
  # Optional document processing support. Kept out of the base install because
@@ -63,20 +62,20 @@ dev = [
63
62
  # The format-pdf extra is still needed because DocumentConverter imports its
64
63
  # PDF backend dependencies even when existing PDFs are passed through directly.
65
64
  docs = [
66
- "docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]==2.111.0",
67
- "torch==2.13.0",
68
- "torchvision==0.28.0",
69
- "weasyprint==69.0",
65
+ "docling-slim[convert-core,format-docx,format-html,format-latex,format-markdown,format-opendocument,format-pdf,format-pptx,format-web,models-local]>=2.111.0",
66
+ "torch>=2.13.0",
67
+ "torchvision>=0.28.0",
68
+ "weasyprint>=69.0",
70
69
  ]
71
70
 
72
71
  # Documentation site tooling.
73
72
  mkdocs = [
74
- "mkdocs==1.6.1",
75
- "mkdocs-material==9.7.6",
76
- "mkdocstrings[python]==0.29.1",
77
- "mkdocs-git-revision-date-localized-plugin==1.4.5",
78
- "mkdocs-material[imaging]==9.7.6",
79
- "mkdocs-mermaid2-plugin==1.2.3",
73
+ "mkdocs>=1.6.1",
74
+ "mkdocs-material>=9.7.6",
75
+ "mkdocstrings[python]>=0.29.1",
76
+ "mkdocs-git-revision-date-localized-plugin>=1.4.5",
77
+ "mkdocs-material[imaging]>=9.7.6",
78
+ "mkdocs-mermaid2-plugin>=1.2.3",
80
79
  ]
81
80
 
82
81
  [[tool.uv.index]]
@@ -93,7 +92,7 @@ torchvision = [
93
92
  ]
94
93
 
95
94
  [tool.bumpver]
96
- current_version = "0.5.1"
95
+ current_version = "0.6.0"
97
96
  version_pattern = "MAJOR.MINOR.PATCH"
98
97
  commit_message = "Bump version {old_version} -> {new_version}"
99
98
  commit = true
@@ -112,6 +111,7 @@ include = ["structx*"]
112
111
 
113
112
  [tool.pytest.ini_options]
114
113
  filterwarnings = [
114
+ "ignore:There is no current event loop:DeprecationWarning:litellm\\._service_logger",
115
115
  "ignore:This field is deprecated.*:DeprecationWarning:docling\\.pipeline\\.standard_pdf_pipeline",
116
116
  "ignore:torch\\.ao\\.quantization is deprecated.*:DeprecationWarning:torch\\.ao\\.quantization\\.quantize",
117
117
  "ignore:torch\\.quantize_per_tensor.*:UserWarning:torch\\.ao\\.nn\\.quantized\\.dynamic\\.modules\\.rnn",
@@ -4,21 +4,22 @@ structx: Structured data extraction using LLMs
4
4
 
5
5
  from structx.core.config import ExtractionConfig, StepConfig
6
6
  from structx.core.models import (
7
- ExtractionGuide,
7
+ ExtractionPlan,
8
8
  ExtractionRequest,
9
+ ExtractionResult,
9
10
  ModelField,
10
- QueryRefinement,
11
+ RowResult,
11
12
  )
12
13
  from structx.extraction.extractor import Extractor
13
14
 
14
- __version__ = "0.5.1"
15
+ __version__ = "0.6.0"
15
16
  __all__ = [
16
17
  "Extractor",
17
18
  "ExtractionConfig",
18
19
  "StepConfig",
19
20
  "ModelField",
20
- "QueryAnalysis",
21
- "QueryRefinement",
22
- "ExtractionGuide",
21
+ "ExtractionPlan",
23
22
  "ExtractionRequest",
23
+ "ExtractionResult",
24
+ "RowResult",
24
25
  ]
@@ -0,0 +1,101 @@
1
+ """Pydantic Settings configuration for model-backed extraction steps."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Type
6
+
7
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
8
+ from pydantic_settings import (
9
+ BaseSettings,
10
+ PydanticBaseSettingsSource,
11
+ SettingsConfigDict,
12
+ YamlConfigSettingsSource,
13
+ )
14
+
15
+ from structx.core.exceptions import ConfigurationError
16
+ from structx.utils.types import DictStrAny
17
+
18
+
19
+ class StepConfig(BaseModel):
20
+ """Provider-specific completion parameters for one extraction step."""
21
+
22
+ model_config = ConfigDict(validate_assignment=True, extra="allow")
23
+
24
+ @model_validator(mode="before")
25
+ @classmethod
26
+ def decode_scalar_values(cls, value: Any) -> Any:
27
+ """Decode numeric and boolean values supplied by nested environment keys."""
28
+ if not isinstance(value, dict):
29
+ return value
30
+ decoded = {}
31
+ for key, item in value.items():
32
+ if isinstance(item, str):
33
+ try:
34
+ item = json.loads(item)
35
+ except json.JSONDecodeError:
36
+ pass
37
+ decoded[key] = item
38
+ return decoded
39
+
40
+ def options(self) -> DictStrAny:
41
+ """Return explicitly configured provider parameters."""
42
+ return self.model_dump(exclude_none=True)
43
+
44
+
45
+ class ExtractionConfig(BaseSettings):
46
+ """Settings loaded from arguments, environment, dotenv, YAML, or secrets.
47
+
48
+ Source priority is constructor values, environment variables, ``.env``,
49
+ YAML, then file secrets. Environment and secret names use the ``STRUCTX_``
50
+ prefix.
51
+ """
52
+
53
+ planning: StepConfig = Field(default_factory=StepConfig)
54
+ extraction: StepConfig = Field(default_factory=StepConfig)
55
+
56
+ model_config = SettingsConfigDict(
57
+ extra="ignore",
58
+ env_prefix="STRUCTX_",
59
+ env_nested_delimiter="__",
60
+ env_file=".env",
61
+ env_file_encoding="utf-8",
62
+ yaml_file=None,
63
+ )
64
+
65
+ @classmethod
66
+ def settings_customise_sources(
67
+ cls,
68
+ settings_cls: Type[BaseSettings],
69
+ init_settings: PydanticBaseSettingsSource,
70
+ env_settings: PydanticBaseSettingsSource,
71
+ dotenv_settings: PydanticBaseSettingsSource,
72
+ file_secret_settings: PydanticBaseSettingsSource,
73
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
74
+ """Use conventional precedence with YAML below dotenv values."""
75
+ return (
76
+ init_settings,
77
+ env_settings,
78
+ dotenv_settings,
79
+ YamlConfigSettingsSource(settings_cls),
80
+ file_secret_settings,
81
+ )
82
+
83
+ @classmethod
84
+ def from_yaml(cls, path: str | Path, **overrides: Any) -> "ExtractionConfig":
85
+ """Load settings from a YAML file with optional highest-priority overrides."""
86
+ yaml_path = Path(path).expanduser()
87
+ if not yaml_path.is_file():
88
+ raise ConfigurationError(f"Configuration file not found: {yaml_path}")
89
+
90
+ class YamlExtractionConfig(cls):
91
+ model_config = SettingsConfigDict(
92
+ **{**cls.model_config, "yaml_file": yaml_path}
93
+ )
94
+
95
+ return YamlExtractionConfig(**overrides)
96
+
97
+ def for_step(self, step: str) -> DictStrAny:
98
+ """Return provider kwargs for ``planning`` or ``extraction``."""
99
+ if step not in {"planning", "extraction"}:
100
+ raise ConfigurationError(f"Unknown extraction step: {step}")
101
+ return getattr(self, step).options()
@@ -0,0 +1,34 @@
1
+ class StructXError(Exception):
2
+ """Base exception for errors raised directly by Structx."""
3
+
4
+ pass
5
+
6
+
7
+ class ConfigurationError(StructXError):
8
+ """Raised for invalid Structx configuration or runtime settings."""
9
+
10
+ pass
11
+
12
+
13
+ class ExtractionError(StructXError):
14
+ """Raised when a public extraction operation cannot complete."""
15
+
16
+ pass
17
+
18
+
19
+ class ValidationError(StructXError):
20
+ """Raised for Structx-specific input validation failures."""
21
+
22
+ pass
23
+
24
+
25
+ class ModelGenerationError(StructXError):
26
+ """Raised when an extraction schema cannot become a Pydantic model."""
27
+
28
+ pass
29
+
30
+
31
+ class FileError(StructXError):
32
+ """Raised for invalid paths, unsupported files, or conversion failures."""
33
+
34
+ pass
@@ -0,0 +1,58 @@
1
+ """Typed contracts for prepared extraction inputs."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from pathlib import Path
5
+ from typing import Any, Dict, List, Optional, Union
6
+
7
+ import pandas as pd
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class PdfRow:
12
+ """A PDF payload associated with one source row.
13
+
14
+ Attributes:
15
+ pdf_path: PDF sent to Instructor's multimodal input.
16
+ source: Original user-supplied document path. For converted documents,
17
+ this differs from ``pdf_path``.
18
+ """
19
+
20
+ pdf_path: Path
21
+ source: Path
22
+
23
+
24
+ RowPayload = Union[str, PdfRow]
25
+
26
+
27
+ @dataclass
28
+ class PreparedInput:
29
+ """Normalized extraction input and its owned resources.
30
+
31
+ ``InputProcessor`` creates this contract before planning or row processing.
32
+ Application code normally receives an ``ExtractionResult`` instead; this
33
+ type is primarily useful when calling ``FileReader`` directly.
34
+
35
+ Attributes:
36
+ dataframe: Source rows retained for planning, provenance, and optional
37
+ DataFrame output.
38
+ pdf_rows: Positional row mappings for multimodal PDF payloads.
39
+ planning_sample: Text extracted once from a converted document for
40
+ schema planning. Existing PDFs usually leave this unset and are
41
+ attached directly to the planning request.
42
+ owned_paths: Temporary files that must be deleted after processing.
43
+ ``InputProcessor.prepared`` handles this automatically.
44
+ """
45
+
46
+ dataframe: pd.DataFrame
47
+ pdf_rows: Dict[int, PdfRow] = field(default_factory=dict)
48
+ planning_sample: Optional[str] = None
49
+ owned_paths: List[Path] = field(default_factory=list)
50
+
51
+ def row_payload(
52
+ self, position: int, row: pd.Series, target_columns: List[str]
53
+ ) -> RowPayload:
54
+ """Build the text or PDF payload for one positional input row."""
55
+ pdf_row = self.pdf_rows.get(position)
56
+ if pdf_row is not None:
57
+ return pdf_row
58
+ return row[target_columns].to_markdown()