structx 0.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- structx/__init__.py +24 -0
- structx/core/__init__.py +0 -0
- structx/core/config.py +127 -0
- structx/core/exceptions.py +34 -0
- structx/core/models.py +147 -0
- structx/extraction/__init__.py +38 -0
- structx/extraction/core/__init__.py +6 -0
- structx/extraction/core/llm_core.py +206 -0
- structx/extraction/core/model_utils.py +143 -0
- structx/extraction/engines/__init__.py +5 -0
- structx/extraction/engines/extraction_engine.py +221 -0
- structx/extraction/extractor.py +540 -0
- structx/extraction/generator.py +232 -0
- structx/extraction/processors/__init__.py +6 -0
- structx/extraction/processors/data_content.py +367 -0
- structx/extraction/processors/model_operations.py +233 -0
- structx/extraction/result_manager.py +125 -0
- structx/utils/__init__.py +0 -0
- structx/utils/constants.py +36 -0
- structx/utils/file_reader.py +298 -0
- structx/utils/helpers.py +233 -0
- structx/utils/prompts.py +183 -0
- structx/utils/types.py +12 -0
- structx/utils/usage.py +327 -0
- structx-0.4.8.dist-info/METADATA +313 -0
- structx-0.4.8.dist-info/RECORD +29 -0
- structx-0.4.8.dist-info/WHEEL +5 -0
- structx-0.4.8.dist-info/licenses/LICENSE +21 -0
- structx-0.4.8.dist-info/top_level.txt +1 -0
structx/utils/usage.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import Any, Dict, List, Optional, Union
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ExtractionStep(Enum):
|
|
8
|
+
"""
|
|
9
|
+
Enumeration of extraction process steps.
|
|
10
|
+
|
|
11
|
+
Represents the different steps in the extraction pipeline where token usage is tracked.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
REFINEMENT = "refinement"
|
|
15
|
+
SCHEMA_GENERATION = "schema_generation"
|
|
16
|
+
EXTRACTION = "extraction"
|
|
17
|
+
GUIDE = "guide"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TokenDetails(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Details about token usage with advanced metrics.
|
|
23
|
+
|
|
24
|
+
Stores specialized token metrics such as reasoning (thinking) tokens,
|
|
25
|
+
audio tokens, and cached tokens when available from the LLM provider.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
audio_tokens: int = 0
|
|
29
|
+
reasoning_tokens: int = 0
|
|
30
|
+
cached_tokens: int = 0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class StepUsage(BaseModel):
|
|
34
|
+
"""
|
|
35
|
+
Token usage information for a single step in the extraction process.
|
|
36
|
+
|
|
37
|
+
Contains detailed token usage statistics for one step, including both prompt and
|
|
38
|
+
completion tokens, along with any provider-specific token details.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
prompt_tokens: int
|
|
42
|
+
completion_tokens: int
|
|
43
|
+
total_tokens: int
|
|
44
|
+
step: ExtractionStep
|
|
45
|
+
completion_tokens_details: Optional[TokenDetails] = None
|
|
46
|
+
prompt_tokens_details: Optional[TokenDetails] = None
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def from_completion(
|
|
50
|
+
cls, completion: Any, step: ExtractionStep
|
|
51
|
+
) -> Optional["StepUsage"]:
|
|
52
|
+
"""
|
|
53
|
+
Create StepUsage from completion object.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
completion: LLM completion response object
|
|
57
|
+
step: Which extraction step this usage belongs to
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
StepUsage object or None if no usage data is available
|
|
61
|
+
"""
|
|
62
|
+
if not completion or not hasattr(completion, "usage"):
|
|
63
|
+
return None
|
|
64
|
+
|
|
65
|
+
usage = completion.usage
|
|
66
|
+
usage_data = {
|
|
67
|
+
"prompt_tokens": getattr(usage, "prompt_tokens", 0),
|
|
68
|
+
"completion_tokens": getattr(usage, "completion_tokens", 0),
|
|
69
|
+
"total_tokens": getattr(usage, "total_tokens", 0),
|
|
70
|
+
"step": step,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
# Add token details if available
|
|
74
|
+
if hasattr(usage, "completion_tokens_details"):
|
|
75
|
+
details = usage.completion_tokens_details
|
|
76
|
+
usage_data["completion_tokens_details"] = TokenDetails(
|
|
77
|
+
audio_tokens=getattr(details, "audio_tokens", 0),
|
|
78
|
+
reasoning_tokens=getattr(details, "reasoning_tokens", 0),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if hasattr(usage, "prompt_tokens_details"):
|
|
82
|
+
details = usage.prompt_tokens_details
|
|
83
|
+
usage_data["prompt_tokens_details"] = TokenDetails(
|
|
84
|
+
audio_tokens=getattr(details, "audio_tokens", 0),
|
|
85
|
+
cached_tokens=getattr(details, "cached_tokens", 0),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return cls(**usage_data)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class StepSummary(BaseModel):
|
|
92
|
+
"""
|
|
93
|
+
Token usage summary for a single step.
|
|
94
|
+
|
|
95
|
+
Provides a simple summary of token consumption for a specific step
|
|
96
|
+
of the extraction process.
|
|
97
|
+
|
|
98
|
+
Attributes:
|
|
99
|
+
tokens: Number of tokens used in this step
|
|
100
|
+
name: Name of the step (analysis, refinement, etc.)
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
tokens: int
|
|
104
|
+
name: str
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class ExtractionSummary(BaseModel):
|
|
108
|
+
"""
|
|
109
|
+
Token usage summary for extraction steps.
|
|
110
|
+
|
|
111
|
+
Extends StepSummary with additional details about individual extraction steps.
|
|
112
|
+
|
|
113
|
+
Attributes:
|
|
114
|
+
tokens: Number of tokens used across all extraction steps
|
|
115
|
+
name: Always "extraction"
|
|
116
|
+
steps: Detailed breakdown of individual extraction calls
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
tokens: int
|
|
120
|
+
name: str = "extraction"
|
|
121
|
+
steps: List[StepSummary] = []
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class UsageSummary(BaseModel):
|
|
125
|
+
"""
|
|
126
|
+
Overall token usage summary across all extraction steps.
|
|
127
|
+
|
|
128
|
+
Provides a complete view of token usage throughout the extraction process,
|
|
129
|
+
including totals and per-step breakdowns.
|
|
130
|
+
|
|
131
|
+
Attributes:
|
|
132
|
+
total_tokens: Total tokens used across all steps
|
|
133
|
+
prompt_tokens: Total tokens used in prompts
|
|
134
|
+
completion_tokens: Total tokens generated in completions
|
|
135
|
+
thinking_tokens: Total thinking/reasoning tokens (if available)
|
|
136
|
+
cached_tokens: Total cached tokens (if available)
|
|
137
|
+
steps: List of per-step usage summaries
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
total_tokens: int
|
|
141
|
+
prompt_tokens: int
|
|
142
|
+
completion_tokens: int
|
|
143
|
+
thinking_tokens: Optional[int] = None
|
|
144
|
+
cached_tokens: Optional[int] = None
|
|
145
|
+
steps: List[Union[StepSummary, ExtractionSummary]]
|
|
146
|
+
|
|
147
|
+
def get_step(
|
|
148
|
+
self, step_name: str
|
|
149
|
+
) -> Optional[Union[StepSummary, ExtractionSummary]]:
|
|
150
|
+
"""
|
|
151
|
+
Get a step summary by its name.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
step_name: Name of the step to retrieve
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
Step summary or None if not found
|
|
158
|
+
"""
|
|
159
|
+
for step in self.steps:
|
|
160
|
+
if step.name == step_name:
|
|
161
|
+
return step
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class ExtractorUsage(BaseModel):
|
|
166
|
+
"""
|
|
167
|
+
Aggregated token usage tracking across all extraction steps.
|
|
168
|
+
|
|
169
|
+
Stores and manages token usage data for the entire extraction process,
|
|
170
|
+
providing methods to calculate totals and generate summaries.
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
steps: Dict[ExtractionStep, Union[StepUsage, List[StepUsage]]] = Field(
|
|
174
|
+
default_factory=dict
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
def add_step_usage(self, usage: Optional[StepUsage]):
|
|
178
|
+
"""Add usage information for a step"""
|
|
179
|
+
if not usage:
|
|
180
|
+
return
|
|
181
|
+
|
|
182
|
+
if usage.step == ExtractionStep.EXTRACTION:
|
|
183
|
+
# For extraction steps, collect as a list
|
|
184
|
+
if ExtractionStep.EXTRACTION not in self.steps:
|
|
185
|
+
self.steps[ExtractionStep.EXTRACTION] = []
|
|
186
|
+
extraction_list = self.steps[ExtractionStep.EXTRACTION]
|
|
187
|
+
if isinstance(extraction_list, list):
|
|
188
|
+
extraction_list.append(usage)
|
|
189
|
+
else:
|
|
190
|
+
# For other steps, store a single usage
|
|
191
|
+
self.steps[usage.step] = usage
|
|
192
|
+
|
|
193
|
+
def _get_attribute_value(self, obj: Any, attr_path: str) -> Optional[int]:
|
|
194
|
+
"""Extract a nested attribute value from an object using a dot-notation path"""
|
|
195
|
+
value = obj
|
|
196
|
+
for attr in attr_path.split("."):
|
|
197
|
+
if not hasattr(value, attr) or getattr(value, attr) is None:
|
|
198
|
+
return None
|
|
199
|
+
value = getattr(value, attr)
|
|
200
|
+
return value if isinstance(value, int) else None
|
|
201
|
+
|
|
202
|
+
def _get_token_sum(self, attr_path: str) -> int:
|
|
203
|
+
"""Get sum of tokens across steps for a specific attribute path"""
|
|
204
|
+
total = 0
|
|
205
|
+
|
|
206
|
+
# Process all steps
|
|
207
|
+
for step_type in ExtractionStep:
|
|
208
|
+
if step_type not in self.steps:
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
step_data = self.steps[step_type]
|
|
212
|
+
|
|
213
|
+
# Single step case
|
|
214
|
+
if isinstance(step_data, StepUsage):
|
|
215
|
+
value = self._get_attribute_value(step_data, attr_path)
|
|
216
|
+
if value is not None:
|
|
217
|
+
total += value
|
|
218
|
+
|
|
219
|
+
# Multiple extraction steps case
|
|
220
|
+
elif isinstance(step_data, list):
|
|
221
|
+
for item in step_data:
|
|
222
|
+
value = self._get_attribute_value(item, attr_path)
|
|
223
|
+
if value is not None:
|
|
224
|
+
total += value
|
|
225
|
+
|
|
226
|
+
return total
|
|
227
|
+
|
|
228
|
+
# Simple property accessors using _get_token_sum
|
|
229
|
+
@property
|
|
230
|
+
def total_tokens(self) -> int:
|
|
231
|
+
"""Total tokens across all steps"""
|
|
232
|
+
return self._get_token_sum("total_tokens")
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def prompt_tokens(self) -> int:
|
|
236
|
+
"""Total prompt tokens"""
|
|
237
|
+
return self._get_token_sum("prompt_tokens")
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def completion_tokens(self) -> int:
|
|
241
|
+
"""Total completion tokens"""
|
|
242
|
+
return self._get_token_sum("completion_tokens")
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def thinking_tokens(self) -> int:
|
|
246
|
+
"""Total thinking tokens"""
|
|
247
|
+
return self._get_token_sum("completion_tokens_details.reasoning_tokens")
|
|
248
|
+
|
|
249
|
+
@property
|
|
250
|
+
def cached_tokens(self) -> int:
|
|
251
|
+
"""Total cached tokens"""
|
|
252
|
+
return self._get_token_sum("prompt_tokens_details.cached_tokens")
|
|
253
|
+
|
|
254
|
+
def get_usage_summary(self, detailed: bool = False) -> UsageSummary:
|
|
255
|
+
"""Get structured token usage summary.
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
detailed: Whether to include detailed breakdown of extraction steps
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
UsageSummary object with token usage information
|
|
262
|
+
"""
|
|
263
|
+
# Create step summaries for standard steps
|
|
264
|
+
step_summaries = []
|
|
265
|
+
for step_type in [
|
|
266
|
+
ExtractionStep.REFINEMENT,
|
|
267
|
+
ExtractionStep.SCHEMA_GENERATION,
|
|
268
|
+
ExtractionStep.GUIDE,
|
|
269
|
+
]:
|
|
270
|
+
if step_type in self.steps and isinstance(self.steps[step_type], StepUsage):
|
|
271
|
+
step_summaries.append(
|
|
272
|
+
StepSummary(
|
|
273
|
+
tokens=self.steps[step_type].total_tokens, name=step_type.value
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
else:
|
|
277
|
+
step_summaries.append(StepSummary(tokens=0, name=step_type.value))
|
|
278
|
+
|
|
279
|
+
# Create extraction summary
|
|
280
|
+
extraction_summary = None
|
|
281
|
+
if ExtractionStep.EXTRACTION in self.steps:
|
|
282
|
+
extraction_steps = self.steps[ExtractionStep.EXTRACTION]
|
|
283
|
+
if isinstance(extraction_steps, list):
|
|
284
|
+
# Total tokens
|
|
285
|
+
total_extraction_tokens = sum(
|
|
286
|
+
step.total_tokens for step in extraction_steps
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
# Individual step details if requested
|
|
290
|
+
extraction_steps_summary = []
|
|
291
|
+
if detailed:
|
|
292
|
+
extraction_steps_summary = [
|
|
293
|
+
StepSummary(tokens=step.total_tokens, name=f"extraction_{i}")
|
|
294
|
+
for i, step in enumerate(extraction_steps)
|
|
295
|
+
]
|
|
296
|
+
|
|
297
|
+
extraction_summary = ExtractionSummary(
|
|
298
|
+
tokens=total_extraction_tokens,
|
|
299
|
+
name=ExtractionStep.EXTRACTION.value,
|
|
300
|
+
steps=extraction_steps_summary,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
if not extraction_summary:
|
|
304
|
+
extraction_summary = ExtractionSummary(
|
|
305
|
+
tokens=0, name=ExtractionStep.EXTRACTION.value
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
step_summaries.append(extraction_summary)
|
|
309
|
+
|
|
310
|
+
# Create the complete summary
|
|
311
|
+
summary = UsageSummary(
|
|
312
|
+
total_tokens=self.total_tokens,
|
|
313
|
+
prompt_tokens=self.prompt_tokens,
|
|
314
|
+
completion_tokens=self.completion_tokens,
|
|
315
|
+
steps=step_summaries,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# Add optional fields if they have values
|
|
319
|
+
thinking_tokens = self.thinking_tokens
|
|
320
|
+
if thinking_tokens > 0:
|
|
321
|
+
summary.thinking_tokens = thinking_tokens
|
|
322
|
+
|
|
323
|
+
cached_tokens = self.cached_tokens
|
|
324
|
+
if cached_tokens > 0:
|
|
325
|
+
summary.cached_tokens = cached_tokens
|
|
326
|
+
|
|
327
|
+
return summary
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: structx
|
|
3
|
+
Version: 0.4.8
|
|
4
|
+
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
|
+
Author-email: blacksuan19 <py@blacksuan19.dev>
|
|
6
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
7
|
+
Classifier: Framework :: AsyncIO
|
|
8
|
+
Classifier: Framework :: Pydantic
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Database
|
|
17
|
+
Classifier: Topic :: Documentation
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
19
|
+
Classifier: Topic :: Office/Business
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
25
|
+
Classifier: Topic :: Text Processing :: Markup
|
|
26
|
+
Classifier: Topic :: Text Processing
|
|
27
|
+
Classifier: Topic :: Utilities
|
|
28
|
+
Classifier: Typing :: Typed
|
|
29
|
+
Requires-Python: >=3.10
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Requires-Dist: instructor
|
|
33
|
+
Requires-Dist: litellm
|
|
34
|
+
Requires-Dist: loguru
|
|
35
|
+
Requires-Dist: omegaconf
|
|
36
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
37
|
+
Requires-Dist: pandas
|
|
38
|
+
Requires-Dist: pydantic
|
|
39
|
+
Requires-Dist: tabulate>=0.9.0
|
|
40
|
+
Requires-Dist: tenacity
|
|
41
|
+
Requires-Dist: tqdm
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: black; extra == "dev"
|
|
44
|
+
Requires-Dist: bumpver; extra == "dev"
|
|
45
|
+
Requires-Dist: pip-tools; extra == "dev"
|
|
46
|
+
Requires-Dist: wheel; extra == "dev"
|
|
47
|
+
Provides-Extra: pdf
|
|
48
|
+
Requires-Dist: pypdf2; extra == "pdf"
|
|
49
|
+
Requires-Dist: instructor[multimodal]; extra == "pdf"
|
|
50
|
+
Requires-Dist: weasyprint; extra == "pdf"
|
|
51
|
+
Requires-Dist: markdown; extra == "pdf"
|
|
52
|
+
Provides-Extra: docx
|
|
53
|
+
Requires-Dist: python-docx; extra == "docx"
|
|
54
|
+
Requires-Dist: docling; extra == "docx"
|
|
55
|
+
Requires-Dist: markdown; extra == "docx"
|
|
56
|
+
Requires-Dist: weasyprint; extra == "docx"
|
|
57
|
+
Provides-Extra: docs
|
|
58
|
+
Requires-Dist: pypdf2; extra == "docs"
|
|
59
|
+
Requires-Dist: python-docx; extra == "docs"
|
|
60
|
+
Requires-Dist: instructor[multimodal]; extra == "docs"
|
|
61
|
+
Requires-Dist: docling; extra == "docs"
|
|
62
|
+
Requires-Dist: weasyprint; extra == "docs"
|
|
63
|
+
Requires-Dist: markdown; extra == "docs"
|
|
64
|
+
Provides-Extra: unstructured
|
|
65
|
+
Requires-Dist: unstructured[all-docs]; extra == "unstructured"
|
|
66
|
+
Provides-Extra: docling
|
|
67
|
+
Requires-Dist: docling; extra == "docling"
|
|
68
|
+
Requires-Dist: markdown; extra == "docling"
|
|
69
|
+
Requires-Dist: weasyprint; extra == "docling"
|
|
70
|
+
Provides-Extra: advanced
|
|
71
|
+
Requires-Dist: pypdf2; extra == "advanced"
|
|
72
|
+
Requires-Dist: python-docx; extra == "advanced"
|
|
73
|
+
Requires-Dist: instructor[multimodal]; extra == "advanced"
|
|
74
|
+
Requires-Dist: docling; extra == "advanced"
|
|
75
|
+
Requires-Dist: weasyprint; extra == "advanced"
|
|
76
|
+
Requires-Dist: markdown; extra == "advanced"
|
|
77
|
+
Requires-Dist: unstructured[all-docs]; extra == "advanced"
|
|
78
|
+
Provides-Extra: mkdocs
|
|
79
|
+
Requires-Dist: mkdocs; extra == "mkdocs"
|
|
80
|
+
Requires-Dist: mkdocs-material; extra == "mkdocs"
|
|
81
|
+
Requires-Dist: mkdocstrings[markdown]; extra == "mkdocs"
|
|
82
|
+
Requires-Dist: mkdocstrings[python]; extra == "mkdocs"
|
|
83
|
+
Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == "mkdocs"
|
|
84
|
+
Requires-Dist: mkdocs-material[imaging]; extra == "mkdocs"
|
|
85
|
+
Requires-Dist: mkdocs-mermaid2-plugin; extra == "mkdocs"
|
|
86
|
+
Dynamic: license-file
|
|
87
|
+
|
|
88
|
+
# structx
|
|
89
|
+
|
|
90
|
+
Advanced structured data extraction from any document using LLMs with multimodal
|
|
91
|
+
support.
|
|
92
|
+
|
|
93
|
+
[](https://structx.blacksuan19.dev "Documentation")
|
|
94
|
+
[](https://pypi.org/project/structx "Package")
|
|
95
|
+
[](# "Build with GitHub Actions")
|
|
96
|
+
|
|
97
|
+
`structx` is a powerful Python library for extracting structured data from any
|
|
98
|
+
document or text using Large Language Models (LLMs). It features an innovative
|
|
99
|
+
multimodal PDF processing pipeline that converts any document to PDF and uses
|
|
100
|
+
instructor's vision capabilities for superior extraction quality.
|
|
101
|
+
|
|
102
|
+
## 🔔 Package rename notice (PyPI)
|
|
103
|
+
|
|
104
|
+
The PyPI distribution has been renamed from `structx-llm` to `structx`
|
|
105
|
+
(September 2025).
|
|
106
|
+
|
|
107
|
+
- Imports are unchanged: continue using `import structx`
|
|
108
|
+
- Extras are unchanged: `structx[docs]`, `structx[pdf]`, `structx[docx]`
|
|
109
|
+
- Please update your environments and requirement files to use the new name
|
|
110
|
+
|
|
111
|
+
Upgrade commands:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
pip uninstall -y structx-llm
|
|
115
|
+
pip install -U structx
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
If you previously pinned `structx-llm` in requirements or lock files, replace it
|
|
119
|
+
with `structx`.
|
|
120
|
+
|
|
121
|
+
## ✨ Key Features
|
|
122
|
+
|
|
123
|
+
### 🎯 **Advanced Document Processing**
|
|
124
|
+
|
|
125
|
+
- **� Multimodal PDF Pipeline**: Converts any document (TXT, DOCX, etc.) to PDF
|
|
126
|
+
for optimal extraction
|
|
127
|
+
- **🖼️ Vision-Enabled Extraction**: Native instructor multimodal support for
|
|
128
|
+
PDFs and images
|
|
129
|
+
- **🔄 Smart Format Detection**: Automatic processing mode selection for best
|
|
130
|
+
results
|
|
131
|
+
- **📊 Universal File Support**: CSV, Excel, JSON, Parquet, PDF, DOCX, TXT,
|
|
132
|
+
Markdown, and more
|
|
133
|
+
|
|
134
|
+
### 🚀 **Intelligent Data Extraction**
|
|
135
|
+
|
|
136
|
+
- **🔄 Dynamic Model Generation**: Create type-safe Pydantic models from natural
|
|
137
|
+
language queries
|
|
138
|
+
- **🎯 Automatic Schema Inference**: Intelligent schema generation and
|
|
139
|
+
refinement
|
|
140
|
+
- **📊 Complex Data Structures**: Support for nested and hierarchical data
|
|
141
|
+
- **🔄 Natural Language Refinement**: Improve models with conversational
|
|
142
|
+
instructions
|
|
143
|
+
|
|
144
|
+
### ⚡ **Performance & Reliability**
|
|
145
|
+
|
|
146
|
+
- **🚀 High-Performance Processing**: Multi-threaded and async operations
|
|
147
|
+
- **🔄 Robust Error Handling**: Automatic retry mechanism with exponential
|
|
148
|
+
backoff
|
|
149
|
+
- **📈 Token Usage Tracking**: Detailed step-by-step metrics for cost monitoring
|
|
150
|
+
- **� Flexible Configuration**: Configurable extraction using OmegaConf
|
|
151
|
+
- **🔌 Multiple LLM Providers**: Support through litellm integration
|
|
152
|
+
|
|
153
|
+
## Installation
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
# Core package with basic extraction capabilities
|
|
157
|
+
pip install structx
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### 📄 Enhanced Document Processing (Recommended)
|
|
161
|
+
|
|
162
|
+
For the best experience with all document types including advanced multimodal
|
|
163
|
+
PDF processing:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
# Complete document processing support
|
|
167
|
+
pip install structx[docs]
|
|
168
|
+
|
|
169
|
+
# Individual components
|
|
170
|
+
pip install structx[pdf] # PDF processing with multimodal support
|
|
171
|
+
pip install structx[docx] # Advanced DOCX conversion via docling
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### 🔧 What Each Extra Provides
|
|
175
|
+
|
|
176
|
+
- **`[docs]`**: Complete multimodal document processing pipeline
|
|
177
|
+
- PDF conversion from any document type
|
|
178
|
+
- Instructor multimodal vision support
|
|
179
|
+
- Advanced DOCX processing via docling
|
|
180
|
+
- Enhanced extraction quality
|
|
181
|
+
- **`[pdf]`**: PDF-specific processing
|
|
182
|
+
|
|
183
|
+
- Multimodal PDF support via instructor
|
|
184
|
+
- PDF generation capabilities
|
|
185
|
+
- Basic PDF text extraction fallback
|
|
186
|
+
|
|
187
|
+
- **`[docx]`**: Advanced DOCX support
|
|
188
|
+
- Document conversion via docling
|
|
189
|
+
- Structure preservation
|
|
190
|
+
- Markdown-based processing pipeline
|
|
191
|
+
|
|
192
|
+
## Quick Start
|
|
193
|
+
|
|
194
|
+
### Basic Text Extraction
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
from structx import Extractor
|
|
198
|
+
|
|
199
|
+
# Initialize extractor
|
|
200
|
+
extractor = Extractor.from_litellm(
|
|
201
|
+
model="gpt-4o",
|
|
202
|
+
api_key="your-api-key",
|
|
203
|
+
max_retries=3, # Automatically retry on transient errors
|
|
204
|
+
min_wait=1, # Start with 1 second wait
|
|
205
|
+
max_wait=10 # Maximum 10 seconds between retries
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Extract from text
|
|
209
|
+
result = extractor.extract(
|
|
210
|
+
data="System check on 2024-01-15 detected high CPU usage (92%) on server-01.",
|
|
211
|
+
query="extract incident date and details"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
# Access results
|
|
215
|
+
print(f"Extracted {result.success_count} items")
|
|
216
|
+
print(result.data[0].model_dump_json(indent=2))
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### 📄 Document Processing with Multimodal Support
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
# Process a PDF invoice directly with vision capabilities
|
|
223
|
+
result = extractor.extract(
|
|
224
|
+
data="scripts/example_input/S0305SampleInvoice.pdf", # Direct multimodal processing
|
|
225
|
+
query="extract the invoice number, total amount, and line items"
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# Convert a DOCX contract and process with multimodal support
|
|
229
|
+
result = extractor.extract(
|
|
230
|
+
data="scripts/example_input/free-consultancy-agreement.docx", # Auto-converted to PDF -> multimodal
|
|
231
|
+
query="extract parties, effective date, and payment terms"
|
|
232
|
+
)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### 📊 Token Usage Monitoring
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
# Check token usage for cost monitoring
|
|
239
|
+
usage = result.get_token_usage()
|
|
240
|
+
if usage:
|
|
241
|
+
print(f"Total tokens: {usage.total_tokens}")
|
|
242
|
+
print(f"By step: {[(s.name, s.tokens) for s in usage.steps]}")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
## 🚀 Why Multimodal PDF Processing?
|
|
246
|
+
|
|
247
|
+
The innovative multimodal approach provides significant advantages over
|
|
248
|
+
traditional text-based extraction:
|
|
249
|
+
|
|
250
|
+
- **📄 Context Preservation**: Full document layout and structure are maintained
|
|
251
|
+
- **🎯 Higher Accuracy**: Vision models can interpret tables, charts, and
|
|
252
|
+
complex layouts
|
|
253
|
+
- **🔄 No Chunking Issues**: Eliminates problems with information split across
|
|
254
|
+
chunks
|
|
255
|
+
- **📊 Universal Format**: Any document type becomes processable through PDF
|
|
256
|
+
conversion
|
|
257
|
+
- **🖼️ Visual Understanding**: Handles documents with visual elements,
|
|
258
|
+
formatting, and structure
|
|
259
|
+
|
|
260
|
+
## 📚 Documentation
|
|
261
|
+
|
|
262
|
+
For comprehensive documentation, examples, and guides, visit our
|
|
263
|
+
[documentation site](https://structx.blacksuan19.dev).
|
|
264
|
+
|
|
265
|
+
- [Getting Started](https://structx.blacksuan19.dev/getting-started)
|
|
266
|
+
- [Basic Extraction](https://structx.blacksuan19.dev/guides/basic-extraction)
|
|
267
|
+
- [Unstructured Text Processing](https://structx.blacksuan19.dev/guides/unstructured-text)
|
|
268
|
+
- [Async Operations](https://structx.blacksuan19.dev/guides/async-operations)
|
|
269
|
+
- [Multiple Queries](https://structx.blacksuan19.dev/guides/multiple-queries)
|
|
270
|
+
- [Custom Models](https://structx.blacksuan19.dev/guides/custom-models)
|
|
271
|
+
- [Token Usage Tracking](https://structx.blacksuan19.dev/guides/token-tracking)
|
|
272
|
+
- [API Reference](https://structx.blacksuan19.dev/api/extractor)
|
|
273
|
+
|
|
274
|
+
## Examples
|
|
275
|
+
|
|
276
|
+
Check out our [example gallery](https://structx.blacksuan19.dev/examples) for
|
|
277
|
+
real-world use cases,
|
|
278
|
+
|
|
279
|
+
## 📁 Supported File Formats
|
|
280
|
+
|
|
281
|
+
### 📊 Structured Data (Direct Processing)
|
|
282
|
+
|
|
283
|
+
- **CSV**: Comma-separated values with custom delimiters
|
|
284
|
+
- **Excel**: .xlsx/.xls with sheet selection and custom options
|
|
285
|
+
- **JSON**: JavaScript Object Notation with nested support
|
|
286
|
+
- **Parquet**: Columnar storage format for large datasets
|
|
287
|
+
- **Feather**: Fast binary format for data frames
|
|
288
|
+
|
|
289
|
+
### 📄 Unstructured Documents (Multimodal Pipeline)
|
|
290
|
+
|
|
291
|
+
| Format | Extensions | Processing Method | Quality |
|
|
292
|
+
| -------- | --------------------------------------------- | ------------------------------------- | ---------- |
|
|
293
|
+
| **PDF** | `.pdf` | Direct multimodal processing | ⭐⭐⭐⭐⭐ |
|
|
294
|
+
| **Word** | `.docx`, `.doc` | Docling → Markdown → PDF → Multimodal | ⭐⭐⭐⭐⭐ |
|
|
295
|
+
| **Text** | `.txt`, `.md`, `.py`, `.log`, `.xml`, `.html` | Styled PDF → Multimodal | ⭐⭐⭐⭐ |
|
|
296
|
+
|
|
297
|
+
### 🔄 Processing Modes
|
|
298
|
+
|
|
299
|
+
- **Multimodal PDF** (default): Best quality, preserves layout and context
|
|
300
|
+
- **Simple Text**: Fallback mode with chunking for memory-constrained
|
|
301
|
+
environments
|
|
302
|
+
- **Simple PDF**: Basic PDF text extraction without vision capabilities
|
|
303
|
+
|
|
304
|
+
## Contributing
|
|
305
|
+
|
|
306
|
+
Contributions are welcome! Please read our
|
|
307
|
+
[Contributing Guidelines](https://structx.blacksuan19.dev/contributing) for
|
|
308
|
+
details.
|
|
309
|
+
|
|
310
|
+
## License
|
|
311
|
+
|
|
312
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|
313
|
+
for details.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
structx/__init__.py,sha256=J1IujfdUMitYEYVU7BnZzZ2sBMo_4aqyeZH4hhzWE1g,491
|
|
2
|
+
structx/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
structx/core/config.py,sha256=RjrEPc-c195PwUpJYd0Dvsv__DG3UvsGrPsBVhnVoqo,4252
|
|
4
|
+
structx/core/exceptions.py,sha256=hl9CFe5lFzGryILdmOv1xaoj8Bwbqzgw-5U6e8-mWXM,525
|
|
5
|
+
structx/core/models.py,sha256=ynNnWtVbwHe4KiftQ-e0s75DsuOUl9B2Z2yOc0_Xoj0,4607
|
|
6
|
+
structx/extraction/__init__.py,sha256=Z6bmSAlMYAf_HeaeqrPKzg5rC20UqKiD7ZcdaFb6iJI,1001
|
|
7
|
+
structx/extraction/extractor.py,sha256=v29f-BUwdoEUWksi3fB0uQkkd1d_nIqilKXhVihOyok,18998
|
|
8
|
+
structx/extraction/generator.py,sha256=9n6o8FZRtgTiUEG0d-2jL4-K_VEBimOhClyCX8iMdVQ,8550
|
|
9
|
+
structx/extraction/result_manager.py,sha256=k_lLM50-pNjHlwBzpBSnsRbrfE302Q45fAoKXEplUZU,3682
|
|
10
|
+
structx/extraction/core/__init__.py,sha256=VAUGPbtVCcyBvCuqym7ageNpjvhkT3ZPEAbXBbLD_h4,153
|
|
11
|
+
structx/extraction/core/llm_core.py,sha256=pGtCtuoXLZ-lPTIYNItAAwgmLRJr2fI7udLMjXoC92E,6434
|
|
12
|
+
structx/extraction/core/model_utils.py,sha256=anvE1q6w9jhdznOLhr1LJFHBGyl1anysrDFDNLqCHkA,4611
|
|
13
|
+
structx/extraction/engines/__init__.py,sha256=xNnMFX7kS_DGUFXJFKffzou9VzMjS8npd0PCoWxm6Dw,143
|
|
14
|
+
structx/extraction/engines/extraction_engine.py,sha256=uR7UX6GkyOyoHLQwP9SkLfTZ6kD9K9SuSC8qYuJrboQ,8009
|
|
15
|
+
structx/extraction/processors/__init__.py,sha256=x4AFg5ixktVUNget5CNyY9Y9rHOsNUdauNRpeuFqHvo,229
|
|
16
|
+
structx/extraction/processors/data_content.py,sha256=OQNrVyegwCFedRzuhzO75H1BEIc6ebHg0joDWQ8580k,13560
|
|
17
|
+
structx/extraction/processors/model_operations.py,sha256=ty_JdzAVDogoSHObVjXS-s078eEnWmMBX__gLihTUpI,8457
|
|
18
|
+
structx/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
structx/utils/constants.py,sha256=3KMeumm-6q7zNcWyxixHCDmERpmSNIVS01uUo4N4o0I,872
|
|
20
|
+
structx/utils/file_reader.py,sha256=TYQ_G1d2vIaw9BZnVNUne4S3n8UMG48pUOwHqvxVj5A,11612
|
|
21
|
+
structx/utils/helpers.py,sha256=papsgnRYXNDnvo_d7nUaRHJXTLbcHoYXOw36vj4eczg,8078
|
|
22
|
+
structx/utils/prompts.py,sha256=GGfWNUBq8qOBoOz6-h_BINEPAKn5Q5VVkn2LPbMZJ2o,6619
|
|
23
|
+
structx/utils/types.py,sha256=dojNDaX2t3YTYkcG13UU8c9Bs498plH75ypTrMYJGoQ,269
|
|
24
|
+
structx/utils/usage.py,sha256=eA2L_L3eizCBtO2L7D3hLMZdUpsryNxZowXpGl07hKg,10663
|
|
25
|
+
structx-0.4.8.dist-info/licenses/LICENSE,sha256=H6mof4YPl56UolCTeVfkg-HxKUxTXU6nrmhCg0SxUng,1070
|
|
26
|
+
structx-0.4.8.dist-info/METADATA,sha256=WlpZq1aGdy79aXtodoyX19zuVUaDTIcCy9NGg7qPWPA,11688
|
|
27
|
+
structx-0.4.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
28
|
+
structx-0.4.8.dist-info/top_level.txt,sha256=H2eTQyV33YUusz-yJiJs5YouRw3vPdGRC7WnwLMZw9k,8
|
|
29
|
+
structx-0.4.8.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Abubakar Omer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|