structx 0.4.8__py3-none-any.whl

structx/utils/helpers.py

@@ -0,0 +1,233 @@
+import functools
+import json
+from functools import wraps
+from typing import Any, Dict, Type
+
+from loguru import logger
+
+from structx.core.exceptions import ExtractionError
+from structx.core.models import ExtractionRequest, ModelField
+from structx.utils.types import P, R
+
+
+def handle_errors(
+    error_message: str,
+    error_type: Type[Exception] = ExtractionError,
+    default_return: Any = None,
+):
+    """
+    Decorator for consistent error handling and logging
+
+    Args:
+        error_message: Base message for the error
+        error_type: Type of exception to raise
+        default_return: Default return value if an error occurs
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                # Call the function
+                return func(*args, **kwargs)
+
+            except Exception as e:
+                logger.error(
+                    f"{error_message}: {str(e)}\n" f"Function: {func.__name__}"
+                )
+
+                if default_return is not None:
+                    return default_return
+
+                raise error_type(f"{error_message}: {str(e)}") from e
+
+        return wrapper
+
+    return decorator
+
+
+def flatten_extracted_data(data: Dict[str, Any], prefix: str = "") -> Dict[str, Any]:
+    """
+    Flatten nested structures for DataFrame storage
+
+    Args:
+        data: Nested dictionary of extracted data
+        prefix: Prefix for nested keys
+    """
+    flattened = {}
+
+    for key, value in data.items():
+        new_key = f"{prefix}_{key}" if prefix else key
+
+        if isinstance(value, dict):
+            nested = flatten_extracted_data(value, new_key)
+            flattened.update(nested)
+        elif isinstance(value, list):
+            if value and isinstance(value[0], dict):
+                for i, item in enumerate(value):
+                    nested = flatten_extracted_data(item, f"{new_key}_{i}")
+                    flattened.update(nested)
+            else:
+                flattened[new_key] = json.dumps(value)
+        else:
+            flattened[new_key] = value
+
+    return flattened
+
+
+def convert_pydantic_v1_to_v2(request: ExtractionRequest) -> ExtractionRequest:
+    """
+    Comprehensively convert Pydantic v1 syntax to v2 syntax in an extraction request
+
+    Args:
+        request: The extraction request potentially containing v1 syntax
+
+    Returns:
+        Updated extraction request with v2 syntax
+    """
+
+    def convert_field(field: ModelField) -> ModelField:
+        """Convert a single field from v1 to v2 syntax"""
+        # Create a copy of the validation dict
+        validation = field.validation.copy() if field.validation else {}
+
+        # Field validation conversions
+        v1_to_v2_mappings = {
+            # String validations
+            "regex": "pattern",
+            "min_items": "min_length",
+            "max_items": "max_length",
+            "anystr_strip_whitespace": "strip_whitespace",
+            "anystr_lower": "to_lower",
+            "anystr_upper": "to_upper",
+            # Numeric validations
+            "gt": "gt",  # Same in v2, but included for completeness
+            "ge": "ge",  # Same in v2, but included for completeness
+            "lt": "lt",  # Same in v2, but included for completeness
+            "le": "le",  # Same in v2, but included for completeness
+            "multiple_of": "multiple_of",  # Same in v2, but included for completeness
+            # Collection validations
+            "min_items": "min_length",
+            "max_items": "max_length",
+            "unique_items": "unique_items",  # Same in v2, but included for completeness
+            # Other validations
+            "const": "frozen",
+            "allow_mutation": "frozen",  # Inverse logic: !allow_mutation = frozen
+        }
+
+        # Apply mappings
+        for v1_key, v2_key in v1_to_v2_mappings.items():
+            if v1_key in validation:
+                # Special case for allow_mutation which has inverse logic
+                if v1_key == "allow_mutation":
+                    validation[v2_key] = not validation.pop(v1_key)
+                else:
+                    validation[v2_key] = validation.pop(v1_key)
+
+        # Handle special cases and complex conversions
+
+        # Convert type strings from v1 to v2 format
+        field_type = field.type
+
+        # Handle EmailStr which moved from pydantic.EmailStr to pydantic.EmailStr
+        if field_type == "EmailStr" or field_type == "pydantic.EmailStr":
+            field_type = "EmailStr"
+            # Add import hint in validation for model generator
+            validation["_import_"] = "from pydantic import EmailStr"
+
+        # Handle PositiveInt, NegativeInt, etc. which were removed in v2
+        type_conversions = {
+            "PositiveInt": "int",
+            "NegativeInt": "int",
+            "PositiveFloat": "float",
+            "NegativeFloat": "float",
+            "conint": "int",
+            "confloat": "float",
+            "constr": "str",
+            "conlist": "List",
+            "conset": "Set",
+            "confrozenset": "FrozenSet",
+        }
+
+        for old_type, new_type in type_conversions.items():
+            if old_type in field_type:
+                field_type = new_type
+                # Add appropriate constraints based on the original type
+                if old_type == "PositiveInt":
+                    validation["gt"] = 0
+                elif old_type == "NegativeInt":
+                    validation["lt"] = 0
+                elif old_type == "PositiveFloat":
+                    validation["gt"] = 0.0
+                elif old_type == "NegativeFloat":
+                    validation["lt"] = 0.0
+
+        # Process nested fields if present
+        nested_fields = None
+        if hasattr(field, "nested_fields") and field.nested_fields:
+            nested_fields = [
+                convert_field(nested_field) for nested_field in field.nested_fields
+            ]
+
+        # Create updated field with converted validation
+        return ModelField(
+            name=field.name,
+            type=field_type,
+            description=field.description,
+            validation=validation,
+            nested_fields=nested_fields,
+        )
+
+    # Convert all fields
+    updated_fields = [convert_field(field) for field in request.fields]
+
+    # Create updated request
+    return ExtractionRequest(
+        model_name=request.model_name,
+        model_description=request.model_description,
+        fields=updated_fields,
+    )
+
+
+def sanitize_regex_patterns(
+    extraction_request: "ExtractionRequest",
+) -> "ExtractionRequest":
+    """
+    Sanitize regex patterns in field validations to make them compatible with Pydantic V2
+
+    Args:
+        extraction_request: The extraction request to sanitize
+
+    Returns:
+        Sanitized extraction request
+    """
+
+    def fix_field_validation(field):
+        # Fix regex patterns in validation dict
+        if field.validation:
+            # For pattern/regex fields, sanitize the pattern
+            for pattern_key in ["pattern", "regex"]:
+                if pattern_key in field.validation:
+                    try:
+                        # Just test if the pattern compiles - if it does, leave it alone
+                        import re
+
+                        re.compile(field.validation[pattern_key])
+                    except re.error:
+                        # If pattern doesn't compile, we'll use a simpler approach:
+                        # Remove the pattern validation entirely rather than trying to fix it
+                        logger.warning(
+                            f"Removing invalid regex pattern: {field.validation[pattern_key]}"
+                        )
+                        del field.validation[pattern_key]
+
+        # Process nested fields recursively
+        if field.nested_fields:
+            for nested_field in field.nested_fields:
+                fix_field_validation(nested_field)
+
+    # Process all fields
+    for field in extraction_request.fields:
+        fix_field_validation(field)
+
+    return extraction_request

structx/utils/prompts.py

@@ -0,0 +1,183 @@
+from string import Template
+
+query_refinement_system_prompt = """You are a data structuring specialist.
+Analyze queries to determine the inherent structure of the data
+and provide clear requirements for extraction."""
+
+query_refinement_template = Template(
+    """
+    Analyze and expand the following query to ensure structured extraction.
+    Consider:
+    1. What are the inherent characteristics of the data?
+    2. What structural patterns would best represent it?
+    3. How should multiple related items be organized?
+    4. Are there nested relationships or hierarchies?
+    5. What data types would best represent each piece?
+
+    Query: ${query}
+
+    """
+)
+
+
+schema_system_prompt = """You are a schema design specialist.
+Create detailed schemas that capture complex data structures."""
+
+schema_template = Template(
+    """
+    Design a data extraction schema that enforces structured organization.
+    
+    Refined Query: ${refined_query} 
+    
+    Data Characteristics:
+    ${data_characteristics}
+    
+    Structural Requirements:
+    ${structural_requirements}
+    
+    Organization Principles:
+    ${organization_principles}
+    
+    Sample text:
+    ${sample_text}
+    
+    Important:
+    1. Create structured objects for complex information
+    2. Define appropriate data types
+    3. Include nested models where needed
+    4. Maintain consistent patterns
+    5. Preserve relationships
+    """
+)
+
+
+guide_system_prompt = """You are a data modeling specialist.
+Create clear patterns for organizing complex, nested data structures.
+Provide patterns as simple string descriptions and identify the correct target columns to analyze."""
+
+
+guide_template = Template(
+    """
+    Create guidelines for structured data extraction based on these characteristics:
+    ${data_characteristics}
+
+    here is the list of available columns in the dataset:
+    ${available_columns}
+    
+    Provide:
+    1. Structural patterns as key-value pairs of string descriptions
+    2. Relationship rules as a list of clear instructions
+    3. Organization principles as a list of guidelines
+    4. Target columns (target_columns) as a list of column names that contain the text to analyze
+    
+    The target_columns field is very important - it must contain only column names that actually exist in the dataset.
+    If unsure, use the first column or the 'text' column if available.
+    Keep all values as simple strings.
+    """
+)
+
+# Specialized template for custom model extraction
+custom_model_guide_template = Template(
+    """
+    Create guidelines for structured data extraction based on these model field characteristics:
+    ${data_characteristics}
+
+    These are the available columns in the dataset:
+    ${available_columns}
+    
+    The extraction needs to fill these model fields:
+    ${model_fields}
+    
+    Here are suggested column mappings for each model field:
+    ${column_suggestions}
+    
+    Provide:
+    1. Structural patterns describing how each field should be extracted
+    2. Relationship rules as a list of clear instructions for extracting each required field
+    3. Organization principles for populating the model consistently
+    4. Target columns (target_columns) as a list of data columns that should be analyzed
+    
+    For target_columns:
+    - Include all columns from the suggested mappings that are likely to contain relevant information
+    - If text/description columns exist, include them
+    - Only include column names that actually exist in the dataset
+    - Do not omit any columns that might contain information needed for the model fields
+
+    For relationship_rules:
+    - Include explicit rules for mapping specific columns to model fields
+    - Specify how to extract each field from the appropriate column(s)
+    - Include rules for handling special field types (dates, numbers, enumerations)
+    - Specify that fields should be left as null/None when no reliable information exists
+    - Add rules to prevent invented or fabricated values for any field
+
+    Keep all values as simple strings.
+    """
+)
+
+
+extraction_system_prompt = """
+You are a precise data extraction system specialized in transforming raw data into structured formats. Always:
+1. Return structured objects for complex data according to the specified model
+2. Maintain consistent formats and data types as defined in the schema
+3. Use exact field types as specified (strings, numbers, dates, enumerations)
+4. Use null/None for fields when no reliable information exists in the source data
+5. Follow structural patterns exactly as specified
+6. Return lists when the field expects a list of items
+7. Be thorough but do not invent data that isn't present in the source
+8. Only make inferences when there is strong supporting evidence in the data
+9. When working with custom models, leave fields as null rather than inventing values
+"""
+
+extraction_template = Template(
+    """
+    Extract structured information following these guidelines:
+    
+    Query: ${query} 
+    Patterns: ${patterns}
+    Rules: ${rules}
+
+    Important Notes:
+    - Always return a list of structured objects
+    - For list fields, return an array of items
+    - Each item in a list should follow the specified structure
+    - Use null/None when no reliable information is available (do NOT invent data)
+    - Dates should be in ISO format (YYYY-MM-DDTHH:MM:SS)
+    - For fields with enumerated values, use only values from the provided options
+    - It's better to leave a field null than to fill it with incorrect information
+
+    Text to analyze:
+    ${text}
+    """
+)
+
+# Refinement prompt for existing models
+refinement_system_prompt = """You are a data model refinement specialist.
+Analyze the existing model and the refinement instructions to create
+a new model that incorporates the requested changes."""
+
+refinement_template = Template(
+    """
+    Refine the following data model according to these instructions:
+    
+    EXISTING MODEL SCHEMA:
+    ```json
+    ${model_schema}
+    ```
+    
+    REFINEMENT INSTRUCTIONS:
+    ${instructions}
+    
+    Create a new model schema that:
+    1. Keeps fields from the original model that shouldn't change
+    2. Modifies fields as specified in the instructions
+    3. Adds new fields as specified in the instructions
+    4. Removes fields as specified in the instructions
+    
+    Important: Use Pydantic v2 syntax:
+    - Use `pattern` instead of `regex` for string patterns
+    - Use `model_config` instead of `Config` class
+    - Use `Field` with validation parameters instead of validators where possible
+    
+    Include a clear description of the model and each field.
+    """
+)

structx/utils/types.py

@@ -0,0 +1,12 @@
+from typing import Any, Dict, ParamSpec, TypeVar
+
+from pydantic import BaseModel
+
+ResponseType = TypeVar("ResponseType")
+
+DictStrAny = Dict[str, Any]
+
+# Type variables for parameters and return type
+T = TypeVar("T", bound=BaseModel)
+R = TypeVar("R")
+P = ParamSpec("P")