openaivec 0.14.0__py3-none-any.whl → 0.14.2__py3-none-any.whl

openaivec/_serialize.py

@@ -1,31 +1,10 @@
-"""Serialization utilities for Pydantic BaseModel classes.
+"""Refactored serialization utilities for Pydantic BaseModel classes.
 
 This module provides utilities for converting Pydantic BaseModel classes
-to and from JSON schema representations. It supports dynamic model creation
-from JSON schemas with special handling for enum fields, which are converted
-to Literal types for better type safety and compatibility.
-
-Example:
-    Basic serialization and deserialization:
-
-    ```python
-    from pydantic import BaseModel
-    from typing import Literal
-
-    class Status(BaseModel):
-        value: Literal["active", "inactive"]
-        description: str
-
-    # Serialize to JSON schema
-    schema = serialize_base_model(Status)
-
-    # Deserialize back to BaseModel class
-    DynamicStatus = deserialize_base_model(schema)
-    instance = DynamicStatus(value="active", description="User is active")
-    ```
+to and from JSON schema representations with simplified, maintainable code.
 """
 
-from typing import Any, Dict, List, Literal, Type
+from typing import Any, Dict, List, Literal, Tuple, Type, Union
 
 from pydantic import BaseModel, Field, create_model
 
@@ -33,66 +12,38 @@ __all__ = []
 
 
 def serialize_base_model(obj: Type[BaseModel]) -> Dict[str, Any]:
-    """Serialize a Pydantic BaseModel to JSON schema.
-
-    Args:
-        obj (Type[BaseModel]): The Pydantic BaseModel class to serialize.
-
-    Returns:
-        A dictionary containing the JSON schema representation of the model.
-
-    Example:
-        ```python
-        from pydantic import BaseModel
-
-        class Person(BaseModel):
-            name: str
-            age: int
-
-        schema = serialize_base_model(Person)
-        ```
-    """
+    """Serialize a Pydantic BaseModel to JSON schema."""
     return obj.model_json_schema()
 
 
 def dereference_json_schema(json_schema: Dict[str, Any]) -> Dict[str, Any]:
-    """Dereference JSON schema by resolving $ref pointers.
-
-    This function resolves all $ref references in a JSON schema by replacing
-    them with the actual referenced definitions from the $defs section.
-
-    Args:
-        json_schema (Dict[str, Any]): The JSON schema containing potential $ref references.
-
-    Returns:
-        A dereferenced JSON schema with all $ref pointers resolved.
-
-    Example:
-        ```python
-        schema = {
-            "properties": {
-                "user": {"$ref": "#/$defs/User"}
-            },
-            "$defs": {
-                "User": {"type": "object", "properties": {"name": {"type": "string"}}}
-            }
-        }
-        dereferenced = dereference_json_schema(schema)
-        # user property will contain the actual User definition
-        ```
-    """
+    """Dereference JSON schema by resolving $ref pointers with circular reference protection."""
     model_map = json_schema.get("$defs", {})
 
-    def dereference(obj):
+    def dereference(obj, current_path=None):
+        if current_path is None:
+            current_path = []
+
         if isinstance(obj, dict):
             if "$ref" in obj:
                 ref = obj["$ref"].split("/")[-1]
-                return dereference(model_map[ref])
-            else:
-                return {k: dereference(v) for k, v in obj.items()}
 
+                # Check for circular reference
+                if ref in current_path:
+                    # Return a placeholder to break the cycle
+                    return {"type": "object", "description": f"Circular reference to {ref}"}
+
+                if ref in model_map:
+                    # Add to path and recurse
+                    new_path = current_path + [ref]
+                    return dereference(model_map[ref], new_path)
+                else:
+                    # Invalid reference, return placeholder
+                    return {"type": "object", "description": f"Invalid reference to {ref}"}
+            else:
+                return {k: dereference(v, current_path) for k, v in obj.items()}
         elif isinstance(obj, list):
-            return [dereference(x) for x in obj]
+            return [dereference(x, current_path) for x in obj]
         else:
             return obj
 
@@ -100,134 +51,180 @@ def dereference_json_schema(json_schema: Dict[str, Any]) -> Dict[str, Any]:
     for k, v in json_schema.items():
         if k == "$defs":
             continue
-
         result[k] = dereference(v)
 
     return result
 
 
-def parse_field(v: Dict[str, Any]) -> Any:
+# ============================================================================
+# Type Resolution - Separated into focused functions
+# ============================================================================
+
+
+def _resolve_union_type(union_options: List[Dict[str, Any]]) -> Type:
+    """Resolve anyOf/oneOf to Union type."""
+    union_types = []
+    for option in union_options:
+        if option.get("type") == "null":
+            union_types.append(type(None))
+        else:
+            union_types.append(parse_field(option))
+
+    if len(union_types) == 1:
+        return union_types[0]
+    elif len(union_types) == 2 and type(None) in union_types:
+        # Optional type: T | None
+        non_none_type = next(t for t in union_types if t is not type(None))
+        return Union[non_none_type, type(None)]  # type: ignore[return-value]
+    else:
+        return Union[tuple(union_types)]  # type: ignore[return-value]
+
+
+def _resolve_basic_type(type_name: str, field_def: Dict[str, Any]) -> Type:
+    """Resolve basic JSON schema types to Python types."""
+    type_mapping = {
+        "string": str,
+        "integer": int,
+        "number": float,
+        "boolean": bool,
+        "null": type(None),
+    }
+
+    if type_name in type_mapping:
+        return type_mapping[type_name]  # type: ignore[return-value]
+    elif type_name == "object":
+        # Check if it's a nested model or generic dict
+        if "properties" in field_def:
+            return deserialize_base_model(field_def)
+        else:
+            return dict
+    elif type_name == "array":
+        if "items" in field_def:
+            inner_type = parse_field(field_def["items"])
+            return List[inner_type]
+        else:
+            return List[Any]
+    else:
+        raise ValueError(f"Unsupported type: {type_name}")
+
+
+def parse_field(field_def: Dict[str, Any]) -> Type:
     """Parse a JSON schema field definition to a Python type.
 
-    Converts JSON schema field definitions to corresponding Python types
-    for use in Pydantic model creation.
+    Simplified version with clear separation of concerns.
+    """
+    # Handle union types
+    if "anyOf" in field_def:
+        return _resolve_union_type(field_def["anyOf"])
+    if "oneOf" in field_def:
+        return _resolve_union_type(field_def["oneOf"])
 
-    Args:
-        v (Dict[str, Any]): A dictionary containing the JSON schema field definition.
+    # Handle basic types
+    if "type" not in field_def:
+        return Any  # type: ignore[return-value]
 
-    Returns:
-        The corresponding Python type (str, int, float, bool, dict, List, or BaseModel).
+    return _resolve_basic_type(field_def["type"], field_def)
 
-    Raises:
-        ValueError: If the field type is not supported.
 
-    Example:
-        ```python
-        field_def = {"type": "string"}
-        python_type = parse_field(field_def)  # Returns str
+# ============================================================================
+# Field Information Creation - Centralized logic
+# ============================================================================
 
-        array_def = {"type": "array", "items": {"type": "integer"}}
-        python_type = parse_field(array_def)  # Returns List[int]
-        ```
-    """
-    t = v["type"]
-    if t == "string":
-        return str
-    elif t == "integer":
-        return int
-    elif t == "number":
-        return float
-    elif t == "boolean":
-        return bool
-    elif t == "object":
-        # Check if it's a generic object (dict) or a nested model
-        if "properties" in v:
-            return deserialize_base_model(v)
-        else:
-            return dict
-    elif t == "array":
-        inner_type = parse_field(v["items"])
-        return List[inner_type]
+
+def _create_field_info(description: str | None, default_value: Any, is_required: bool) -> Field:  # type: ignore[type-arg]
+    """Create Field info with consistent logic."""
+    if is_required and default_value is None:
+        # Required field without default
+        return Field(description=description) if description else Field()
     else:
-        raise ValueError(f"Unsupported type: {t}")
+        # Optional field or field with default
+        return Field(default=default_value, description=description) if description else Field(default=default_value)
+
+
+def _make_optional_if_needed(field_type: Type, is_required: bool, has_default: bool) -> Type:
+    """Make field type optional if needed."""
+    if is_required or has_default:
+        return field_type
+
+    # Check if already nullable
+    if hasattr(field_type, "__origin__") and field_type.__origin__ is Union and type(None) in field_type.__args__:
+        return field_type
+
+    # Make optional
+    return Union[field_type, type(None)]  # type: ignore[return-value]
+
+
+# ============================================================================
+# Field Processing - Separated enum and regular field logic
+# ============================================================================
+
+
+def _process_enum_field(field_name: str, field_def: Dict[str, Any], is_required: bool) -> Tuple[Type, Field]:  # type: ignore[type-arg]
+    """Process enum field with Literal type."""
+    enum_values = field_def["enum"]
+
+    # Create Literal type
+    if len(enum_values) == 1:
+        literal_type = Literal[enum_values[0]]
+    else:
+        literal_type = Literal[tuple(enum_values)]
+
+    # Handle optionality
+    description = field_def.get("description")
+    default_value = field_def.get("default")
+    has_default = default_value is not None
+
+    if not is_required and not has_default:
+        literal_type = Union[literal_type, type(None)]  # type: ignore[assignment]
+        default_value = None
+
+    field_info = _create_field_info(description, default_value, is_required)
+    return literal_type, field_info  # type: ignore[return-value]
+
+
+def _process_regular_field(field_name: str, field_def: Dict[str, Any], is_required: bool) -> Tuple[Type, Field]:  # type: ignore[type-arg]
+    """Process regular (non-enum) field."""
+    field_type = parse_field(field_def)
+    description = field_def.get("description")
+    default_value = field_def.get("default")
+    has_default = default_value is not None
+
+    # Handle optionality
+    field_type = _make_optional_if_needed(field_type, is_required, has_default)
+
+    if not is_required and not has_default:
+        default_value = None
+
+    field_info = _create_field_info(description, default_value, is_required)
+    return field_type, field_info
+
+
+# ============================================================================
+# Main Schema Processing - Clean and focused
+# ============================================================================
 
 
 def deserialize_base_model(json_schema: Dict[str, Any]) -> Type[BaseModel]:
     """Deserialize a JSON schema to a Pydantic BaseModel class.
 
-    Creates a dynamic Pydantic BaseModel class from a JSON schema definition.
-    For enum fields, this function uses Literal types instead of Enum classes
-    for better type safety and compatibility with systems like Apache Spark.
-
-    Args:
-        json_schema (Dict[str, Any]): A dictionary containing the JSON schema definition.
-
-    Returns:
-        A dynamically created Pydantic BaseModel class.
-
-    Example:
-        ```python
-        schema = {
-            "title": "Person",
-            "type": "object",
-            "properties": {
-                "name": {"type": "string", "description": "Person's name"},
-                "status": {
-                    "type": "string",
-                    "enum": ["active", "inactive"],
-                    "description": "Person's status"
-                }
-            }
-        }
-
-        PersonModel = deserialize_base_model(schema)
-        person = PersonModel(name="John", status="active")
-        ```
-
-    Note:
-        Enum fields are converted to Literal types for improved compatibility
-        and type safety. This ensures better integration with data processing
-        frameworks like Apache Spark.
+    Refactored version with clear separation of concerns and simplified logic.
     """
-    fields = {}
-    properties = dereference_json_schema(json_schema).get("properties", {})
-
-    for k, v in properties.items():
-        if "enum" in v:
-            enum_values = v["enum"]
+    # Basic setup
+    title = json_schema.get("title", "DynamicModel")
+    dereferenced_schema = dereference_json_schema(json_schema)
+    properties = dereferenced_schema.get("properties", {})
+    required_fields = set(dereferenced_schema.get("required", []))
 
-            # Always use Literal instead of Enum for better type safety and Spark compatibility
-            if len(enum_values) == 1:
-                literal_type = Literal[enum_values[0]]
-            else:
-                # Create Literal with multiple values
-                literal_type = Literal[tuple(enum_values)]
-
-            description = v.get("description")
-            default_value = v.get("default")
-
-            if default_value is not None:
-                field_info = (
-                    Field(default=default_value, description=description)
-                    if description is not None
-                    else Field(default=default_value)
-                )
-            else:
-                field_info = Field(description=description) if description is not None else Field()
+    # Process each field
+    fields = {}
+    for field_name, field_def in properties.items():
+        is_required = field_name in required_fields
 
-            fields[k] = (literal_type, field_info)
+        if "enum" in field_def:
+            field_type, field_info = _process_enum_field(field_name, field_def, is_required)
         else:
-            description = v.get("description")
-            default_value = v.get("default")
-
-            if default_value is not None:
-                field_info = (
-                    Field(default=default_value, description=description)
-                    if description is not None
-                    else Field(default=default_value)
-                )
-            else:
-                field_info = Field(description=description) if description is not None else Field()
+            field_type, field_info = _process_regular_field(field_name, field_def, is_required)
+
+        fields[field_name] = (field_type, field_info)
 
-            fields[k] = (parse_field(v), field_info)
-    return create_model(json_schema["title"], **fields)
+    return create_model(title, **fields)

openaivec/task/table/fillna.py

@@ -79,7 +79,7 @@ __all__ = ["fillna", "FillNaResponse"]
 def get_examples(df: pd.DataFrame, target_column_name: str, max_examples: int) -> List[Dict]:
     examples: List[Dict] = []
 
-    samples: pd.DataFrame = df.sample(frac=1)
+    samples: pd.DataFrame = df.sample(frac=1).reset_index(drop=True).drop_duplicates()
     samples = samples.dropna(subset=[target_column_name])
 
     for i, row in samples.head(max_examples).iterrows():
@@ -109,7 +109,7 @@ def get_instructions(df: pd.DataFrame, target_column_name: str, max_examples: in
             output_value=json.dumps({"index": row["index"], "output": row["output"]}, ensure_ascii=False),
         )
 
-    return builder.build()
+    return builder.improve().build()
 
 
 class FillNaResponse(BaseModel):

{openaivec-0.14.0.dist-info → openaivec-0.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openaivec
-Version: 0.14.0
+Version: 0.14.2
 Summary: Generative mutation for tabular calculation
 Project-URL: Homepage, https://microsoft.github.io/openaivec/
 Project-URL: Repository, https://github.com/microsoft/openaivec
@@ -98,7 +98,7 @@ survey_responses.assign(
 ).ai.extract("structured")  # Auto-expands to columns
 ```
 
-📓 **[See more examples →](https://microsoft.github.io/openaivec/examples/)**
+📓 **[See more examples →](https://microsoft.github.io/openaivec/examples/pandas/)**
 
 # Overview
 
@@ -746,7 +746,7 @@ uv run ruff check . --fix
 📓 **[Survey data transformation →](https://microsoft.github.io/openaivec/examples/survey_transformation/)** - Unstructured to structured data  
 📓 **[Asynchronous processing examples →](https://microsoft.github.io/openaivec/examples/aio/)** - High-performance async workflows  
 📓 **[Auto-generate FAQs from documents →](https://microsoft.github.io/openaivec/examples/generate_faq/)** - Create FAQs using AI  
-📓 **[All examples →](https://microsoft.github.io/openaivec/examples/)** - Complete collection of tutorials and use cases
+📓 **[All examples →](https://microsoft.github.io/openaivec/examples/pandas/)** - Complete collection of tutorials and use cases
 
 ## Community
 

{openaivec-0.14.0.dist-info → openaivec-0.14.2.dist-info}/RECORD

@@ -8,7 +8,7 @@ openaivec/_prompt.py,sha256=KoJbFK4gTEDRtu9OMweJq_jQLkSPFy2Kcvao30qKhAQ,20844
 openaivec/_provider.py,sha256=dNr9Y2C97GK-pkY81odurKoDup59dLK31V3EGT2HOwE,6711
 openaivec/_proxy.py,sha256=giOxRlCCO11XQ0gNVf2IksjZZj9RwvTHkHbmbQXadEk,28916
 openaivec/_responses.py,sha256=SSa52mCYh2jF52pHaXRtj0tJQUEn52Uy8ypMT57vSks,21428
-openaivec/_serialize.py,sha256=mSuSVfSpXMnsFZ4JjlOEe5b22ttBUHviPs9mF99lf0A,7277
+openaivec/_serialize.py,sha256=NLCKl4opc1WS24_duwpI2UGBepQ8SBh4YRxBlLwzDLw,8403
 openaivec/_util.py,sha256=dFWwjouJyvF-tqNPs2933OAt5Fw9I2Q2BvmGIfGH5k4,6423
 openaivec/pandas_ext.py,sha256=xOZvuJN6qPQZZmaHKr870kelG2XmuFHkqeAbFNayUZU,58825
 openaivec/spark.py,sha256=MqzS8nZe6R3wT_QJdopYSMZWDpagN6gyJNh-CAtvHA8,24403
@@ -28,8 +28,8 @@ openaivec/task/nlp/named_entity_recognition.py,sha256=oC6GuvfITlK-izTXoGGpYxVSOr
 openaivec/task/nlp/sentiment_analysis.py,sha256=BNwWtNT-MNA76eIJbb31641upukmRwM9ajfz8x398gE,3091
 openaivec/task/nlp/translation.py,sha256=XTZM11JFjbgTK9wHnxFgVDabXZ5bqbabXK_bq2nEkyQ,6627
 openaivec/task/table/__init__.py,sha256=kJz15WDJXjyC7UIHKBvlTRhCf347PCDMH5T5fONV2sU,83
-openaivec/task/table/fillna.py,sha256=hQn619rZoed0YJb9dX1uOKxvnfJtWy0yptzGC4yljCw,6585
-openaivec-0.14.0.dist-info/METADATA,sha256=R3pr1Ibcqn0MZnEwQqwlqRvss8oS8zas7lWksqUUTSk,27552
-openaivec-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-openaivec-0.14.0.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
-openaivec-0.14.0.dist-info/RECORD,,
+openaivec/task/table/fillna.py,sha256=ZVcOpuh7ULVhrt1VsWy5fPhk53XNaiD7kXGCPhh83M8,6636
+openaivec-0.14.2.dist-info/METADATA,sha256=LfzZLjiWdjJASIdXqIb0Tniou0ZYPYuwjsGtka9sIYg,27566
+openaivec-0.14.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+openaivec-0.14.2.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
+openaivec-0.14.2.dist-info/RECORD,,