openaivec 0.14.8__py3-none-any.whl → 0.14.9__py3-none-any.whl

openaivec/_dynamic.py

@@ -0,0 +1,350 @@
+from __future__ import annotations
+
+import re
+from enum import Enum
+from typing import Literal
+
+from pydantic import BaseModel, Field, create_model
+
+__all__: list[str] = []
+
+_MAX_ENUM_VALUES = 24
+
+
+class FieldSpec(BaseModel):
+    name: str = Field(
+        description=(
+            "Field name in lower_snake_case. Rules: (1) Use only lowercase letters, numbers, and underscores; "
+            "must start with a letter. (2) For numeric quantities append an explicit unit (e.g. 'duration_seconds', "
+            "'price_usd'). (3) Boolean fields use an affirmative 'is_' prefix (e.g. 'is_active'); avoid negative / "
+            "ambiguous forms like 'is_deleted' (prefer 'is_active', 'is_enabled'). (4) Name must be unique within the "
+            "containing object."
+        )
+    )
+    type: Literal[
+        "string",
+        "integer",
+        "float",
+        "boolean",
+        "enum",
+        "object",
+        "string_array",
+        "integer_array",
+        "float_array",
+        "boolean_array",
+        "enum_array",
+        "object_array",
+    ] = Field(
+        description=(
+            "Logical data type. Allowed values: string | integer | float | boolean | enum | object | string_array | "
+            "integer_array | float_array | boolean_array | enum_array | object_array. *_array variants represent a "
+            "homogeneous list of the base type. 'enum' / 'enum_array' require 'enum_spec'. 'object' / 'object_array' "
+            "require 'object_spec'. Primitives must not define 'enum_spec' or 'object_spec'."
+        )
+    )
+    description: str = Field(
+        description=(
+            "Human‑readable, concise explanation of the field's meaning and business intent. Should clarify units, "
+            "value semantics, and any domain constraints not captured by type. 1–2 sentences; no implementation notes."
+        )
+    )
+    enum_spec: EnumSpec | None = Field(
+        default=None,
+        description=(
+            "Enumeration specification for 'enum' / 'enum_array'. Must be provided (non-empty) for those types and "
+            "omitted for all others. Maximum size enforced by constant."
+        ),
+    )
+    object_spec: ObjectSpec | None = Field(
+        default=None,
+        description=(
+            "Nested object schema. Required for 'object' / 'object_array'; must be omitted for every other type. The "
+            "contained 'name' is used to derive the generated nested Pydantic model class name."
+        ),
+    )
+
+
+class EnumSpec(BaseModel):
+    """Enumeration specification for enum / enum_array field types.
+
+    Attributes:
+        name: Required Enum class name (UpperCamelCase). Must match ^[A-Z][A-Za-z0-9]*$. Previously optional; now
+            explicit to remove implicit coupling to the field name and make schemas self‑describing.
+        values: Raw label values (1–_MAX_ENUM_VALUES before de‑dup). Values are uppercased then
+            de-duplicated using a set; ordering of generated Enum members is not guaranteed. Any
+            casing variants collapse silently to a single member.
+    """
+
+    name: str = Field(
+        description=("Required Enum class name (UpperCamelCase). Valid pattern: ^[A-Z][A-Za-z0-9]*$."),
+    )
+    values: list[str] = Field(
+        description=(
+            f"Raw enum label values (1–{_MAX_ENUM_VALUES}). Uppercased then deduplicated; order of members "
+            "not guaranteed."
+        )
+    )
+
+
+class ObjectSpec(BaseModel):
+    name: str = Field(
+        description=(
+            "Object model class name in UpperCamelCase (singular noun). Must match ^[A-Z][A-Za-z0-9]*$ and is used "
+            "directly as the generated Pydantic model class name (no transformation)."
+        )
+    )
+    fields: list[FieldSpec] = Field(
+        description=(
+            "Non-empty list of FieldSpec definitions composing the object. Each field name must be unique; order is "
+            "preserved in the generated model."
+        )
+    )
+
+
+def _build_model(object_spec: ObjectSpec) -> type[BaseModel]:
+    lower_sname_pattern = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$")
+    upper_camel_pattern = re.compile(r"^[A-Z][A-Za-z0-9]*$")
+    type_map: dict[str, type] = {
+        "string": str,
+        "integer": int,
+        "float": float,
+        "boolean": bool,
+        "string_array": list[str],
+        "integer_array": list[int],
+        "float_array": list[float],
+        "boolean_array": list[bool],
+    }
+    output_fields: dict[str, tuple[type, object]] = {}
+
+    field_names: list[str] = [field.name for field in object_spec.fields]
+
+    # Assert that names of fields are not duplicated
+    if len(field_names) != len(set(field_names)):
+        raise ValueError("Field names must be unique within the object spec.")
+
+    for field in object_spec.fields:
+        # Assert that field names are lower_snake_case
+        if not lower_sname_pattern.match(field.name):
+            raise ValueError(f"Field name '{field.name}' must be in lower_snake_case format (e.g., 'my_field_name').")
+
+        # (EnumSpec.name now mandatory; no need to derive a fallback name from the field.)
+        match field:
+            case FieldSpec(
+                name=name,
+                type="string"
+                | "integer"
+                | "float"
+                | "boolean"
+                | "string_array"
+                | "integer_array"
+                | "float_array"
+                | "boolean_array",
+                description=description,
+                enum_spec=None,
+                object_spec=None,
+            ):
+                field_type = type_map[field.type]
+                output_fields[name] = (field_type, Field(description=description))
+
+            case FieldSpec(name=name, type="enum", description=description, enum_spec=enum_spec, object_spec=None) if (
+                enum_spec
+                and 0 < len(enum_spec.values) <= _MAX_ENUM_VALUES
+                and upper_camel_pattern.match(enum_spec.name)
+            ):
+                member_names = list({v.upper() for v in enum_spec.values})
+                enum_type = Enum(enum_spec.name, member_names)
+                output_fields[name] = (enum_type, Field(description=description))
+
+            case FieldSpec(
+                name=name, type="enum_array", description=description, enum_spec=enum_spec, object_spec=None
+            ) if (
+                enum_spec
+                and 0 < len(enum_spec.values) <= _MAX_ENUM_VALUES
+                and upper_camel_pattern.match(enum_spec.name)
+            ):
+                member_names = list({v.upper() for v in enum_spec.values})
+                enum_type = Enum(enum_spec.name, member_names)
+                output_fields[name] = (list[enum_type], Field(description=description))
+
+            case FieldSpec(
+                name=name, type="object", description=description, enum_spec=None, object_spec=object_spec
+            ) if object_spec and upper_camel_pattern.match(object_spec.name):
+                nested_model = _build_model(object_spec)
+                output_fields[name] = (nested_model, Field(description=description))
+
+            case FieldSpec(
+                name=name, type="object_array", description=description, enum_spec=None, object_spec=object_spec
+            ) if object_spec and upper_camel_pattern.match(object_spec.name):
+                nested_model = _build_model(object_spec)
+                output_fields[name] = (list[nested_model], Field(description=description))
+
+            # ---- Error cases (explicit reasons) ----
+            # Enum type without enum_spec (None or empty)
+            case FieldSpec(
+                name=name,
+                type="enum",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if not enum_spec or not enum_spec.values:
+                raise ValueError(f"Field '{name}': enum type requires non-empty enum_spec values list.")
+            # Enum type exceeding max length
+            case FieldSpec(
+                name=name,
+                type="enum",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if enum_spec and len(enum_spec.values) > _MAX_ENUM_VALUES:
+                raise ValueError(
+                    (
+                        f"Field '{name}': enum type supports at most {_MAX_ENUM_VALUES} enum_spec values "
+                        f"(got {len(enum_spec.values)})."
+                    )
+                )
+            # Enum type invalid explicit name pattern
+            case FieldSpec(
+                name=name,
+                type="enum",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if enum_spec and not upper_camel_pattern.match(enum_spec.name):
+                raise ValueError(
+                    (f"Field '{name}': enum_spec.name '{enum_spec.name}' invalid – must match ^[A-Z][A-Za-z0-9]*$")
+                )
+            # Enum type incorrectly provides an object_spec
+            case FieldSpec(
+                name=name,
+                type="enum",
+                enum_spec=enum_spec,
+                object_spec=object_spec,
+            ) if object_spec is not None:
+                raise ValueError(
+                    f"Field '{name}': enum type must not provide object_spec (got object_spec={object_spec!r})."
+                )
+            # Enum array type without enum_spec
+            case FieldSpec(
+                name=name,
+                type="enum_array",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if not enum_spec or not enum_spec.values:
+                raise ValueError(f"Field '{name}': enum_array type requires non-empty enum_spec values list.")
+            # Enum array type exceeding max length
+            case FieldSpec(
+                name=name,
+                type="enum_array",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if enum_spec and len(enum_spec.values) > _MAX_ENUM_VALUES:
+                raise ValueError(
+                    (
+                        f"Field '{name}': enum_array type supports at most {_MAX_ENUM_VALUES} enum_spec values "
+                        f"(got {len(enum_spec.values)})."
+                    )
+                )
+            # Enum array type invalid explicit name pattern
+            case FieldSpec(
+                name=name,
+                type="enum_array",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ) if enum_spec and not upper_camel_pattern.match(enum_spec.name):
+                raise ValueError(
+                    (f"Field '{name}': enum_spec.name '{enum_spec.name}' invalid – must match ^[A-Z][A-Za-z0-9]*$")
+                )
+            # Enum array type incorrectly provides an object_spec
+            case FieldSpec(
+                name=name,
+                type="enum_array",
+                enum_spec=enum_spec,
+                object_spec=object_spec,
+            ) if object_spec is not None:
+                raise ValueError(
+                    f"Field '{name}': enum_array type must not provide object_spec (got object_spec={object_spec!r})."
+                )
+            # Object type missing object_spec
+            case FieldSpec(
+                name=name,
+                type="object",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ):
+                raise ValueError(f"Field '{name}': object type requires object_spec (got object_spec=None).")
+            # Object array type missing object_spec
+            case FieldSpec(
+                name=name,
+                type="object_array",
+                enum_spec=enum_spec,
+                object_spec=None,
+            ):
+                raise ValueError(f"Field '{name}': object_array type requires object_spec (got object_spec=None).")
+            # Object/object_array provided but invalid name pattern
+            case FieldSpec(
+                name=name,
+                type="object" | "object_array",
+                enum_spec=enum_spec,
+                object_spec=object_spec,
+            ) if object_spec is not None and not upper_camel_pattern.match(object_spec.name):
+                raise ValueError(
+                    (
+                        f"Field '{name}': object_spec.name '{object_spec.name}' must be UpperCamelCase "
+                        "(regex ^[A-Z][A-Za-z0-9]*$) and contain only letters and digits."
+                    )
+                )
+            # Object/object_array types must not provide enum_spec
+            case FieldSpec(
+                name=name,
+                type="object" | "object_array",
+                enum_spec=enum_spec,
+                object_spec=object_spec,
+            ) if enum_spec is not None:
+                raise ValueError(
+                    f"Field '{name}': {field.type} must not define enum_spec (got enum_spec={enum_spec!r})."
+                )
+            # Primitive / simple array types must not have enum_spec
+            case FieldSpec(
+                name=name,
+                type="string"
+                | "integer"
+                | "float"
+                | "boolean"
+                | "string_array"
+                | "integer_array"
+                | "float_array"
+                | "boolean_array",
+                enum_spec=enum_spec,
+                object_spec=object_spec,
+            ) if enum_spec is not None:
+                raise ValueError(
+                    (f"Field '{name}': type '{field.type}' must not define enum_spec (got enum_spec={enum_spec!r}).")
+                )
+            # Primitive / simple array types must not have object_spec
+            case FieldSpec(
+                name=name,
+                type="string"
+                | "integer"
+                | "float"
+                | "boolean"
+                | "string_array"
+                | "integer_array"
+                | "float_array"
+                | "boolean_array",
+                enum_spec=None,
+                object_spec=object_spec,
+            ) if object_spec is not None:
+                raise ValueError(
+                    (
+                        f"Field '{name}': type '{field.type}' must not define object_spec "
+                        f"(got object_spec={object_spec!r})."
+                    )
+                )
+            # Any other unmatched combination
+            case FieldSpec() as f:
+                raise ValueError(
+                    (
+                        "Field configuration invalid / unrecognized combination: "
+                        f"name={f.name!r}, type={f.type!r}, enum_spec={'set' if f.enum_spec else None}, "
+                        f"object_spec={'set' if f.object_spec else None}."
+                    )
+                )
+
+    return create_model(object_spec.name, **output_fields)

openaivec/_schema.py

@@ -3,165 +3,89 @@
 This (non-public) module converts a small *representative* sample of free‑text
 examples plus a *purpose* statement into:
 
-1. A vetted, flat list of scalar field specifications (``FieldSpec``) that can
-    be *reliably* extracted across similar future inputs.
+1. A vetted hierarchical object specification (``ObjectSpec``) whose recursively
+     defined ``fields`` (``FieldSpec``) capture reliably extractable signals.
 2. A reusable, self‑contained extraction prompt (``inference_prompt``) that
-    freezes the agreed schema contract (no additions / renames / omissions).
-3. A dynamically generated Pydantic model whose fields mirror the inferred
-    schema, enabling immediate typed parsing with the OpenAI Responses API.
+     freezes the agreed schema contract (no additions / renames / omissions).
+3. A dynamically generated Pydantic model mirroring the hierarchical schema,
+     enabling immediate typed parsing with the OpenAI Responses API.
 4. A ``PreparedTask`` wrapper (``InferredSchema.task``) for downstream batched
-    responses/structured extraction flows in pandas or Spark.
+     responses / structured extraction flows in pandas or Spark.
 
 Core goals:
 * Minimize manual, subjective schema design iterations.
 * Enforce objective naming / typing / enum rules early (guard rails rather than
-  after‑the‑fact cleaning).
-* Provide deterministic reusability: the same prompt + model yield stable
-  column ordering & types for analytics or feature engineering.
+    after‑the‑fact cleaning).
+* Provide deterministic reusability: the same prompt + model yield stable field
+    ordering & types for analytics or feature engineering.
 * Avoid outcome / target label leakage in predictive (feature engineering)
-  contexts by explicitly excluding direct target restatements.
+    contexts by explicitly excluding direct target restatements.
 
 This module is intentionally **internal** (``__all__ = []``). Public users
 should interact through higher‑level batch APIs once a schema has been inferred.
 
-Design constraints:
-* Flat schema only (no nested objects). Top-level arrays permitted ONLY as homogeneous arrays of primitives
-    (e.g. array of strings) – represented via specialized primitive array type names
-    (string_array, integer_array, float_array, boolean_array).
-* Primitive scalar types limited to {string, integer, float, boolean}; optional array variants
-    {string_array, integer_array, float_array, boolean_array}.
-* Optional enumerations for *closed*, *observed* categorical sets only.
+Design constraints (updated):
+* Root: single ``ObjectSpec`` (UpperCamelCase name) containing one or more fields.
+* Field types: string | integer | float | boolean | enum | object |
+    string_array | integer_array | float_array | boolean_array | enum_array | object_array
+* Arrays are homogeneous lists of their base type.
+* Nested objects / arrays of objects are allowed when semantically cohesive; keep
+    depth shallow and avoid gratuitous nesting.
+* Enumerations use ``enum_spec`` with explicit ``name`` (UpperCamelCase) and 1–24
+    raw label values (project constant). Values collapse by uppercasing; order not guaranteed.
+* Field names: lower_snake_case; unique per containing object.
+* Boolean names: affirmative 'is_' prefix.
+* Numeric (integer/float) names encode unit / measure suffix (e.g. *_count, *_ratio, *_ms).
 * Validation retries ensure a structurally coherent suggestion before returning.
 
 Example (conceptual):
-     from openai import OpenAI
-     client = OpenAI()
-     inferer = SchemaInferer(client=client, model_name="gpt-4.1-mini")
-     schema = inferer.infer_schema(
-          SchemaInferenceInput(
-                examples=["Order #123 delayed due to weather", "Order #456 delivered"],
-                purpose="Extract operational status signals for logistics analytics",
-          )
-     )
-     Model = schema.model  # dynamic Pydantic model
-     task = schema.task    # PreparedTask for batch extraction
+        from openai import OpenAI
+        client = OpenAI()
+        inferer = SchemaInferer(client=client, model_name="gpt-4.1-mini")
+        schema = inferer.infer_schema(
+                SchemaInferenceInput(
+                        examples=["Order #123 delayed due to weather", "Order #456 delivered"],
+                        purpose="Extract operational status signals for logistics analytics",
+                )
+        )
+        Model = schema.model  # dynamic Pydantic model
+        task = schema.task    # PreparedTask for batch extraction
 
 The implementation purposefully does *not* emit or depend on JSON Schema; the
-authoritative contract is the ordered list of ``FieldSpec`` instances.
+authoritative contract is the recursive ``ObjectSpec`` tree.
 """
 
 from dataclasses import dataclass
-from enum import Enum
-from typing import Literal
 
 from openai import OpenAI
 from openai.types.responses import ParsedResponse
-from pydantic import BaseModel, Field, create_model
+from pydantic import BaseModel, Field
 
+from openaivec._dynamic import ObjectSpec, _build_model
 from openaivec._model import PreparedTask
 
 # Internal module: explicitly not part of public API
 __all__: list[str] = []
 
 
-class FieldSpec(BaseModel):
-    """Specification for a single candidate output field.
-
-    Each ``FieldSpec`` encodes a *flat*, scalar, semantically atomic unit the
-    model should extract. These become columns in downstream DataFrames.
-
-    Validation focuses on: objective naming, primitive typing, and *optional*
-    closed categorical vocabularies. Enumerations are intentionally conservative
-    (must derive from clear evidence) to reduce over‑fitted schemas.
-
-    Attributes:
-        name: Lower snake_case unique identifier (regex ^[a-z][a-z0-9_]*$). Avoid
-            subjective modifiers ("best", "great", "high_quality").
-        type: One of ``string|integer|float|boolean``. ``integer`` only if all
-            observed numeric values are whole numbers; ``float`` if any decimal
-            or ratio appears. ``boolean`` strictly for explicit binary forms.
-        description: Concise, objective extraction rule (what qualifies / what
-            to ignore). Disambiguate from overlapping fields if needed.
-        enum_values: Optional stable closed set of lowercase string labels
-            (2–24). Only for *string* type when the vocabulary is clearly
-            evidenced; never hallucinate or extrapolate.
-    """
-
-    name: str = Field(
-        description=(
-            "Lower snake_case identifier (regex: ^[a-z][a-z0-9_]*$). Must be unique across all fields and "
-            "express the semantic meaning succinctly (no adjectives like 'best', 'great'). For numeric (integer|float) "
-            "fields the name MUST include an explicit unit or measure suffix (e.g. _count, _total_count, "
-            "_duration_seconds, _ms, _price_usd, _ratio, _score) to eliminate ambiguity. Avoid bare numeric nouns like "
-            "'duration' or 'value' without unit/scale. Boolean field names MUST begin with 'is_' followed by a "
-            "descriptive predicate (e.g. is_active, is_delayed). Use positive forms (is_active) rather than "
-            "negated forms (is_not_active)."
-        )
-    )
-    type: Literal[
-        "string",
-        "integer",
-        "float",
-        "boolean",
-        "string_array",
-        "integer_array",
-        "float_array",
-        "boolean_array",
-    ] = Field(
-        description=(
-            "Primitive type. Use 'integer' only if all observed numeric values are whole numbers. "
-            "Use 'float' if any value can contain a decimal or represents a ratio/score. Use 'boolean' only for "
-            "explicit binary states (yes/no, true/false, present/absent) consistently encoded. Use 'string' otherwise. "
-            "Array variants (string_array, integer_array, float_array, boolean_array) are ONLY allowed when the value "
-            "is a repeatable homogeneous collection whose individual elements would otherwise stand as valid scalar "
-            "extractions (e.g. keywords, error_codes, tag_ids). Do not encode objects or mixed-type arrays; flatten or "
-            "choose the most informative level."
-        )
-    )
-    description: str = Field(
-        description=(
-            "Concise, objective definition plus extraction rule (what qualifies / what to ignore). Avoid subjective, "
-            "speculative, or promotional language. If ambiguity exists with another field, clarify the distinction. "
-            "Do NOT simply restate an original JSON/key name if the examples are already structured; only include a "
-            "raw key verbatim when it is already the minimal, irreducible analytic unit. For derived fields, clearly "
-            "state the transformation (e.g. sentiment of comment_text, normalized date, language code)."
-        )
-    )
-    enum_values: list[str] | None = Field(
-        default=None,
-        description=(
-            "Optional finite categorical label set (classification) for a string field. Provide ONLY when a closed, "
-            "stable vocabulary (2–24 lowercase tokens) is clearly evidenced or strongly implied by examples. "
-            "Do NOT invent labels. Omit if open-ended or ambiguous. Order must be stable and semantically natural."
-        ),
-    )
-
-
 class InferredSchema(BaseModel):
     """Result of a schema inference round.
 
-    Contains the normalized *purpose*, an objective *examples_summary*, the
-    ordered ``fields`` contract, and the canonical reusable ``inference_prompt``.
-
-    The prompt is constrained to be fully derivable from the other components;
-    adding novel unstated facts is disallowed to preserve traceability.
+    Contains the normalized *purpose*, objective *examples_summary*, the root
+    hierarchical ``object_spec`` contract, and the canonical reusable
+    ``inference_prompt``. The prompt MUST be fully derivable from the other
+    components (no new unstated facts) to preserve traceability.
 
     Attributes:
-        purpose: Unambiguous restatement of the user's objective (noise &
-            redundancy removed).
+        purpose: Unambiguous restatement of the user's objective.
         examples_summary: Neutral description of structural / semantic patterns
-            observed in the examples (domain, recurring signals, constraints).
-        examples_purpose_alignment: Analytical explanation of how the concrete
-            recurring patterns in the provided examples *justify*, *constrain*,
-            or *refine* the stated purpose. Should map purpose facets to
-            observed evidence (or explicitly note gaps) to discourage
-            hallucinated fields and anchor extraction scope. This is an
-            internal quality aid – downstream consumers typically ignore it.
-        fields: Ordered list of ``FieldSpec`` objects comprising the schema's
-            sole authoritative contract.
-        inference_prompt: Self-contained extraction instructions enforcing an
-            exact field set (names, order, primitive types) with prohibition on
-            alterations or subjective flourishes.
+            observed in the examples.
+        examples_purpose_alignment: Mapping from purpose facets to concrete
+            recurring evidence (or explicit gaps) anchoring extraction scope.
+        object_spec: Root ``ObjectSpec`` (UpperCamelCase name) whose ``fields``
+            recursively define the extraction schema.
+        inference_prompt: Canonical instructions enforcing exact field names,
+            hierarchy, and types (no additions/removals/renames).
     """
 
     purpose: str = Field(
@@ -183,20 +107,17 @@ class InferredSchema(BaseModel):
             "reduce hallucinated fields. Internal diagnostic / quality aid; not required for downstream extraction."
         )
     )
-    fields: list[FieldSpec] = Field(
+    object_spec: ObjectSpec = Field(
         description=(
-            "Ordered list of proposed fields derived strictly from observable, repeatable signals in the "
-            "examples and aligned with the purpose."
+            "Root ObjectSpec (recursive). Each contained object's field list is unique-name ordered and derived "
+            "strictly from observable, repeatable signals aligned with the purpose."
         )
     )
     inference_prompt: str = Field(
         description=(
-            "Canonical, reusable extraction prompt for structuring future inputs with this schema. "
-            "Must be fully derivable from 'purpose', 'examples_summary', and 'fields' (no new unstated facts or "
-            "speculation). It MUST: (1) instruct the model to output only the listed fields with the exact names "
-            "and primitive types; (2) forbid adding, removing, or renaming fields; (3) avoid subjective or "
-            "marketing language; (4) be self-contained (no TODOs, no external references, no unresolved "
-            "placeholders). Intended for direct reuse as the prompt for deterministic alignment with 'fields'."
+            "Canonical, reusable extraction prompt. Must be derivable from purpose + summaries + object_spec. Enforces "
+            "exact hierarchical field set (names, order per object, types) forbidding additions, removals, renames, or "
+            "subjective language. Self-contained (no TODOs, external refs, or placeholders)."
         )
     )
 
@@ -236,53 +157,7 @@ class InferredSchema(BaseModel):
         )
 
     def build_model(self) -> type[BaseModel]:
-        """Create a new dynamic ``BaseModel`` class adhering to this schema.
-
-        Implementation details:
-            * Maps primitive types: string→``str``, integer→``int``, float→``float``, boolean→``bool``.
-            * For enumerated string fields, constructs an ad‑hoc ``Enum`` subclass with
-              stable member names (collision‑safe, normalized to ``UPPER_SNAKE``).
-            * All fields are required (ellipsis ``...``). Optionality can be
-              introduced later by modifying this logic if needed.
-
-        Returns:
-            type[BaseModel]: New (not cached) model type; order matches ``fields``.
-        """
-        type_map: dict[str, type] = {
-            "string": str,
-            "integer": int,
-            "float": float,
-            "boolean": bool,
-        }
-        fields: dict[str, tuple[type, object]] = {}
-
-        for spec in self.fields:
-            py_type: type
-            if spec.enum_values:
-                enum_class_name = "Enum_" + "".join(part.capitalize() for part in spec.name.split("_"))
-                members: dict[str, str] = {}
-                for raw in spec.enum_values:
-                    sanitized = raw.upper().replace("-", "_").replace(" ", "_")
-                    if not sanitized or sanitized[0].isdigit():
-                        sanitized = f"V_{sanitized}"
-                    base = sanitized
-                    i = 2
-                    while sanitized in members:
-                        sanitized = f"{base}_{i}"
-                        i += 1
-                    members[sanitized] = raw
-                enum_cls = Enum(enum_class_name, members)  # type: ignore[arg-type]
-                py_type = enum_cls
-            else:
-                if spec.type.endswith("_array"):
-                    base = spec.type.rsplit("_", 1)[0]
-                    py_type = list[type_map[base]]  # type: ignore[index]
-                else:
-                    py_type = type_map[spec.type]
-            fields[spec.name] = (py_type, Field(description=spec.description))
-
-        model = create_model("InferredSchema", **fields)  # type: ignore[call-arg]
-        return model
+        return _build_model(self.object_spec)
 
     def save(self, path: str) -> None:
         """Persist this inferred schema as pretty‑printed JSON.
@@ -326,56 +201,41 @@ You are a schema inference engine.
 Task:
 1. Normalize the user's purpose (eliminate ambiguity, redundancy, contradictions).
 2. Objectively summarize observable patterns in the example texts.
-3. Produce an "examples_purpose_alignment" explanation that explicitly maps purpose facets
-   to concrete recurring evidence in the examples (or flags gaps). Use concise bullet‑style
-   sentences (still a plain string) such as: "purpose facet -> supporting pattern / gap".
-   This MUST NOT introduce new domain facts beyond the examples & purpose.
-4. Propose a minimal flat set of scalar fields (and ONLY when justified,
-   homogeneous primitive arrays) that are reliably extractable.
+3. Produce an "examples_purpose_alignment" explanation mapping purpose facets to concrete recurring evidence (or gaps).
+4. Propose a minimal hierarchical schema (root ObjectSpec) comprised of reliably extractable fields. Use nesting ONLY
+     when a group of fields forms a cohesive sub-entity repeated in the data; otherwise keep flat.
 5. Skip fields likely missing in a large share (>~20%) of realistic inputs.
-6. Provide enum_values ONLY when a small stable closed categorical set (2–24 lowercase tokens)
-    is clearly evidenced; never invent.
-7. If the purpose indicates prediction (predict / probability / likelihood), output only
-    explanatory features (no target restatement).
+6. Provide enum_spec ONLY when a small stable closed categorical set (1–{_MAX_ENUM_VALUES} raw tokens) is clearly
+     evidenced; never invent unseen categories.
+7. If the purpose indicates prediction (predict / probability / likelihood),
+   output only explanatory features (no target restatement).
 
 Rules:
-- Names: lower snake_case, unique, regex ^[a-z][a-z0-9_]*$, no subjective adjectives.
-- Types: string | integer | float | boolean
-    * integer = all whole numbers
-    * float = any decimals / ratios
-    * boolean = explicit binary
-    * else use string
-- Numeric (integer|float) field names MUST encode an explicit unit / scale / measure suffix
-    (e.g. *_count, *_seconds, *_ms, *_usd, *_ratio, *_score). Avoid ambiguous bare numeric names.
-- Boolean field names MUST start with 'is_' followed by a positive predicate (e.g. is_active,
-  is_delayed). Avoid negated forms.
-- No nested objects or mixed-type arrays. Homogeneous primitive arrays are allowed ONLY if each element is an atomic
-    scalar signal (use *_array types: string_array, integer_array, float_array, boolean_array). The array is expected to
-    contain 0..N such elements per record.
-- Array field names MUST end with '_array' (e.g. keywords_array, tag_ids_array). Do not use plural-only forms
-    (e.g. keywords) for arrays; the suffix makes container semantics explicit.
-- Descriptions: concise, objective extraction rules (no marketing/emotion/speculation).
-- enum_values only for string fields with stable closed vocab; omit otherwise.
-- Exclude direct outcome labels (e.g. attrition_probability, will_buy, purchase_likelihood)
-    in predictive / feature engineering contexts.
-- When examples already appear as serialized JSON / key-value records, DO NOT merely relist the
-    raw original keys unless each is already an atomic, irreducible analytic signal. Prefer high-signal
-    derived / normalized / aggregated features (e.g. sentiment, category, language_code, boolean flags,
-    normalized_date, count metrics).
-- Superficial renames (adding trivial prefixes/suffixes like _value, _field, new_) are forbidden; a new
-    field name must reflect a semantic transformation.
-- Keep field count focused (typically <= 12) prioritizing reusable analytical / ML features over low-signal
-    restatements.
-- If you retain an original raw key unchanged, its description must justify why it is minimal and cannot
-    be further decomposed without losing analytical value.
+- Field names: lower snake_case, unique within each object, regex ^[a-z][a-z0-9_]*$, no subjective adjectives.
+- Field types: string | integer | float | boolean | enum | object | string_array | integer_array | float_array |
+    boolean_array | enum_array | object_array
+    * *_array are homogeneous lists of their primitive / enum / object base type.
+    * Use object/object_array ONLY for semantically cohesive grouped attributes; avoid gratuitous layers.
+- Enumerations: use enum_spec { name (UpperCamelCase), values [raw_tokens...] }. values length 1–{_MAX_ENUM_VALUES}.
+    Use ONLY when closed set is evidenced. Otherwise, use string.
+- Numeric (integer|float) names encode explicit unit/measure suffix (e.g. *_count, *_seconds, *_usd, *_ratio, *_score).
+- Boolean names start with 'is_' followed by positive predicate (no negations like is_not_*).
+- Array field names SHOULD end with '_array' for primitive/enum arrays; object_array
+    fields may use plural noun or *_array pattern.
+- Descriptions: concise, objective extraction criteria (no marketing/emotion/speculation).
+- Exclude direct outcome labels in predictive contexts.
+- Avoid superficial renames; semantic transformation only.
+- Keep total field count focused (typically <= 16) optimizing for reusable analytical / ML features.
 
 Output contract:
-Return exactly an InferredSchema object with JSON keys:
-    - purpose (string)
-    - examples_summary (string)
-    - examples_purpose_alignment (string)
-    - fields (array of FieldSpec objects: name, type, description, enum_values?)
-    - inference_prompt (string)
+Return exactly an InferredSchema JSON object with keys:
+        - purpose (string)
+        - examples_summary (string)
+        - examples_purpose_alignment (string)
+        - object_spec (ObjectSpec: name, fields[list[FieldSpec]])
+        - inference_prompt (string)
+Where each FieldSpec includes: name, type, description, optional enum_spec (for
+enum / enum_array), optional object_spec (for object / object_array).
 """.strip()
 
 
@@ -401,14 +261,15 @@ class SchemaInferer:
     client: OpenAI
     model_name: str
 
-    def infer_schema(self, data: "SchemaInferenceInput", *args, max_retries: int = 3, **kwargs) -> "InferredSchema":
+    def infer_schema(self, data: SchemaInferenceInput, *args, max_retries: int = 8, **kwargs) -> InferredSchema:
         """Infer a validated schema from representative examples.
 
-        Workflow:
-            1. Submit ``SchemaInferenceInput`` (JSON) + instructions via
-               ``responses.parse`` requesting an ``InferredSchema`` object.
-            2. Validate the returned field list with ``_basic_field_list_validation``.
-            3. Retry (up to ``max_retries``) if validation fails.
+          Workflow:
+                1. Submit ``SchemaInferenceInput`` (JSON) + instructions via
+                    ``responses.parse`` requesting an ``InferredSchema`` object.
+                2. Attempt dynamic model build (``parsed.build_model()``) which performs recursive
+                    structural validation (names, types, enum/object specs) via the dynamic layer.
+                3. Retry (up to ``max_retries``) on validation failure.
 
         Args:
             data (SchemaInferenceInput): Representative examples + purpose.
@@ -460,55 +321,17 @@ class SchemaInferer:
             )
             parsed = response.output_parsed
             try:
-                _basic_field_list_validation(parsed)
-                parsed.build_model()  # ensure dynamic model creation succeeds
+                # Validate the field list structure
+                parsed.build_model()
+                return parsed
             except ValueError as e:
                 last_err = e
                 previous_errors.append(str(e))
                 if attempt == max_retries - 1:
-                    raise
-                continue
-            return parsed
-        if last_err:  # pragma: no cover
-            raise last_err
-        raise RuntimeError("unreachable retry loop state")  # pragma: no cover
-
-
-def _basic_field_list_validation(parsed: InferredSchema) -> None:
-    """Lightweight structural validation of an inferred field list.
+                    raise ValueError(
+                        f"Schema validation failed after {max_retries} attempts. Last error: {last_err}"
+                    ) from last_err
 
-    Checks:
-        * Non-empty field set.
-        * No duplicate names.
-        * All types in the allowed primitive set.
-        * ``enum_values`` only on string fields and size within bounds (2–24).
-
-    Args:
-        parsed (InferredSchema): Candidate ``InferredSchema`` instance.
-
-    Raises:
-        ValueError: Any invariant is violated.
-    """
-    names = [f.name for f in parsed.fields]
-    if not names:
-        raise ValueError("no fields suggested")
-    if len(names) != len(set(names)):
-        raise ValueError("duplicate field names detected")
-    allowed = {
-        "string",
-        "integer",
-        "float",
-        "boolean",
-        "string_array",
-        "integer_array",
-        "float_array",
-        "boolean_array",
-    }
-    for f in parsed.fields:
-        if f.type not in allowed:
-            raise ValueError(f"unsupported field type: {f.type}")
-        if f.enum_values is not None:
-            if f.type != "string":
-                raise ValueError(f"enum_values only allowed for plain string field: {f.name}")
-            if not (2 <= len(f.enum_values) <= 24):
-                raise ValueError(f"enum_values length out of bounds for field {f.name}")
+        if last_err:
+            raise last_err
+        raise RuntimeError("unreachable retry loop state")

{openaivec-0.14.8.dist-info → openaivec-0.14.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openaivec
-Version: 0.14.8
+Version: 0.14.9
 Summary: Generative mutation for tabular calculation
 Project-URL: Homepage, https://microsoft.github.io/openaivec/
 Project-URL: Repository, https://github.com/microsoft/openaivec

{openaivec-0.14.8.dist-info → openaivec-0.14.9.dist-info}/RECORD

@@ -1,5 +1,6 @@
 openaivec/__init__.py,sha256=mXCGNNTjYbmE4CAXGvAs78soxUsoy_mxxnvaCk_CL6Y,361
 openaivec/_di.py,sha256=1MXaBzaH_ZenQnWKQzBY2z-egHwiteMvg7byoUH3ZZI,10658
+openaivec/_dynamic.py,sha256=7ZaC59w2Edemnao57XeZVO4qmSOA-Kus6TchZC3Dd5o,14821
 openaivec/_embeddings.py,sha256=upCjl8m9h1CihP6t7wvIH_vivOAPSgmgooAxIhnUMUw,7449
 openaivec/_log.py,sha256=LHNs6AbJzM4weaRARZFroigxR6D148d7WSIMLk1IhbU,1439
 openaivec/_model.py,sha256=toS2oBubrJa9jrdYy-87Fb2XivjXUlk_8Zn5gKUAcFI,3345
@@ -8,7 +9,7 @@ openaivec/_prompt.py,sha256=zLv13q47CKV3jnETUyWAIlnjXFSEMs70c8m0yN7_Hek,20820
 openaivec/_provider.py,sha256=YLrEcb4aWBD1fj0n6PNcJpCtEXK6jkUuRH_WxcLDCuI,7145
 openaivec/_proxy.py,sha256=AiGuC1MCFjZCRXCac-pHUI3Np3nf1HIpWY6nC9ZVCFY,29671
 openaivec/_responses.py,sha256=lVJRa_Uc7hQJnYJRgumqwBbu6GToZqsLFS6tIAFO1Fc,24014
-openaivec/_schema.py,sha256=fVsFkCZWSbh2-fiGxnT8cSVrlUQOYWJX5EeL2F6aX4s,24039
+openaivec/_schema.py,sha256=RKjDPqet1TlReYibah0R0NIvCV1VWN5SZxiaBeV0gCY,15492
 openaivec/_serialize.py,sha256=u2Om94Sc_QgJkTlW2BAGw8wd6gYDhc6IRqvS-qevFSs,8399
 openaivec/_util.py,sha256=XfueAycVCQvgRLS7wF7e306b53lebORvZOBzbQjy4vE,6438
 openaivec/pandas_ext.py,sha256=rCkh8g9eqHn0gUG8j_-jdppQt_Yq_1Wg6FmsCEcpv3k,85985
@@ -30,7 +31,7 @@ openaivec/task/nlp/sentiment_analysis.py,sha256=Np-yY0d4Kr5WEjGjq4tNFHDNarBLajJr
 openaivec/task/nlp/translation.py,sha256=VYgiXtr2TL1tbqZkBpyVAy4ahrgd8UO4ZjhIL6xMdkI,6609
 openaivec/task/table/__init__.py,sha256=kJz15WDJXjyC7UIHKBvlTRhCf347PCDMH5T5fONV2sU,83
 openaivec/task/table/fillna.py,sha256=g_CpLnLzK1C5rCiVq15L3X0kywJK6CtSrKRYxQFuhn8,6606
-openaivec-0.14.8.dist-info/METADATA,sha256=ItqzTCNsPigyX9fe5WBQHih3gzT68XjdwFsAOa9-qrI,27566
-openaivec-0.14.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-openaivec-0.14.8.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
-openaivec-0.14.8.dist-info/RECORD,,
+openaivec-0.14.9.dist-info/METADATA,sha256=C7UqwVFLIVYiMJdRdUMTuUbhamacXoM2EHfS1nIxROQ,27566
+openaivec-0.14.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+openaivec-0.14.9.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
+openaivec-0.14.9.dist-info/RECORD,,