cocoindex 0.1.73__cp313-cp313-macosx_11_0_arm64.whl → 0.1.74__cp313-cp313-macosx_11_0_arm64.whl

cocoindex/convert.py

@@ -2,9 +2,12 @@
 Utilities to convert between Python and engine values.
 """
 
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import inspect
+import warnings
 from enum import Enum
 from typing import Any, Callable, Mapping, get_origin
 
@@ -29,6 +32,24 @@ from .typing import (
 )
 
 
+class ChildFieldPath:
+    """Context manager to append a field to field_path on enter and pop it on exit."""
+
+    _field_path: list[str]
+    _field_name: str
+
+    def __init__(self, field_path: list[str], field_name: str):
+        self._field_path: list[str] = field_path
+        self._field_name = field_name
+
+    def __enter__(self) -> ChildFieldPath:
+        self._field_path.append(self._field_name)
+        return self
+
+    def __exit__(self, _exc_type: Any, _exc_val: Any, _exc_tb: Any) -> None:
+        self._field_path.pop()
+
+
 def encode_engine_value(value: Any) -> Any:
     """Encode a Python value to an engine value."""
     if dataclasses.is_dataclass(value):
@@ -73,7 +94,7 @@ def _is_type_kind_convertible_to(src_type_kind: str, dst_type_kind: str) -> bool
 def make_engine_value_decoder(
     field_path: list[str],
     src_type: dict[str, Any],
-    dst_annotation: Any,
+    dst_type_info: AnalyzedTypeInfo,
 ) -> Callable[[Any], Any]:
     """
     Make a decoder from an engine value to a Python value.
@@ -89,7 +110,6 @@ def make_engine_value_decoder(
 
     src_type_kind = src_type["kind"]
 
-    dst_type_info = analyze_type_info(dst_annotation)
     dst_type_variant = dst_type_info.variant
 
     if isinstance(dst_type_variant, AnalyzedUnknownType):
@@ -99,96 +119,97 @@ def make_engine_value_decoder(
         )
 
     if src_type_kind == "Struct":
-        return _make_engine_struct_value_decoder(
+        return make_engine_struct_decoder(
             field_path,
             src_type["fields"],
             dst_type_info,
         )
 
     if src_type_kind in TABLE_TYPES:
-        field_path.append("[*]")
-        engine_fields_schema = src_type["row"]["fields"]
+        with ChildFieldPath(field_path, "[*]"):
+            engine_fields_schema = src_type["row"]["fields"]
 
-        if src_type_kind == "LTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ltable_to_list_dict_decoder(
-                    field_path, engine_fields_schema
-                )
-            if not isinstance(dst_type_variant, AnalyzedListType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a list type expected"
+            if src_type_kind == "LTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    return _make_engine_ltable_to_list_dict_decoder(
+                        field_path, engine_fields_schema
+                    )
+                if not isinstance(dst_type_variant, AnalyzedListType):
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a list type expected"
+                    )
+                row_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema,
+                    analyze_type_info(dst_type_variant.elem_type),
                 )
-            row_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema,
-                analyze_type_info(dst_type_variant.elem_type),
-            )
 
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return [row_decoder(v) for v in value]
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return [row_decoder(v) for v in value]
 
-        elif src_type_kind == "KTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ktable_to_dict_dict_decoder(
-                    field_path, engine_fields_schema
+            elif src_type_kind == "KTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    return _make_engine_ktable_to_dict_dict_decoder(
+                        field_path, engine_fields_schema
+                    )
+                if not isinstance(dst_type_variant, AnalyzedDictType):
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a dict type expected"
+                    )
+
+                key_field_schema = engine_fields_schema[0]
+                field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+                key_decoder = make_engine_value_decoder(
+                    field_path,
+                    key_field_schema["type"],
+                    analyze_type_info(dst_type_variant.key_type),
                 )
-            if not isinstance(dst_type_variant, AnalyzedDictType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a dict type expected"
+                field_path.pop()
+                value_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema[1:],
+                    analyze_type_info(dst_type_variant.value_type),
                 )
 
-            key_field_schema = engine_fields_schema[0]
-            field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-            key_decoder = make_engine_value_decoder(
-                field_path, key_field_schema["type"], dst_type_variant.key_type
-            )
-            field_path.pop()
-            value_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema[1:],
-                analyze_type_info(dst_type_variant.value_type),
-            )
-
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
 
-        field_path.pop()
         return decode
 
     if src_type_kind == "Union":
         if isinstance(dst_type_variant, AnalyzedAnyType):
             return lambda value: value[1]
 
-        dst_type_variants = (
-            dst_type_variant.variant_types
+        dst_type_info_variants = (
+            [analyze_type_info(t) for t in dst_type_variant.variant_types]
             if isinstance(dst_type_variant, AnalyzedUnionType)
-            else [dst_annotation]
+            else [dst_type_info]
         )
         src_type_variants = src_type["types"]
         decoders = []
         for i, src_type_variant in enumerate(src_type_variants):
-            src_field_path = field_path + [f"[{i}]"]
-            decoder = None
-            for dst_type_variant in dst_type_variants:
-                try:
-                    decoder = make_engine_value_decoder(
-                        src_field_path, src_type_variant, dst_type_variant
+            with ChildFieldPath(field_path, f"[{i}]"):
+                decoder = None
+                for dst_type_info_variant in dst_type_info_variants:
+                    try:
+                        decoder = make_engine_value_decoder(
+                            field_path, src_type_variant, dst_type_info_variant
+                        )
+                        break
+                    except ValueError:
+                        pass
+                if decoder is None:
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"cannot find matched target type for source type variant {src_type_variant}"
                     )
-                    break
-                except ValueError:
-                    pass
-            if decoder is None:
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"cannot find matched target type for source type variant {src_type_variant}"
-                )
-            decoders.append(decoder)
+                decoders.append(decoder)
         return lambda value: decoders[value[0]](value[1])
 
     if isinstance(dst_type_variant, AnalyzedAnyType):
@@ -216,7 +237,7 @@ def make_engine_value_decoder(
             vec_elem_decoder = make_engine_value_decoder(
                 field_path + ["[*]"],
                 src_type["element_type"],
-                dst_type_variant and dst_type_variant.elem_type,
+                analyze_type_info(dst_type_variant and dst_type_variant.elem_type),
             )
 
         def decode_vector(value: Any) -> Any | None:
@@ -247,7 +268,7 @@ def make_engine_value_decoder(
         if not _is_type_kind_convertible_to(src_type_kind, dst_type_variant.kind):
             raise ValueError(
                 f"Type mismatch for `{''.join(field_path)}`: "
-                f"passed in {src_type_kind}, declared {dst_annotation} ({dst_type_variant.kind})"
+                f"passed in {src_type_kind}, declared {dst_type_info.core_type} ({dst_type_variant.kind})"
             )
 
         if dst_type_variant.kind in ("Float32", "Float64", "Int64"):
@@ -267,7 +288,31 @@ def make_engine_value_decoder(
     return lambda value: value
 
 
-def _make_engine_struct_value_decoder(
+def _get_auto_default_for_type(
+    type_info: AnalyzedTypeInfo,
+) -> tuple[Any, bool]:
+    """
+    Get an auto-default value for a type annotation if it's safe to do so.
+
+    Returns:
+        A tuple of (default_value, is_supported) where:
+        - default_value: The default value if auto-defaulting is supported
+        - is_supported: True if auto-defaulting is supported for this type
+    """
+    # Case 1: Nullable types (Optional[T] or T | None)
+    if type_info.nullable:
+        return None, True
+
+    # Case 2: Table types (KTable or LTable) - check if it's a list or dict type
+    if isinstance(type_info.variant, AnalyzedListType):
+        return [], True
+    elif isinstance(type_info.variant, AnalyzedDictType):
+        return {}, True
+
+    return None, False
+
+
+def make_engine_struct_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
     dst_type_info: AnalyzedTypeInfo,
@@ -321,32 +366,39 @@ def _make_engine_struct_value_decoder(
     else:
         raise ValueError(f"Unsupported struct type: {dst_struct_type}")
 
-    def make_closure_for_value(
+    def make_closure_for_field(
         name: str, param: inspect.Parameter
     ) -> Callable[[list[Any]], Any]:
         src_idx = src_name_to_idx.get(name)
-        if src_idx is not None:
-            field_path.append(f".{name}")
-            field_decoder = make_engine_value_decoder(
-                field_path, src_fields[src_idx]["type"], param.annotation
-            )
-            field_path.pop()
-            return (
-                lambda values: field_decoder(values[src_idx])
-                if len(values) > src_idx
-                else param.default
-            )
+        type_info = analyze_type_info(param.annotation)
+
+        with ChildFieldPath(field_path, f".{name}"):
+            if src_idx is not None:
+                field_decoder = make_engine_value_decoder(
+                    field_path, src_fields[src_idx]["type"], type_info
+                )
+                return lambda values: field_decoder(values[src_idx])
+
+            default_value = param.default
+            if default_value is not inspect.Parameter.empty:
+                return lambda _: default_value
+
+            auto_default, is_supported = _get_auto_default_for_type(type_info)
+            if is_supported:
+                warnings.warn(
+                    f"Field '{name}' (type {param.annotation}) without default value is missing in input: "
+                    f"{''.join(field_path)}. Auto-assigning default value: {auto_default}",
+                    UserWarning,
+                    stacklevel=4,
+                )
+                return lambda _: auto_default
 
-        default_value = param.default
-        if default_value is inspect.Parameter.empty:
             raise ValueError(
-                f"Field without default value is missing in input: {''.join(field_path)}"
+                f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}"
             )
 
-        return lambda _: default_value
-
     field_value_decoder = [
-        make_closure_for_value(name, param) for (name, param) in parameters.items()
+        make_closure_for_field(name, param) for (name, param) in parameters.items()
     ]
 
     return lambda values: dst_struct_type(
@@ -363,13 +415,12 @@ def _make_engine_struct_to_dict_decoder(
     field_decoders = []
     for i, field_schema in enumerate(src_fields):
         field_name = field_schema["name"]
-        field_path.append(f".{field_name}")
-        field_decoder = make_engine_value_decoder(
-            field_path,
-            field_schema["type"],
-            Any,  # Use Any for recursive decoding
-        )
-        field_path.pop()
+        with ChildFieldPath(field_path, f".{field_name}"):
+            field_decoder = make_engine_value_decoder(
+                field_path,
+                field_schema["type"],
+                analyze_type_info(Any),  # Use Any for recursive decoding
+            )
         field_decoders.append((field_name, field_decoder))
 
     def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None:
@@ -426,9 +477,10 @@ def _make_engine_ktable_to_dict_dict_decoder(
     value_fields_schema = src_fields[1:]
 
     # Create decoders
-    field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-    key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any)
-    field_path.pop()
+    with ChildFieldPath(field_path, f".{key_field_schema.get('name', KEY_FIELD_NAME)}"):
+        key_decoder = make_engine_value_decoder(
+            field_path, key_field_schema["type"], analyze_type_info(Any)
+        )
 
     value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema)
 

cocoindex/flow.py

@@ -16,6 +16,7 @@ from .validation import (
     validate_full_flow_name,
     validate_target_name,
 )
+from .typing import analyze_type_info
 
 from dataclasses import dataclass
 from enum import Enum
@@ -1053,7 +1054,7 @@ class TransformFlow(Generic[T]):
             sig.return_annotation
         )
         result_decoder = make_engine_value_decoder(
-            [], engine_return_type["type"], python_return_type
+            [], engine_return_type["type"], analyze_type_info(python_return_type)
         )
 
         return TransformFlowInfo(engine_flow, result_decoder)

cocoindex/op.py

@@ -6,11 +6,31 @@ import asyncio
 import dataclasses
 import inspect
 from enum import Enum
-from typing import Any, Awaitable, Callable, Protocol, dataclass_transform, Annotated
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Protocol,
+    dataclass_transform,
+    Annotated,
+    get_args,
+)
 
 from . import _engine  # type: ignore
-from .convert import encode_engine_value, make_engine_value_decoder
-from .typing import TypeAttr, encode_enriched_type, resolve_forward_ref
+from .convert import (
+    encode_engine_value,
+    make_engine_value_decoder,
+    make_engine_struct_decoder,
+)
+from .typing import (
+    TypeAttr,
+    encode_enriched_type,
+    resolve_forward_ref,
+    analyze_type_info,
+    AnalyzedAnyType,
+    AnalyzedBasicType,
+    AnalyzedDictType,
+)
 
 
 class OpCategory(Enum):
@@ -65,6 +85,22 @@ class Executor(Protocol):
     op_category: OpCategory
 
 
+def _load_spec_from_engine(spec_cls: type, spec: dict[str, Any]) -> Any:
+    """
+    Load a spec from the engine.
+    """
+    return spec_cls(**spec)
+
+
+def _get_required_method(cls: type, name: str) -> Callable[..., Any]:
+    method = getattr(cls, name, None)
+    if method is None:
+        raise ValueError(f"Method {name}() is required for {cls.__name__}")
+    if not inspect.isfunction(method):
+        raise ValueError(f"Method {cls.__name__}.{name}() is not a function")
+    return method
+
+
 class _FunctionExecutorFactory:
     _spec_cls: type
     _executor_cls: type
@@ -76,7 +112,7 @@ class _FunctionExecutorFactory:
     def __call__(
         self, spec: dict[str, Any], *args: Any, **kwargs: Any
     ) -> tuple[dict[str, Any], Executor]:
-        spec = self._spec_cls(**spec)
+        spec = _load_spec_from_engine(self._spec_cls, spec)
         executor = self._executor_cls(spec)
         result_type = executor.analyze(*args, **kwargs)
         return (encode_enriched_type(result_type), executor)
@@ -185,7 +221,9 @@ def _register_op_factory(
                     )
                 self._args_decoders.append(
                     make_engine_value_decoder(
-                        [arg_name], arg.value_type["type"], arg_param.annotation
+                        [arg_name],
+                        arg.value_type["type"],
+                        analyze_type_info(arg_param.annotation),
                     )
                 )
                 process_attribute(arg_name, arg)
@@ -217,7 +255,9 @@ def _register_op_factory(
                     )
                 arg_param = expected_arg[1]
                 self._kwargs_decoders[kwarg_name] = make_engine_value_decoder(
-                    [kwarg_name], kwarg.value_type["type"], arg_param.annotation
+                    [kwarg_name],
+                    kwarg.value_type["type"],
+                    analyze_type_info(arg_param.annotation),
                 )
                 process_attribute(kwarg_name, kwarg)
 
@@ -359,3 +399,220 @@ def function(**args: Any) -> Callable[[Callable[..., Any]], FunctionSpec]:
         return _Spec()
 
     return _inner
+
+
+########################################################
+# Custom target connector
+########################################################
+
+
+@dataclasses.dataclass
+class _TargetConnectorContext:
+    target_name: str
+    spec: Any
+    prepared_spec: Any
+    key_decoder: Callable[[Any], Any]
+    value_decoder: Callable[[Any], Any]
+
+
+class _TargetConnector:
+    """
+    The connector class passed to the engine.
+    """
+
+    _spec_cls: type
+    _connector_cls: type
+
+    _get_persistent_key_fn: Callable[[_TargetConnectorContext, str], Any]
+    _apply_setup_change_async_fn: Callable[
+        [Any, dict[str, Any] | None, dict[str, Any] | None], Awaitable[None]
+    ]
+    _mutate_async_fn: Callable[..., Awaitable[None]]
+    _mutatation_type: AnalyzedDictType | None
+
+    def __init__(self, spec_cls: type, connector_cls: type):
+        self._spec_cls = spec_cls
+        self._connector_cls = connector_cls
+
+        self._get_persistent_key_fn = _get_required_method(
+            connector_cls, "get_persistent_key"
+        )
+        self._apply_setup_change_async_fn = _to_async_call(
+            _get_required_method(connector_cls, "apply_setup_change")
+        )
+
+        mutate_fn = _get_required_method(connector_cls, "mutate")
+        self._mutate_async_fn = _to_async_call(mutate_fn)
+
+        # Store the type annotation for later use
+        self._mutatation_type = self._analyze_mutate_mutation_type(
+            connector_cls, mutate_fn
+        )
+
+    @staticmethod
+    def _analyze_mutate_mutation_type(
+        connector_cls: type, mutate_fn: Callable[..., Any]
+    ) -> AnalyzedDictType | None:
+        # Validate mutate_fn signature and extract type annotation
+        mutate_sig = inspect.signature(mutate_fn)
+        params = list(mutate_sig.parameters.values())
+
+        if len(params) != 1:
+            raise ValueError(
+                f"Method {connector_cls.__name__}.mutate(*args) must have exactly one parameter, "
+                f"got {len(params)}"
+            )
+
+        param = params[0]
+        if param.kind != inspect.Parameter.VAR_POSITIONAL:
+            raise ValueError(
+                f"Method {connector_cls.__name__}.mutate(*args) parameter must be *args format, "
+                f"got {param.kind.name}"
+            )
+
+        # Extract type annotation
+        analyzed_args_type = analyze_type_info(param.annotation)
+        if isinstance(analyzed_args_type.variant, AnalyzedAnyType):
+            return None
+
+        if analyzed_args_type.base_type is tuple:
+            args = get_args(analyzed_args_type.core_type)
+            if not args:
+                return None
+            if len(args) == 2:
+                mutation_type = analyze_type_info(args[1])
+                if isinstance(mutation_type.variant, AnalyzedAnyType):
+                    return None
+                if isinstance(mutation_type.variant, AnalyzedDictType):
+                    return mutation_type.variant
+
+        raise ValueError(
+            f"Method {connector_cls.__name__}.mutate(*args) parameter must be a tuple with "
+            f"2 elements (tuple[SpecType, dict[str, ValueStruct]], spec and mutation in dict), "
+            "got {args_type}"
+        )
+
+    def create_export_context(
+        self,
+        name: str,
+        spec: dict[str, Any],
+        key_fields_schema: list[Any],
+        value_fields_schema: list[Any],
+    ) -> _TargetConnectorContext:
+        key_annotation, value_annotation = (
+            (
+                self._mutatation_type.key_type,
+                self._mutatation_type.value_type,
+            )
+            if self._mutatation_type is not None
+            else (None, None)
+        )
+
+        key_type_info = analyze_type_info(key_annotation)
+        if (
+            len(key_fields_schema) == 1
+            and key_fields_schema[0]["type"]["kind"] != "Struct"
+            and isinstance(key_type_info.variant, (AnalyzedAnyType, AnalyzedBasicType))
+        ):
+            # Special case for ease of use: single key column can be mapped to a basic type without the wrapper struct.
+            key_decoder = make_engine_value_decoder(
+                ["(key)"],
+                key_fields_schema[0]["type"],
+                key_type_info,
+            )
+        else:
+            key_decoder = make_engine_struct_decoder(
+                ["(key)"], key_fields_schema, key_type_info
+            )
+
+        value_decoder = make_engine_struct_decoder(
+            ["(value)"], value_fields_schema, analyze_type_info(value_annotation)
+        )
+
+        loaded_spec = _load_spec_from_engine(self._spec_cls, spec)
+        prepare_method = getattr(self._connector_cls, "prepare", None)
+        if prepare_method is None:
+            prepared_spec = loaded_spec
+        else:
+            prepared_spec = prepare_method(loaded_spec)
+
+        return _TargetConnectorContext(
+            target_name=name,
+            spec=loaded_spec,
+            prepared_spec=prepared_spec,
+            key_decoder=key_decoder,
+            value_decoder=value_decoder,
+        )
+
+    def get_persistent_key(self, export_context: _TargetConnectorContext) -> Any:
+        return self._get_persistent_key_fn(
+            export_context.spec, export_context.target_name
+        )
+
+    def describe_resource(self, key: Any) -> str:
+        describe_fn = getattr(self._connector_cls, "describe", None)
+        if describe_fn is None:
+            return str(key)
+        return str(describe_fn(key))
+
+    async def apply_setup_changes_async(
+        self,
+        changes: list[tuple[Any, list[dict[str, Any] | None], dict[str, Any] | None]],
+    ) -> None:
+        for key, previous, current in changes:
+            prev_specs = [
+                _load_spec_from_engine(self._spec_cls, spec)
+                if spec is not None
+                else None
+                for spec in previous
+            ]
+            curr_spec = (
+                _load_spec_from_engine(self._spec_cls, current)
+                if current is not None
+                else None
+            )
+            for prev_spec in prev_specs:
+                await self._apply_setup_change_async_fn(key, prev_spec, curr_spec)
+
+    @staticmethod
+    def _decode_mutation(
+        context: _TargetConnectorContext, mutation: list[tuple[Any, Any | None]]
+    ) -> tuple[Any, dict[Any, Any | None]]:
+        return (
+            context.prepared_spec,
+            {
+                context.key_decoder(key): (
+                    context.value_decoder(value) if value is not None else None
+                )
+                for key, value in mutation
+            },
+        )
+
+    async def mutate_async(
+        self,
+        mutations: list[tuple[_TargetConnectorContext, list[tuple[Any, Any | None]]]],
+    ) -> None:
+        await self._mutate_async_fn(
+            *(
+                self._decode_mutation(context, mutation)
+                for context, mutation in mutations
+            )
+        )
+
+
+def target_connector(spec_cls: type) -> Callable[[type], type]:
+    """
+    Decorate a class to provide a target connector for an op.
+    """
+
+    # Validate the spec_cls is a TargetSpec.
+    if not issubclass(spec_cls, TargetSpec):
+        raise ValueError(f"Expect a TargetSpec, got {spec_cls}")
+
+    # Register the target connector.
+    def _inner(connector_cls: type) -> type:
+        connector = _TargetConnector(spec_cls, connector_cls)
+        _engine.register_target_connector(spec_cls.__name__, connector)
+        return connector_cls
+
+    return _inner

cocoindex/tests/test_convert.py

@@ -1,6 +1,6 @@
 import datetime
 import uuid
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from typing import Annotated, Any, Callable, Literal, NamedTuple
 
 import numpy as np
@@ -19,6 +19,7 @@ from cocoindex.typing import (
     TypeKind,
     Vector,
     encode_enriched_type,
+    analyze_type_info,
 )
 
 
@@ -75,7 +76,9 @@ def build_engine_value_decoder(
     If python_type is not specified, uses engine_type_in_py as the target.
     """
     engine_type = encode_enriched_type(engine_type_in_py)["type"]
-    return make_engine_value_decoder([], engine_type, python_type or engine_type_in_py)
+    return make_engine_value_decoder(
+        [], engine_type, analyze_type_info(python_type or engine_type_in_py)
+    )
 
 
 def validate_full_roundtrip_to(
@@ -103,7 +106,9 @@ def validate_full_roundtrip_to(
     )
 
     for other_value, other_type in decoded_values:
-        decoder = make_engine_value_decoder([], encoded_output_type, other_type)
+        decoder = make_engine_value_decoder(
+            [], encoded_output_type, analyze_type_info(other_type)
+        )
         other_decoded_value = decoder(value_from_engine)
         assert eq(other_decoded_value, other_value), (
             f"Expected {other_value} but got {other_decoded_value} for {other_type}"
@@ -364,7 +369,9 @@ def test_decode_scalar_numpy_values() -> None:
         ({"kind": "Float64"}, np.float64, 2.718, np.float64(2.718)),
     ]
     for src_type, dst_type, input_value, expected in test_cases:
-        decoder = make_engine_value_decoder(["field"], src_type, dst_type)
+        decoder = make_engine_value_decoder(
+            ["field"], src_type, analyze_type_info(dst_type)
+        )
         result = decoder(input_value)
         assert isinstance(result, dst_type)
         assert result == expected
@@ -378,7 +385,9 @@ def test_non_ndarray_vector_decoding() -> None:
         "dimension": None,
     }
     dst_type_float = list[np.float64]
-    decoder = make_engine_value_decoder(["field"], src_type, dst_type_float)
+    decoder = make_engine_value_decoder(
+        ["field"], src_type, analyze_type_info(dst_type_float)
+    )
     input_numbers = [1.0, 2.0, 3.0]
     result = decoder(input_numbers)
     assert isinstance(result, list)
@@ -388,7 +397,9 @@ def test_non_ndarray_vector_decoding() -> None:
     # Test list[Uuid]
     src_type = {"kind": "Vector", "element_type": {"kind": "Uuid"}, "dimension": None}
     dst_type_uuid = list[uuid.UUID]
-    decoder = make_engine_value_decoder(["field"], src_type, dst_type_uuid)
+    decoder = make_engine_value_decoder(
+        ["field"], src_type, analyze_type_info(dst_type_uuid)
+    )
     uuid1 = uuid.uuid4()
     uuid2 = uuid.uuid4()
     input_uuids = [uuid1, uuid2]
@@ -398,124 +409,15 @@ def test_non_ndarray_vector_decoding() -> None:
     assert result == [uuid1, uuid2]
 
 
-@pytest.mark.parametrize(
-    "data_type, engine_val, expected",
-    [
-        # All fields match (dataclass)
-        (
-            Order,
-            ["O123", "mixed nuts", 25.0, "default_extra"],
-            Order("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        # All fields match (NamedTuple)
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 25.0, "default_extra"],
-            OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        # Extra field in engine value (should ignore extra)
-        (
-            Order,
-            ["O123", "mixed nuts", 25.0, "default_extra", "unexpected"],
-            Order("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 25.0, "default_extra", "unexpected"],
-            OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        # Fewer fields in engine value (should fill with default)
-        (
-            Order,
-            ["O123", "mixed nuts", 0.0, "default_extra"],
-            Order("O123", "mixed nuts", 0.0, "default_extra"),
-        ),
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 0.0, "default_extra"],
-            OrderNamedTuple("O123", "mixed nuts", 0.0, "default_extra"),
-        ),
-        # More fields in engine value (should ignore extra)
-        (
-            Order,
-            ["O123", "mixed nuts", 25.0, "unexpected"],
-            Order("O123", "mixed nuts", 25.0, "unexpected"),
-        ),
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 25.0, "unexpected"],
-            OrderNamedTuple("O123", "mixed nuts", 25.0, "unexpected"),
-        ),
-        # Truly extra field (should ignore the fifth field)
-        (
-            Order,
-            ["O123", "mixed nuts", 25.0, "default_extra", "ignored"],
-            Order("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 25.0, "default_extra", "ignored"],
-            OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        # Missing optional field in engine value (tags=None)
-        (
-            Customer,
-            ["Alice", ["O1", "item1", 10.0, "default_extra"], None],
-            Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), None),
-        ),
-        (
-            CustomerNamedTuple,
-            ["Alice", ["O1", "item1", 10.0, "default_extra"], None],
-            CustomerNamedTuple(
-                "Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), None
-            ),
-        ),
-        # Extra field in engine value for Customer (should ignore)
-        (
-            Customer,
-            ["Alice", ["O1", "item1", 10.0, "default_extra"], [["vip"]], "extra"],
-            Customer(
-                "Alice", Order("O1", "item1", 10.0, "default_extra"), [Tag("vip")]
-            ),
-        ),
-        (
-            CustomerNamedTuple,
-            ["Alice", ["O1", "item1", 10.0, "default_extra"], [["vip"]], "extra"],
-            CustomerNamedTuple(
-                "Alice",
-                OrderNamedTuple("O1", "item1", 10.0, "default_extra"),
-                [Tag("vip")],
-            ),
-        ),
-        # Missing optional field with default
-        (
-            Order,
-            ["O123", "mixed nuts", 25.0],
-            Order("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        (
-            OrderNamedTuple,
-            ["O123", "mixed nuts", 25.0],
-            OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra"),
-        ),
-        # Partial optional fields
-        (
-            Customer,
-            ["Alice", ["O1", "item1", 10.0]],
-            Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), None),
-        ),
-        (
-            CustomerNamedTuple,
-            ["Alice", ["O1", "item1", 10.0]],
-            CustomerNamedTuple(
-                "Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), None
-            ),
-        ),
-    ],
-)
-def test_struct_decoder_cases(data_type: Any, engine_val: Any, expected: Any) -> None:
-    decoder = build_engine_value_decoder(data_type)
-    assert decoder(engine_val) == expected
+def test_roundtrip_struct() -> None:
+    validate_full_roundtrip(
+        Order("O123", "mixed nuts", 25.0, "default_extra"),
+        Order,
+    )
+    validate_full_roundtrip(
+        OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra"),
+        OrderNamedTuple,
+    )
 
 
 def test_make_engine_value_decoder_list_of_struct() -> None:
@@ -974,7 +876,9 @@ def test_decode_nullable_ndarray_none_or_value_input() -> None:
         "dimension": None,
     }
     dst_annotation = NDArrayFloat32Type | None
-    decoder = make_engine_value_decoder([], src_type_dict, dst_annotation)
+    decoder = make_engine_value_decoder(
+        [], src_type_dict, analyze_type_info(dst_annotation)
+    )
 
     none_engine_value = None
     decoded_array = decoder(none_engine_value)
@@ -997,7 +901,9 @@ def test_decode_vector_string() -> None:
         "element_type": {"kind": "Str"},
         "dimension": None,
     }
-    decoder = make_engine_value_decoder([], src_type_dict, Vector[str])
+    decoder = make_engine_value_decoder(
+        [], src_type_dict, analyze_type_info(Vector[str])
+    )
     assert decoder(["hello", "world"]) == ["hello", "world"]
 
 
@@ -1008,7 +914,9 @@ def test_decode_error_non_nullable_or_non_list_vector() -> None:
         "element_type": {"kind": "Float32"},
         "dimension": None,
     }
-    decoder = make_engine_value_decoder([], src_type_dict, NDArrayFloat32Type)
+    decoder = make_engine_value_decoder(
+        [], src_type_dict, analyze_type_info(NDArrayFloat32Type)
+    )
     with pytest.raises(ValueError, match="Received null for non-nullable vector"):
         decoder(None)
     with pytest.raises(TypeError, match="Expected NDArray or list for vector"):
@@ -1489,3 +1397,41 @@ def test_roundtrip_ktable_with_list_fields() -> None:
 
     # Test Any annotation
     validate_full_roundtrip(teams, dict[str, Team], (expected_dict_dict, Any))
+
+
+def test_auto_default_for_supported_and_unsupported_types() -> None:
+    @dataclass
+    class Base:
+        a: int
+
+    @dataclass
+    class NullableField:
+        a: int
+        b: int | None
+
+    @dataclass
+    class LTableField:
+        a: int
+        b: list[Base]
+
+    @dataclass
+    class KTableField:
+        a: int
+        b: dict[str, Base]
+
+    @dataclass
+    class UnsupportedField:
+        a: int
+        b: int
+
+    validate_full_roundtrip(NullableField(1, None), NullableField)
+
+    validate_full_roundtrip(LTableField(1, []), LTableField)
+
+    # validate_full_roundtrip(KTableField(1, {}), KTableField)
+
+    with pytest.raises(
+        ValueError,
+        match=r"Field 'b' \(type <class 'int'>\) without default value is missing in input: ",
+    ):
+        build_engine_value_decoder(Base, UnsupportedField)

cocoindex/tests/test_typing.py

@@ -13,6 +13,7 @@ from cocoindex.typing import (
     AnalyzedDictType,
     AnalyzedListType,
     AnalyzedStructType,
+    AnalyzedUnknownType,
     AnalyzedTypeInfo,
     TypeAttr,
     TypeKind,
@@ -422,15 +423,7 @@ def test_annotated_list_with_type_kind() -> None:
     assert result.variant.kind == "Struct"
 
 
-def test_unsupported_type() -> None:
-    with pytest.raises(
-        ValueError,
-        match="Unsupported as a specific type annotation for CocoIndex data type.*: <class 'set'>",
-    ):
-        analyze_type_info(set)
-
-    with pytest.raises(
-        ValueError,
-        match="Unsupported as a specific type annotation for CocoIndex data type.*: <class 'numpy.complex64'>",
-    ):
-        Vector[np.complex64]
+def test_unknown_type() -> None:
+    typ = set
+    result = analyze_type_info(typ)
+    assert isinstance(result.variant, AnalyzedUnknownType)

cocoindex/typing.py

@@ -313,11 +313,11 @@ def analyze_type_info(t: Any) -> AnalyzedTypeInfo:
             kind = "OffsetDateTime"
         elif t is datetime.timedelta:
             kind = "TimeDelta"
+
+        if kind is None:
+            variant = AnalyzedUnknownType()
         else:
-            raise ValueError(
-                f"Unsupported as a specific type annotation for CocoIndex data type (https://cocoindex.io/docs/core/data_types): {t}"
-            )
-        variant = AnalyzedBasicType(kind=kind)
+            variant = AnalyzedBasicType(kind=kind)
 
     return AnalyzedTypeInfo(
         core_type=core_type,

{cocoindex-0.1.73.dist-info → cocoindex-0.1.74.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.73
+Version: 0.1.74
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -11,7 +11,7 @@ Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: mypy ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
-Requires-Dist: cocoindex[embeddings] ; extra == 'all'
+Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'all'
 Provides-Extra: dev
 Provides-Extra: embeddings
 Provides-Extra: all

{cocoindex-0.1.73.dist-info → cocoindex-0.1.74.dist-info}/RECORD

@@ -1,18 +1,18 @@
-cocoindex-0.1.73.dist-info/METADATA,sha256=7TqVP8FrHbi9wz-XrnSYX2C158sVJV-SveG7VmXZN6k,11304
-cocoindex-0.1.73.dist-info/WHEEL,sha256=ZlnHGE4Mo7ps1v16zvcAfrDnL3ds3Klm21sZZ52w3vY,104
-cocoindex-0.1.73.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.1.73.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex-0.1.74.dist-info/METADATA,sha256=kkA1L7X7ypbNcSLBtJ92Zp1lYEgEJEfXpR1d1c8V_Lw,11311
+cocoindex-0.1.74.dist-info/WHEEL,sha256=ZlnHGE4Mo7ps1v16zvcAfrDnL3ds3Klm21sZZ52w3vY,104
+cocoindex-0.1.74.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.1.74.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 cocoindex/__init__.py,sha256=kfTgbh2haepo7kIbzJqfxU6Kx7wPol5_t1SYF2x6cBM,2114
-cocoindex/_engine.cpython-313-darwin.so,sha256=Hdp4FUzsvvEtPu-9po5mc7rnwOO1oliI9z0pVHXrH5o,65071008
+cocoindex/_engine.cpython-313-darwin.so,sha256=Zcg8ZL5BZwc25OIlgcid6cAweJhz-GqC8f6XP_wdF6Y,65114640
 cocoindex/auth_registry.py,sha256=PE1-kVkcyC1G2C_V7b1kvYzeq73OFQehWKQP7ln7fJ8,1478
 cocoindex/cli.py,sha256=-gp639JSyQN6YjnhGqCakIzYoSSqXxQMbxbkcYGP0QY,22359
-cocoindex/convert.py,sha256=ZENKZ47Vmah2_EEvE03WqfDF9wCs9h0CDEjq08GK3R0,17365
-cocoindex/flow.py,sha256=HN24rsihO3BkSYGnTtxgovgka2IobxhFuLmDlqw3fAk,36127
+cocoindex/convert.py,sha256=jO95WD8X7AYS5COfRyPiq4t7ewVC1TdRbYpTdHrkCvY,19564
+cocoindex/flow.py,sha256=C2VwexNxes_7b70jhlp5yAPeBQPtdbKabOp1AarsacA,36184
 cocoindex/functions.py,sha256=LLu_ausirvqnsx_k3euZpv8sLCpBZ4DF77h2HOzbinE,3109
 cocoindex/index.py,sha256=j93B9jEvvLXHtpzKWL88SY6wCGEoPgpsQhEGHlyYGFg,540
 cocoindex/lib.py,sha256=f--9dAYd84CZosbDZqNW0oGbBLsY3dXiUTR1VrfQ_QY,817
 cocoindex/llm.py,sha256=WxmWUbNcf9HOCM5xkbDeFs9lF67M3mr810B7deDDc-8,673
-cocoindex/op.py,sha256=Afi5CfgU3wPQoPPKFb2WUYCVLmCPhBuK-2NT1AzC2zU,13161
+cocoindex/op.py,sha256=DNAlZGpGFZWAfRFuxXKmT72kSORdGbirDYsSlD7sDqk,21474
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=povilB3HH3y1JF-yxKwU-pD8n2WnAqyQxIgvXXHNc60,1080
 cocoindex/setting.py,sha256=TwhQ6pEeZmvc8ZXlnT9d8Wn8Vz_u7Z5LJUkGsKmKSno,4859
@@ -20,11 +20,11 @@ cocoindex/setup.py,sha256=7uIHKN4FOCuoidPXcKyGTrkqpkl9luL49-6UcnMxYzw,3068
 cocoindex/sources.py,sha256=69COA4qbZDipzGYfXv-WJSmicFkA509xIShRGDh6A0A,2083
 cocoindex/targets.py,sha256=Nfh_tpFd1goTnS_cxBjIs4j9zl3Z4Z1JomAQ1dl3Sic,2796
 cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cocoindex/tests/test_convert.py,sha256=Ofx0grI-tIWCkefwS8XMTQlQvDDV0YI1fjx5ZhUwRZs,49738
+cocoindex/tests/test_convert.py,sha256=guQ6-AJh4z1NoR60wlm0lJ2R-rOstmFDihDL2oYGouI,46875
 cocoindex/tests/test_optional_database.py,sha256=snAmkNa6wtOSaxoZE1HgjvL5v_ylitt3Jt_9df4Cgdc,8506
-cocoindex/tests/test_typing.py,sha256=MeirjvSVlMqIJsvLF0B11EYpczi6o7WsJMS2hdHry5c,12636
+cocoindex/tests/test_typing.py,sha256=9OF3lO2uSpZBefkEJx7WRbnkXjwQtvlQIeeARYQID68,12391
 cocoindex/tests/test_validation.py,sha256=X6AQzVs-hVKIXcrHMEMQnhfUE8at7iXQnPq8nHNhZ2Q,4543
-cocoindex/typing.py,sha256=9g-pV1O-CUnUQwzE22nPHVUTY6Zrr-ChzAEq4TVGAig,13487
+cocoindex/typing.py,sha256=RK-UYiNkxL9DPjhitFeJVB0b3eV2DsGjOg8V-jrnUYU,13383
 cocoindex/utils.py,sha256=hUhX-XV6XGCtJSEIpBOuDv6VvqImwPlgBxztBTw7u0U,598
 cocoindex/validation.py,sha256=PZnJoby4sLbsmPv9fOjOQXuefjfZ7gmtsiTGU8SH-tc,3090
-cocoindex-0.1.73.dist-info/RECORD,,
+cocoindex-0.1.74.dist-info/RECORD,,