cocoindex 0.1.71__cp313-cp313-macosx_11_0_arm64.whl → 0.1.73__cp313-cp313-macosx_11_0_arm64.whl

cocoindex/convert.py

@@ -13,12 +13,19 @@ import numpy as np
 from .typing import (
     KEY_FIELD_NAME,
     TABLE_TYPES,
-    DtypeRegistry,
     analyze_type_info,
     encode_enriched_type,
-    extract_ndarray_scalar_dtype,
     is_namedtuple_type,
     is_struct_type,
+    AnalyzedTypeInfo,
+    AnalyzedAnyType,
+    AnalyzedDictType,
+    AnalyzedListType,
+    AnalyzedBasicType,
+    AnalyzedUnionType,
+    AnalyzedUnknownType,
+    AnalyzedStructType,
+    is_numpy_number_type,
 )
 
 
@@ -79,46 +86,88 @@ def make_engine_value_decoder(
     Returns:
         A decoder from an engine value to a Python value.
     """
+
     src_type_kind = src_type["kind"]
 
-    dst_is_any = (
-        dst_annotation is None
-        or dst_annotation is inspect.Parameter.empty
-        or dst_annotation is Any
-    )
-    if dst_is_any:
-        if src_type_kind == "Union":
-            return lambda value: value[1]
-        if src_type_kind == "Struct":
-            return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
-        if src_type_kind in TABLE_TYPES:
-            if src_type_kind == "LTable":
+    dst_type_info = analyze_type_info(dst_annotation)
+    dst_type_variant = dst_type_info.variant
+
+    if isinstance(dst_type_variant, AnalyzedUnknownType):
+        raise ValueError(
+            f"Type mismatch for `{''.join(field_path)}`: "
+            f"declared `{dst_type_info.core_type}`, an unsupported type"
+        )
+
+    if src_type_kind == "Struct":
+        return _make_engine_struct_value_decoder(
+            field_path,
+            src_type["fields"],
+            dst_type_info,
+        )
+
+    if src_type_kind in TABLE_TYPES:
+        field_path.append("[*]")
+        engine_fields_schema = src_type["row"]["fields"]
+
+        if src_type_kind == "LTable":
+            if isinstance(dst_type_variant, AnalyzedAnyType):
                 return _make_engine_ltable_to_list_dict_decoder(
-                    field_path, src_type["row"]["fields"]
+                    field_path, engine_fields_schema
+                )
+            if not isinstance(dst_type_variant, AnalyzedListType):
+                raise ValueError(
+                    f"Type mismatch for `{''.join(field_path)}`: "
+                    f"declared `{dst_type_info.core_type}`, a list type expected"
                 )
-            elif src_type_kind == "KTable":
+            row_decoder = _make_engine_struct_value_decoder(
+                field_path,
+                engine_fields_schema,
+                analyze_type_info(dst_type_variant.elem_type),
+            )
+
+            def decode(value: Any) -> Any | None:
+                if value is None:
+                    return None
+                return [row_decoder(v) for v in value]
+
+        elif src_type_kind == "KTable":
+            if isinstance(dst_type_variant, AnalyzedAnyType):
                 return _make_engine_ktable_to_dict_dict_decoder(
-                    field_path, src_type["row"]["fields"]
+                    field_path, engine_fields_schema
+                )
+            if not isinstance(dst_type_variant, AnalyzedDictType):
+                raise ValueError(
+                    f"Type mismatch for `{''.join(field_path)}`: "
+                    f"declared `{dst_type_info.core_type}`, a dict type expected"
                 )
-        return lambda value: value
 
-    # Handle struct -> dict binding for explicit dict annotations
-    is_dict_annotation = False
-    if dst_annotation is dict:
-        is_dict_annotation = True
-    elif getattr(dst_annotation, "__origin__", None) is dict:
-        args = getattr(dst_annotation, "__args__", ())
-        if args == (str, Any):
-            is_dict_annotation = True
-    if is_dict_annotation and src_type_kind == "Struct":
-        return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
+            key_field_schema = engine_fields_schema[0]
+            field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+            key_decoder = make_engine_value_decoder(
+                field_path, key_field_schema["type"], dst_type_variant.key_type
+            )
+            field_path.pop()
+            value_decoder = _make_engine_struct_value_decoder(
+                field_path,
+                engine_fields_schema[1:],
+                analyze_type_info(dst_type_variant.value_type),
+            )
 
-    dst_type_info = analyze_type_info(dst_annotation)
+            def decode(value: Any) -> Any | None:
+                if value is None:
+                    return None
+                return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
+
+        field_path.pop()
+        return decode
 
     if src_type_kind == "Union":
+        if isinstance(dst_type_variant, AnalyzedAnyType):
+            return lambda value: value[1]
+
         dst_type_variants = (
-            dst_type_info.union_variant_types
-            if dst_type_info.union_variant_types is not None
+            dst_type_variant.variant_types
+            if isinstance(dst_type_variant, AnalyzedUnionType)
             else [dst_annotation]
         )
         src_type_variants = src_type["types"]
@@ -142,43 +191,33 @@ def make_engine_value_decoder(
             decoders.append(decoder)
         return lambda value: decoders[value[0]](value[1])
 
-    if not _is_type_kind_convertible_to(src_type_kind, dst_type_info.kind):
-        raise ValueError(
-            f"Type mismatch for `{''.join(field_path)}`: "
-            f"passed in {src_type_kind}, declared {dst_annotation} ({dst_type_info.kind})"
-        )
-
-    if dst_type_info.kind in ("Float32", "Float64", "Int64"):
-        dst_core_type = dst_type_info.core_type
-
-        def decode_scalar(value: Any) -> Any | None:
-            if value is None:
-                if dst_type_info.nullable:
-                    return None
-                raise ValueError(
-                    f"Received null for non-nullable scalar `{''.join(field_path)}`"
-                )
-            return dst_core_type(value)
-
-        return decode_scalar
+    if isinstance(dst_type_variant, AnalyzedAnyType):
+        return lambda value: value
 
     if src_type_kind == "Vector":
         field_path_str = "".join(field_path)
+        if not isinstance(dst_type_variant, AnalyzedListType):
+            raise ValueError(
+                f"Type mismatch for `{''.join(field_path)}`: "
+                f"declared `{dst_type_info.core_type}`, a list type expected"
+            )
         expected_dim = (
-            dst_type_info.vector_info.dim if dst_type_info.vector_info else None
+            dst_type_variant.vector_info.dim
+            if dst_type_variant and dst_type_variant.vector_info
+            else None
         )
 
-        elem_decoder = None
+        vec_elem_decoder = None
         scalar_dtype = None
-        if dst_type_info.np_number_type is None:  # for Non-NDArray vector
-            elem_decoder = make_engine_value_decoder(
+        if dst_type_variant and dst_type_info.base_type is np.ndarray:
+            if is_numpy_number_type(dst_type_variant.elem_type):
+                scalar_dtype = dst_type_variant.elem_type
+        else:
+            vec_elem_decoder = make_engine_value_decoder(
                 field_path + ["[*]"],
                 src_type["element_type"],
-                dst_type_info.elem_type,
+                dst_type_variant and dst_type_variant.elem_type,
             )
-        else:  # for NDArray vector
-            scalar_dtype = extract_ndarray_scalar_dtype(dst_type_info.np_number_type)
-            _ = DtypeRegistry.validate_dtype_and_get_kind(scalar_dtype)
 
         def decode_vector(value: Any) -> Any | None:
             if value is None:
@@ -197,54 +236,33 @@ def make_engine_value_decoder(
                     f"expected {expected_dim}, got {len(value)}"
                 )
 
-            if elem_decoder is not None:  # for Non-NDArray vector
-                return [elem_decoder(v) for v in value]
+            if vec_elem_decoder is not None:  # for Non-NDArray vector
+                return [vec_elem_decoder(v) for v in value]
             else:  # for NDArray vector
                 return np.array(value, dtype=scalar_dtype)
 
         return decode_vector
 
-    if dst_type_info.struct_type is not None:
-        return _make_engine_struct_value_decoder(
-            field_path, src_type["fields"], dst_type_info.struct_type
-        )
-
-    if src_type_kind in TABLE_TYPES:
-        field_path.append("[*]")
-        elem_type_info = analyze_type_info(dst_type_info.elem_type)
-        if elem_type_info.struct_type is None:
+    if isinstance(dst_type_variant, AnalyzedBasicType):
+        if not _is_type_kind_convertible_to(src_type_kind, dst_type_variant.kind):
             raise ValueError(
                 f"Type mismatch for `{''.join(field_path)}`: "
-                f"declared `{dst_type_info.kind}`, a dataclass or NamedTuple type expected"
-            )
-        engine_fields_schema = src_type["row"]["fields"]
-        if elem_type_info.key_type is not None:
-            key_field_schema = engine_fields_schema[0]
-            field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-            key_decoder = make_engine_value_decoder(
-                field_path, key_field_schema["type"], elem_type_info.key_type
-            )
-            field_path.pop()
-            value_decoder = _make_engine_struct_value_decoder(
-                field_path, engine_fields_schema[1:], elem_type_info.struct_type
+                f"passed in {src_type_kind}, declared {dst_annotation} ({dst_type_variant.kind})"
             )
 
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
-        else:
-            elem_decoder = _make_engine_struct_value_decoder(
-                field_path, engine_fields_schema, elem_type_info.struct_type
-            )
+        if dst_type_variant.kind in ("Float32", "Float64", "Int64"):
+            dst_core_type = dst_type_info.core_type
 
-            def decode(value: Any) -> Any | None:
+            def decode_scalar(value: Any) -> Any | None:
                 if value is None:
-                    return None
-                return [elem_decoder(v) for v in value]
+                    if dst_type_info.nullable:
+                        return None
+                    raise ValueError(
+                        f"Received null for non-nullable scalar `{''.join(field_path)}`"
+                    )
+                return dst_core_type(value)
 
-        field_path.pop()
-        return decode
+            return decode_scalar
 
     return lambda value: value
 
@@ -252,11 +270,36 @@ def make_engine_value_decoder(
 def _make_engine_struct_value_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
-    dst_struct_type: type,
+    dst_type_info: AnalyzedTypeInfo,
 ) -> Callable[[list[Any]], Any]:
     """Make a decoder from an engine field values to a Python value."""
 
+    dst_type_variant = dst_type_info.variant
+
+    use_dict = False
+    if isinstance(dst_type_variant, AnalyzedAnyType):
+        use_dict = True
+    elif isinstance(dst_type_variant, AnalyzedDictType):
+        analyzed_key_type = analyze_type_info(dst_type_variant.key_type)
+        analyzed_value_type = analyze_type_info(dst_type_variant.value_type)
+        use_dict = (
+            isinstance(analyzed_key_type.variant, AnalyzedAnyType)
+            or (
+                isinstance(analyzed_key_type.variant, AnalyzedBasicType)
+                and analyzed_key_type.variant.kind == "Str"
+            )
+        ) and isinstance(analyzed_value_type.variant, AnalyzedAnyType)
+    if use_dict:
+        return _make_engine_struct_to_dict_decoder(field_path, src_fields)
+
+    if not isinstance(dst_type_variant, AnalyzedStructType):
+        raise ValueError(
+            f"Type mismatch for `{''.join(field_path)}`: "
+            f"declared `{dst_type_info.core_type}`, a dataclass, NamedTuple or dict[str, Any] expected"
+        )
+
     src_name_to_idx = {f["name"]: i for i, f in enumerate(src_fields)}
+    dst_struct_type = dst_type_variant.struct_type
 
     parameters: Mapping[str, inspect.Parameter]
     if dataclasses.is_dataclass(dst_struct_type):

cocoindex/flow.py

@@ -10,6 +10,13 @@ import functools
 import inspect
 import re
 
+from .validation import (
+    validate_flow_name,
+    NamingError,
+    validate_full_flow_name,
+    validate_target_name,
+)
+
 from dataclasses import dataclass
 from enum import Enum
 from threading import Lock
@@ -300,6 +307,9 @@ class DataScope:
         )
 
     def __setitem__(self, field_name: str, value: DataSlice[T]) -> None:
+        from .validation import validate_field_name
+
+        validate_field_name(field_name)
         value._state.attach_to_scope(self._engine_data_scope, field_name)
 
     def __enter__(self) -> DataScope:
@@ -367,7 +377,7 @@ class DataCollector:
 
     def export(
         self,
-        name: str,
+        target_name: str,
         target_spec: op.TargetSpec,
         /,
         *,
@@ -381,6 +391,8 @@ class DataCollector:
 
         `vector_index` is for backward compatibility only. Please use `vector_indexes` instead.
         """
+
+        validate_target_name(target_name)
         if not isinstance(target_spec, op.TargetSpec):
             raise ValueError(
                 "export() can only be called on a CocoIndex target storage"
@@ -398,7 +410,7 @@ class DataCollector:
             vector_indexes=vector_indexes,
         )
         self._flow_builder_state.engine_flow_builder.export(
-            name,
+            target_name,
             _spec_kind(target_spec),
             dump_engine_object(target_spec),
             dump_engine_object(index_options),
@@ -660,6 +672,8 @@ class Flow:
     def __init__(
         self, name: str, full_name: str, engine_flow_creator: Callable[[], _engine.Flow]
     ):
+        validate_flow_name(name)
+        validate_full_flow_name(full_name)
         self._name = name
         self._full_name = full_name
         engine_flow = None
@@ -831,11 +845,6 @@ def get_flow_full_name(name: str) -> str:
 
 
 def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
-    """Add a flow definition to the cocoindex library."""
-    if not all(c.isalnum() or c == "_" for c in name):
-        raise ValueError(
-            f"Flow name '{name}' contains invalid characters. Only alphanumeric characters and underscores are allowed."
-        )
     with _flows_lock:
         if name in _flows:
             raise KeyError(f"Flow with name {name} already exists")

cocoindex/setting.py

@@ -6,6 +6,7 @@ import os
 
 from typing import Callable, Self, Any, overload
 from dataclasses import dataclass
+from .validation import validate_app_namespace_name
 
 _app_namespace: str = ""
 
@@ -27,6 +28,8 @@ def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
 
 def set_app_namespace(app_namespace: str) -> None:
     """Set the application namespace."""
+    if app_namespace:
+        validate_app_namespace_name(app_namespace)
     global _app_namespace  # pylint: disable=global-statement
     _app_namespace = app_namespace
 

cocoindex/tests/test_convert.py

@@ -105,7 +105,9 @@ def validate_full_roundtrip_to(
     for other_value, other_type in decoded_values:
         decoder = make_engine_value_decoder([], encoded_output_type, other_type)
         other_decoded_value = decoder(value_from_engine)
-        assert eq(other_decoded_value, other_value)
+        assert eq(other_decoded_value, other_value), (
+            f"Expected {other_value} but got {other_decoded_value} for {other_type}"
+        )
 
 
 def validate_full_roundtrip(
@@ -1096,6 +1098,25 @@ def test_full_roundtrip_vector_numeric_types() -> None:
         validate_full_roundtrip(value_u64, Vector[np.uint64, Literal[3]])
 
 
+def test_full_roundtrip_vector_of_vector() -> None:
+    """Test full roundtrip for vector of vector."""
+    value_f32 = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32)
+    validate_full_roundtrip(
+        value_f32,
+        Vector[Vector[np.float32, Literal[3]], Literal[2]],
+        ([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], list[list[np.float32]]),
+        ([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], list[list[cocoindex.Float32]]),
+        (
+            [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
+            list[Vector[cocoindex.Float32, Literal[3]]],
+        ),
+        (
+            value_f32,
+            np.typing.NDArray[np.typing.NDArray[np.float32]],
+        ),
+    )
+
+
 def test_full_roundtrip_vector_other_types() -> None:
     """Test full roundtrip for Vector with non-numeric basic types."""
     uuid_list = [uuid.uuid4(), uuid.uuid4()]
@@ -1216,7 +1237,7 @@ def test_full_roundtrip_scalar_with_python_types() -> None:
         numpy_float: np.float64
         python_float: float
         string: str
-        annotated_int: Annotated[np.int64, TypeKind("int")]
+        annotated_int: Annotated[np.int64, TypeKind("Int64")]
         annotated_float: Float32
 
     instance = MixedStruct(