cocoindex 0.1.73__cp311-cp311-macosx_10_12_x86_64.whl → 0.1.75__cp311-cp311-macosx_10_12_x86_64.whl

cocoindex/__init__.py

@@ -11,7 +11,8 @@ from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
 from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions, FlowUpdaterStatusUpdates
-from .flow import add_flow_def, remove_flow
+from .flow import open_flow
+from .flow import add_flow_def, remove_flow  # DEPRECATED
 from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
 from .lib import init, start_server, stop
 from .llm import LlmSpec, LlmApiType
@@ -57,8 +58,9 @@ __all__ = [
     "FlowLiveUpdater",
     "FlowLiveUpdaterOptions",
     "FlowUpdaterStatusUpdates",
-    "add_flow_def",
-    "remove_flow",
+    "open_flow",
+    "add_flow_def",  # DEPRECATED
+    "remove_flow",  # DEPRECATED
     "update_all_flows_async",
     "setup_all_flows",
     "drop_all_flows",

cocoindex/convert.py

@@ -2,9 +2,12 @@
 Utilities to convert between Python and engine values.
 """
 
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import inspect
+import warnings
 from enum import Enum
 from typing import Any, Callable, Mapping, get_origin
 
@@ -29,6 +32,24 @@ from .typing import (
 )
 
 
+class ChildFieldPath:
+    """Context manager to append a field to field_path on enter and pop it on exit."""
+
+    _field_path: list[str]
+    _field_name: str
+
+    def __init__(self, field_path: list[str], field_name: str):
+        self._field_path: list[str] = field_path
+        self._field_name = field_name
+
+    def __enter__(self) -> ChildFieldPath:
+        self._field_path.append(self._field_name)
+        return self
+
+    def __exit__(self, _exc_type: Any, _exc_val: Any, _exc_tb: Any) -> None:
+        self._field_path.pop()
+
+
 def encode_engine_value(value: Any) -> Any:
     """Encode a Python value to an engine value."""
     if dataclasses.is_dataclass(value):
@@ -73,7 +94,8 @@ def _is_type_kind_convertible_to(src_type_kind: str, dst_type_kind: str) -> bool
 def make_engine_value_decoder(
     field_path: list[str],
     src_type: dict[str, Any],
-    dst_annotation: Any,
+    dst_type_info: AnalyzedTypeInfo,
+    for_key: bool = False,
 ) -> Callable[[Any], Any]:
     """
     Make a decoder from an engine value to a Python value.
@@ -89,7 +111,6 @@ def make_engine_value_decoder(
 
     src_type_kind = src_type["kind"]
 
-    dst_type_info = analyze_type_info(dst_annotation)
     dst_type_variant = dst_type_info.variant
 
     if isinstance(dst_type_variant, AnalyzedUnknownType):
@@ -99,96 +120,100 @@ def make_engine_value_decoder(
         )
 
     if src_type_kind == "Struct":
-        return _make_engine_struct_value_decoder(
+        return make_engine_struct_decoder(
             field_path,
             src_type["fields"],
             dst_type_info,
+            for_key=for_key,
         )
 
     if src_type_kind in TABLE_TYPES:
-        field_path.append("[*]")
-        engine_fields_schema = src_type["row"]["fields"]
-
-        if src_type_kind == "LTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ltable_to_list_dict_decoder(
-                    field_path, engine_fields_schema
-                )
-            if not isinstance(dst_type_variant, AnalyzedListType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a list type expected"
+        with ChildFieldPath(field_path, "[*]"):
+            engine_fields_schema = src_type["row"]["fields"]
+
+            if src_type_kind == "LTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    dst_elem_type = Any
+                elif isinstance(dst_type_variant, AnalyzedListType):
+                    dst_elem_type = dst_type_variant.elem_type
+                else:
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a list type expected"
+                    )
+                row_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema,
+                    analyze_type_info(dst_elem_type),
                 )
-            row_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema,
-                analyze_type_info(dst_type_variant.elem_type),
-            )
 
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return [row_decoder(v) for v in value]
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return [row_decoder(v) for v in value]
+
+            elif src_type_kind == "KTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    key_type, value_type = Any, Any
+                elif isinstance(dst_type_variant, AnalyzedDictType):
+                    key_type = dst_type_variant.key_type
+                    value_type = dst_type_variant.value_type
+                else:
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a dict type expected"
+                    )
 
-        elif src_type_kind == "KTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ktable_to_dict_dict_decoder(
-                    field_path, engine_fields_schema
+                key_field_schema = engine_fields_schema[0]
+                field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+                key_decoder = make_engine_value_decoder(
+                    field_path,
+                    key_field_schema["type"],
+                    analyze_type_info(key_type),
+                    for_key=True,
                 )
-            if not isinstance(dst_type_variant, AnalyzedDictType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a dict type expected"
+                field_path.pop()
+                value_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema[1:],
+                    analyze_type_info(value_type),
                 )
 
-            key_field_schema = engine_fields_schema[0]
-            field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-            key_decoder = make_engine_value_decoder(
-                field_path, key_field_schema["type"], dst_type_variant.key_type
-            )
-            field_path.pop()
-            value_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema[1:],
-                analyze_type_info(dst_type_variant.value_type),
-            )
-
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
 
-        field_path.pop()
         return decode
 
     if src_type_kind == "Union":
         if isinstance(dst_type_variant, AnalyzedAnyType):
             return lambda value: value[1]
 
-        dst_type_variants = (
-            dst_type_variant.variant_types
+        dst_type_info_variants = (
+            [analyze_type_info(t) for t in dst_type_variant.variant_types]
             if isinstance(dst_type_variant, AnalyzedUnionType)
-            else [dst_annotation]
+            else [dst_type_info]
         )
         src_type_variants = src_type["types"]
         decoders = []
         for i, src_type_variant in enumerate(src_type_variants):
-            src_field_path = field_path + [f"[{i}]"]
-            decoder = None
-            for dst_type_variant in dst_type_variants:
-                try:
-                    decoder = make_engine_value_decoder(
-                        src_field_path, src_type_variant, dst_type_variant
+            with ChildFieldPath(field_path, f"[{i}]"):
+                decoder = None
+                for dst_type_info_variant in dst_type_info_variants:
+                    try:
+                        decoder = make_engine_value_decoder(
+                            field_path, src_type_variant, dst_type_info_variant
+                        )
+                        break
+                    except ValueError:
+                        pass
+                if decoder is None:
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"cannot find matched target type for source type variant {src_type_variant}"
                     )
-                    break
-                except ValueError:
-                    pass
-            if decoder is None:
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"cannot find matched target type for source type variant {src_type_variant}"
-                )
-            decoders.append(decoder)
+                decoders.append(decoder)
         return lambda value: decoders[value[0]](value[1])
 
     if isinstance(dst_type_variant, AnalyzedAnyType):
@@ -216,7 +241,9 @@ def make_engine_value_decoder(
             vec_elem_decoder = make_engine_value_decoder(
                 field_path + ["[*]"],
                 src_type["element_type"],
-                dst_type_variant and dst_type_variant.elem_type,
+                analyze_type_info(
+                    dst_type_variant.elem_type if dst_type_variant else Any
+                ),
             )
 
         def decode_vector(value: Any) -> Any | None:
@@ -247,7 +274,7 @@ def make_engine_value_decoder(
         if not _is_type_kind_convertible_to(src_type_kind, dst_type_variant.kind):
             raise ValueError(
                 f"Type mismatch for `{''.join(field_path)}`: "
-                f"passed in {src_type_kind}, declared {dst_annotation} ({dst_type_variant.kind})"
+                f"passed in {src_type_kind}, declared {dst_type_info.core_type} ({dst_type_variant.kind})"
             )
 
         if dst_type_variant.kind in ("Float32", "Float64", "Int64"):
@@ -267,30 +294,54 @@ def make_engine_value_decoder(
     return lambda value: value
 
 
-def _make_engine_struct_value_decoder(
+def _get_auto_default_for_type(
+    type_info: AnalyzedTypeInfo,
+) -> tuple[Any, bool]:
+    """
+    Get an auto-default value for a type annotation if it's safe to do so.
+
+    Returns:
+        A tuple of (default_value, is_supported) where:
+        - default_value: The default value if auto-defaulting is supported
+        - is_supported: True if auto-defaulting is supported for this type
+    """
+    # Case 1: Nullable types (Optional[T] or T | None)
+    if type_info.nullable:
+        return None, True
+
+    # Case 2: Table types (KTable or LTable) - check if it's a list or dict type
+    if isinstance(type_info.variant, AnalyzedListType):
+        return [], True
+    elif isinstance(type_info.variant, AnalyzedDictType):
+        return {}, True
+
+    return None, False
+
+
+def make_engine_struct_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
     dst_type_info: AnalyzedTypeInfo,
+    for_key: bool = False,
 ) -> Callable[[list[Any]], Any]:
     """Make a decoder from an engine field values to a Python value."""
 
     dst_type_variant = dst_type_info.variant
 
-    use_dict = False
     if isinstance(dst_type_variant, AnalyzedAnyType):
-        use_dict = True
+        if for_key:
+            return _make_engine_struct_to_tuple_decoder(field_path, src_fields)
+        else:
+            return _make_engine_struct_to_dict_decoder(field_path, src_fields, Any)
     elif isinstance(dst_type_variant, AnalyzedDictType):
         analyzed_key_type = analyze_type_info(dst_type_variant.key_type)
-        analyzed_value_type = analyze_type_info(dst_type_variant.value_type)
-        use_dict = (
+        if (
             isinstance(analyzed_key_type.variant, AnalyzedAnyType)
-            or (
-                isinstance(analyzed_key_type.variant, AnalyzedBasicType)
-                and analyzed_key_type.variant.kind == "Str"
+            or analyzed_key_type.core_type is str
+        ):
+            return _make_engine_struct_to_dict_decoder(
+                field_path, src_fields, dst_type_variant.value_type
             )
-        ) and isinstance(analyzed_value_type.variant, AnalyzedAnyType)
-    if use_dict:
-        return _make_engine_struct_to_dict_decoder(field_path, src_fields)
 
     if not isinstance(dst_type_variant, AnalyzedStructType):
         raise ValueError(
@@ -321,32 +372,39 @@ def _make_engine_struct_value_decoder(
     else:
         raise ValueError(f"Unsupported struct type: {dst_struct_type}")
 
-    def make_closure_for_value(
+    def make_closure_for_field(
         name: str, param: inspect.Parameter
     ) -> Callable[[list[Any]], Any]:
         src_idx = src_name_to_idx.get(name)
-        if src_idx is not None:
-            field_path.append(f".{name}")
-            field_decoder = make_engine_value_decoder(
-                field_path, src_fields[src_idx]["type"], param.annotation
-            )
-            field_path.pop()
-            return (
-                lambda values: field_decoder(values[src_idx])
-                if len(values) > src_idx
-                else param.default
-            )
+        type_info = analyze_type_info(param.annotation)
+
+        with ChildFieldPath(field_path, f".{name}"):
+            if src_idx is not None:
+                field_decoder = make_engine_value_decoder(
+                    field_path, src_fields[src_idx]["type"], type_info, for_key=for_key
+                )
+                return lambda values: field_decoder(values[src_idx])
+
+            default_value = param.default
+            if default_value is not inspect.Parameter.empty:
+                return lambda _: default_value
+
+            auto_default, is_supported = _get_auto_default_for_type(type_info)
+            if is_supported:
+                warnings.warn(
+                    f"Field '{name}' (type {param.annotation}) without default value is missing in input: "
+                    f"{''.join(field_path)}. Auto-assigning default value: {auto_default}",
+                    UserWarning,
+                    stacklevel=4,
+                )
+                return lambda _: auto_default
 
-        default_value = param.default
-        if default_value is inspect.Parameter.empty:
             raise ValueError(
-                f"Field without default value is missing in input: {''.join(field_path)}"
+                f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}"
             )
 
-        return lambda _: default_value
-
     field_value_decoder = [
-        make_closure_for_value(name, param) for (name, param) in parameters.items()
+        make_closure_for_field(name, param) for (name, param) in parameters.items()
     ]
 
     return lambda values: dst_struct_type(
@@ -357,19 +415,20 @@ def _make_engine_struct_value_decoder(
 def _make_engine_struct_to_dict_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
+    value_type_annotation: Any,
 ) -> Callable[[list[Any] | None], dict[str, Any] | None]:
     """Make a decoder from engine field values to a Python dict."""
 
     field_decoders = []
-    for i, field_schema in enumerate(src_fields):
+    value_type_info = analyze_type_info(value_type_annotation)
+    for field_schema in src_fields:
         field_name = field_schema["name"]
-        field_path.append(f".{field_name}")
-        field_decoder = make_engine_value_decoder(
-            field_path,
-            field_schema["type"],
-            Any,  # Use Any for recursive decoding
-        )
-        field_path.pop()
+        with ChildFieldPath(field_path, f".{field_name}"):
+            field_decoder = make_engine_value_decoder(
+                field_path,
+                field_schema["type"],
+                value_type_info,
+            )
         field_decoders.append((field_name, field_decoder))
 
     def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None:
@@ -387,75 +446,37 @@ def _make_engine_struct_to_dict_decoder(
     return decode_to_dict
 
 
-def _make_engine_ltable_to_list_dict_decoder(
+def _make_engine_struct_to_tuple_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
-) -> Callable[[list[Any] | None], list[dict[str, Any]] | None]:
-    """Make a decoder from engine LTable values to a list of dicts."""
-
-    # Create a decoder for each row (struct) to dict
-    row_decoder = _make_engine_struct_to_dict_decoder(field_path, src_fields)
+) -> Callable[[list[Any] | None], tuple[Any, ...] | None]:
+    """Make a decoder from engine field values to a Python tuple."""
 
-    def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None:
-        if values is None:
-            return None
-        result = []
-        for i, row_values in enumerate(values):
-            decoded_row = row_decoder(row_values)
-            if decoded_row is None:
-                raise ValueError(
-                    f"LTable row at index {i} decoded to None, which is not allowed."
+    field_decoders = []
+    value_type_info = analyze_type_info(Any)
+    for field_schema in src_fields:
+        field_name = field_schema["name"]
+        with ChildFieldPath(field_path, f".{field_name}"):
+            field_decoders.append(
+                make_engine_value_decoder(
+                    field_path,
+                    field_schema["type"],
+                    value_type_info,
                 )
-            result.append(decoded_row)
-        return result
-
-    return decode_to_list_dict
-
-
-def _make_engine_ktable_to_dict_dict_decoder(
-    field_path: list[str],
-    src_fields: list[dict[str, Any]],
-) -> Callable[[list[Any] | None], dict[Any, dict[str, Any]] | None]:
-    """Make a decoder from engine KTable values to a dict of dicts."""
-
-    if not src_fields:
-        raise ValueError("KTable must have at least one field for the key")
-
-    # First field is the key, remaining fields are the value
-    key_field_schema = src_fields[0]
-    value_fields_schema = src_fields[1:]
-
-    # Create decoders
-    field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-    key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any)
-    field_path.pop()
-
-    value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema)
+            )
 
-    def decode_to_dict_dict(
-        values: list[Any] | None,
-    ) -> dict[Any, dict[str, Any]] | None:
+    def decode_to_tuple(values: list[Any] | None) -> tuple[Any, ...] | None:
         if values is None:
             return None
-        result = {}
-        for row_values in values:
-            if not row_values:
-                raise ValueError("KTable row must have at least 1 value (the key)")
-            key = key_decoder(row_values[0])
-            if len(row_values) == 1:
-                value: dict[str, Any] = {}
-            else:
-                tmp = value_decoder(row_values[1:])
-                if tmp is None:
-                    value = {}
-                else:
-                    value = tmp
-            if isinstance(key, dict):
-                key = tuple(key.values())
-            result[key] = value
-        return result
+        if len(field_decoders) != len(values):
+            raise ValueError(
+                f"Field count mismatch: expected {len(field_decoders)}, got {len(values)}"
+            )
+        return tuple(
+            field_decoder(value) for value, field_decoder in zip(values, field_decoders)
+        )
 
-    return decode_to_dict_dict
+    return decode_to_tuple
 
 
 def dump_engine_object(v: Any) -> Any:

cocoindex/flow.py

@@ -16,6 +16,7 @@ from .validation import (
     validate_full_flow_name,
     validate_target_name,
 )
+from .typing import analyze_type_info
 
 from dataclasses import dataclass
 from enum import Enum
@@ -797,7 +798,7 @@ class Flow:
         The current instance is still valid after it's called.
         For example, you can still call `setup()` after it, to setup the persistent backends again.
 
-        Call `cocoindex.remove_flow()` if you want to remove the flow from the current process.
+        Call `close()` if you want to remove the flow from the current process.
         """
         execution_context.run(self.drop_async(report_to_stdout=report_to_stdout))
 
@@ -809,6 +810,18 @@ class Flow:
             report_to_stdout=report_to_stdout
         )
 
+    def close(self) -> None:
+        """
+        Close the flow. It will remove the flow from the current process to free up resources.
+        After it's called, methods of the flow should no longer be called.
+
+        This will NOT touch the persistent backends of the flow.
+        """
+        _engine.remove_flow_context(self.full_name)
+        self._lazy_engine_flow = None
+        with _flows_lock:
+            del _flows[self.name]
+
 
 def _create_lazy_flow(
     name: str | None, fl_def: Callable[[FlowBuilder, DataScope], None]
@@ -844,7 +857,10 @@ def get_flow_full_name(name: str) -> str:
     return f"{setting.get_app_namespace(trailing_delimiter='.')}{name}"
 
 
-def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
+def open_flow(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
+    """
+    Open a flow, with the given name and definition.
+    """
     with _flows_lock:
         if name in _flows:
             raise KeyError(f"Flow with name {name} already exists")
@@ -852,17 +868,18 @@ def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) ->
     return fl
 
 
-def remove_flow(fl: Flow) -> None:
+def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
     """
-    Remove a flow from the current process to free up resources.
-    After it's called, methods of the flow should no longer be called.
+    DEPRECATED: Use `open_flow()` instead.
+    """
+    return open_flow(name, fl_def)
+
 
-    This will NOT touch the persistent backends of the flow.
+def remove_flow(fl: Flow) -> None:
     """
-    _engine.remove_flow_context(fl.full_name)
-    fl._lazy_engine_flow = None  # pylint: disable=protected-access
-    with _flows_lock:
-        del _flows[fl.name]
+    DEPRECATED: Use `Flow.close()` instead.
+    """
+    fl.close()
 
 
 def flow_def(
@@ -871,7 +888,7 @@ def flow_def(
     """
     A decorator to wrap the flow definition.
     """
-    return lambda fl_def: add_flow_def(name or fl_def.__name__, fl_def)
+    return lambda fl_def: open_flow(name or fl_def.__name__, fl_def)
 
 
 def flow_names() -> list[str]:
@@ -1053,7 +1070,7 @@ class TransformFlow(Generic[T]):
             sig.return_annotation
         )
         result_decoder = make_engine_value_decoder(
-            [], engine_return_type["type"], python_return_type
+            [], engine_return_type["type"], analyze_type_info(python_return_type)
         )
 
         return TransformFlowInfo(engine_flow, result_decoder)