cocoindex 0.1.70__cp311-cp311-win_amd64.whl → 0.1.71__cp311-cp311-win_amd64.whl

cocoindex/__init__.py

@@ -38,6 +38,7 @@ __all__ = [
     "targets",
     "storages",
     "cli",
+    "op",
     "utils",
     # Auth registry
     "AuthEntryReference",

cocoindex/convert.py

@@ -92,10 +92,14 @@ def make_engine_value_decoder(
         if src_type_kind == "Struct":
             return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
         if src_type_kind in TABLE_TYPES:
-            raise ValueError(
-                f"Missing type annotation for `{''.join(field_path)}`."
-                f"It's required for {src_type_kind} type."
-            )
+            if src_type_kind == "LTable":
+                return _make_engine_ltable_to_list_dict_decoder(
+                    field_path, src_type["row"]["fields"]
+                )
+            elif src_type_kind == "KTable":
+                return _make_engine_ktable_to_dict_dict_decoder(
+                    field_path, src_type["row"]["fields"]
+                )
         return lambda value: value
 
     # Handle struct -> dict binding for explicit dict annotations
@@ -340,6 +344,77 @@ def _make_engine_struct_to_dict_decoder(
     return decode_to_dict
 
 
+def _make_engine_ltable_to_list_dict_decoder(
+    field_path: list[str],
+    src_fields: list[dict[str, Any]],
+) -> Callable[[list[Any] | None], list[dict[str, Any]] | None]:
+    """Make a decoder from engine LTable values to a list of dicts."""
+
+    # Create a decoder for each row (struct) to dict
+    row_decoder = _make_engine_struct_to_dict_decoder(field_path, src_fields)
+
+    def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None:
+        if values is None:
+            return None
+        result = []
+        for i, row_values in enumerate(values):
+            decoded_row = row_decoder(row_values)
+            if decoded_row is None:
+                raise ValueError(
+                    f"LTable row at index {i} decoded to None, which is not allowed."
+                )
+            result.append(decoded_row)
+        return result
+
+    return decode_to_list_dict
+
+
+def _make_engine_ktable_to_dict_dict_decoder(
+    field_path: list[str],
+    src_fields: list[dict[str, Any]],
+) -> Callable[[list[Any] | None], dict[Any, dict[str, Any]] | None]:
+    """Make a decoder from engine KTable values to a dict of dicts."""
+
+    if not src_fields:
+        raise ValueError("KTable must have at least one field for the key")
+
+    # First field is the key, remaining fields are the value
+    key_field_schema = src_fields[0]
+    value_fields_schema = src_fields[1:]
+
+    # Create decoders
+    field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+    key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any)
+    field_path.pop()
+
+    value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema)
+
+    def decode_to_dict_dict(
+        values: list[Any] | None,
+    ) -> dict[Any, dict[str, Any]] | None:
+        if values is None:
+            return None
+        result = {}
+        for row_values in values:
+            if not row_values:
+                raise ValueError("KTable row must have at least 1 value (the key)")
+            key = key_decoder(row_values[0])
+            if len(row_values) == 1:
+                value: dict[str, Any] = {}
+            else:
+                tmp = value_decoder(row_values[1:])
+                if tmp is None:
+                    value = {}
+                else:
+                    value = tmp
+            if isinstance(key, dict):
+                key = tuple(key.values())
+            result[key] = value
+        return result
+
+    return decode_to_dict_dict
+
+
 def dump_engine_object(v: Any) -> Any:
     """Recursively dump an object for engine. Engine side uses `Pythonized` to catch."""
     if v is None:

cocoindex/functions.py

@@ -66,14 +66,19 @@ class SentenceTransformerEmbed(op.FunctionSpec):
     args: dict[str, Any] | None = None
 
 
-@op.executor_class(gpu=True, cache=True, behavior_version=1)
+@op.executor_class(
+    gpu=True,
+    cache=True,
+    behavior_version=1,
+    arg_relationship=(op.ArgRelationship.EMBEDDING_ORIGIN_TEXT, "text"),
+)
 class SentenceTransformerEmbedExecutor:
     """Executor for SentenceTransformerEmbed."""
 
     spec: SentenceTransformerEmbed
     _model: Any | None = None
 
-    def analyze(self, text: Any) -> type:
+    def analyze(self, _text: Any) -> type:
         try:
             # Only import sentence_transformers locally when it's needed, as its import is very slow.
             import sentence_transformers  # pylint: disable=import-outside-toplevel
@@ -88,11 +93,7 @@ class SentenceTransformerEmbedExecutor:
         args = self.spec.args or {}
         self._model = sentence_transformers.SentenceTransformer(self.spec.model, **args)
         dim = self._model.get_sentence_embedding_dimension()
-        result: type = Annotated[
-            Vector[np.float32, Literal[dim]],  # type: ignore
-            TypeAttr("cocoindex.io/vector_origin_text", text.analyzed_value),
-        ]
-        return result
+        return Vector[np.float32, Literal[dim]]  # type: ignore
 
     def __call__(self, text: str) -> NDArray[np.float32]:
         assert self._model is not None

cocoindex/op.py

@@ -6,11 +6,11 @@ import asyncio
 import dataclasses
 import inspect
 from enum import Enum
-from typing import Any, Awaitable, Callable, Protocol, dataclass_transform
+from typing import Any, Awaitable, Callable, Protocol, dataclass_transform, Annotated
 
 from . import _engine  # type: ignore
 from .convert import encode_engine_value, make_engine_value_decoder
-from .typing import encode_enriched_type, resolve_forward_ref
+from .typing import TypeAttr, encode_enriched_type, resolve_forward_ref
 
 
 class OpCategory(Enum):
@@ -85,6 +85,17 @@ class _FunctionExecutorFactory:
 _gpu_dispatch_lock = asyncio.Lock()
 
 
+_COCOINDEX_ATTR_PREFIX = "cocoindex.io/"
+
+
+class ArgRelationship(Enum):
+    """Specifies the relationship between an input argument and the output."""
+
+    EMBEDDING_ORIGIN_TEXT = _COCOINDEX_ATTR_PREFIX + "embedding_origin_text"
+    CHUNKS_BASE_TEXT = _COCOINDEX_ATTR_PREFIX + "chunk_base_text"
+    RECTS_BASE_IMAGE = _COCOINDEX_ATTR_PREFIX + "rects_base_image"
+
+
 @dataclasses.dataclass
 class OpArgs:
     """
@@ -92,11 +103,15 @@ class OpArgs:
     - cache: Whether the executor will be cached.
     - behavior_version: The behavior version of the executor. Cache will be invalidated if it
       changes. Must be provided if `cache` is True.
+    - arg_relationship: It specifies the relationship between an input argument and the output,
+      e.g. `(ArgRelationship.CHUNKS_BASE_TEXT, "content")` means the output is chunks for the
+      input argument with name `content`.
     """
 
     gpu: bool = False
     cache: bool = False
     behavior_version: int | None = None
+    arg_relationship: tuple[ArgRelationship, str] | None = None
 
 
 def _to_async_call(call: Callable[..., Any]) -> Callable[..., Awaitable[Any]]:
@@ -143,6 +158,15 @@ def _register_op_factory(
             """
             self._args_decoders = []
             self._kwargs_decoders = {}
+            attributes = []
+
+            def process_attribute(arg_name: str, arg: _engine.OpArgSchema) -> None:
+                if op_args.arg_relationship is not None:
+                    related_attr, related_arg_name = op_args.arg_relationship
+                    if related_arg_name == arg_name:
+                        attributes.append(
+                            TypeAttr(related_attr.value, arg.analyzed_value)
+                        )
 
             # Match arguments with parameters.
             next_param_idx = 0
@@ -164,6 +188,7 @@ def _register_op_factory(
                         [arg_name], arg.value_type["type"], arg_param.annotation
                     )
                 )
+                process_attribute(arg_name, arg)
                 if arg_param.kind != inspect.Parameter.VAR_POSITIONAL:
                     next_param_idx += 1
 
@@ -194,6 +219,7 @@ def _register_op_factory(
                 self._kwargs_decoders[kwarg_name] = make_engine_value_decoder(
                     [kwarg_name], kwarg.value_type["type"], arg_param.annotation
                 )
+                process_attribute(kwarg_name, kwarg)
 
             missing_args = [
                 name
@@ -216,9 +242,12 @@ def _register_op_factory(
 
             prepare_method = getattr(executor_cls, "analyze", None)
             if prepare_method is not None:
-                return prepare_method(self, *args, **kwargs)
+                result = prepare_method(self, *args, **kwargs)
             else:
-                return expected_return
+                result = expected_return
+            if len(attributes) > 0:
+                result = Annotated[result, *attributes]
+            return result
 
         async def prepare(self) -> None:
             """

cocoindex/tests/test_convert.py

@@ -1341,3 +1341,130 @@ def test_roundtrip_namedtuple_to_dict_binding() -> None:
     validate_full_roundtrip(
         instance, Point, (expected_dict, dict), (expected_dict, Any)
     )
+
+
+def test_roundtrip_ltable_to_list_dict_binding() -> None:
+    """Test LTable -> list[dict] binding with Any annotation."""
+
+    @dataclass
+    class User:
+        id: str
+        name: str
+        age: int
+
+    users = [User("u1", "Alice", 25), User("u2", "Bob", 30), User("u3", "Charlie", 35)]
+    expected_list_dict = [
+        {"id": "u1", "name": "Alice", "age": 25},
+        {"id": "u2", "name": "Bob", "age": 30},
+        {"id": "u3", "name": "Charlie", "age": 35},
+    ]
+
+    # Test Any annotation
+    validate_full_roundtrip(users, list[User], (expected_list_dict, Any))
+
+
+def test_roundtrip_ktable_to_dict_dict_binding() -> None:
+    """Test KTable -> dict[K, dict] binding with Any annotation."""
+
+    @dataclass
+    class Product:
+        name: str
+        price: float
+        active: bool
+
+    products = {
+        "p1": Product("Widget", 29.99, True),
+        "p2": Product("Gadget", 49.99, False),
+        "p3": Product("Tool", 19.99, True),
+    }
+    expected_dict_dict = {
+        "p1": {"name": "Widget", "price": 29.99, "active": True},
+        "p2": {"name": "Gadget", "price": 49.99, "active": False},
+        "p3": {"name": "Tool", "price": 19.99, "active": True},
+    }
+
+    # Test Any annotation
+    validate_full_roundtrip(products, dict[str, Product], (expected_dict_dict, Any))
+
+
+def test_roundtrip_ktable_with_complex_key() -> None:
+    """Test KTable with complex key types -> dict binding."""
+
+    @dataclass(frozen=True)
+    class OrderKey:
+        shop_id: str
+        version: int
+
+    @dataclass
+    class Order:
+        customer: str
+        total: float
+
+    orders = {
+        OrderKey("shop1", 1): Order("Alice", 100.0),
+        OrderKey("shop2", 2): Order("Bob", 200.0),
+    }
+    expected_dict_dict = {
+        ("shop1", 1): {"customer": "Alice", "total": 100.0},
+        ("shop2", 2): {"customer": "Bob", "total": 200.0},
+    }
+
+    # Test Any annotation
+    validate_full_roundtrip(orders, dict[OrderKey, Order], (expected_dict_dict, Any))
+
+
+def test_roundtrip_ltable_with_nested_structs() -> None:
+    """Test LTable with nested structs -> list[dict] binding."""
+
+    @dataclass
+    class Address:
+        street: str
+        city: str
+
+    @dataclass
+    class Person:
+        name: str
+        age: int
+        address: Address
+
+    people = [
+        Person("John", 30, Address("123 Main St", "Anytown")),
+        Person("Jane", 25, Address("456 Oak Ave", "Somewhere")),
+    ]
+    expected_list_dict = [
+        {
+            "name": "John",
+            "age": 30,
+            "address": {"street": "123 Main St", "city": "Anytown"},
+        },
+        {
+            "name": "Jane",
+            "age": 25,
+            "address": {"street": "456 Oak Ave", "city": "Somewhere"},
+        },
+    ]
+
+    # Test Any annotation
+    validate_full_roundtrip(people, list[Person], (expected_list_dict, Any))
+
+
+def test_roundtrip_ktable_with_list_fields() -> None:
+    """Test KTable with list fields -> dict binding."""
+
+    @dataclass
+    class Team:
+        name: str
+        members: list[str]
+        active: bool
+
+    teams = {
+        "team1": Team("Dev Team", ["Alice", "Bob"], True),
+        "team2": Team("QA Team", ["Charlie", "David"], False),
+    }
+    expected_dict_dict = {
+        "team1": {"name": "Dev Team", "members": ["Alice", "Bob"], "active": True},
+        "team2": {"name": "QA Team", "members": ["Charlie", "David"], "active": False},
+    }
+
+    # Test Any annotation
+    validate_full_roundtrip(teams, dict[str, Team], (expected_dict_dict, Any))

{cocoindex-0.1.70.dist-info → cocoindex-0.1.71.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.70
+Version: 0.1.71
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -52,18 +52,18 @@ Ultra performant data transformation framework for AI, with core engine written
 ⭐ Drop a star to help us grow!
 
 <div align="center">
-    
+
 <!-- Keep these links. Translations will automatically update with the README. -->
-[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
-[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
-[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
-[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
-[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
-[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
-[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
-[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) |
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) |
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) |
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) |
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) |
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) |
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) |
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) |
 [中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
-    
+
 </div>
 
 </br>
@@ -208,6 +208,7 @@ It defines an index flow like this:
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
 | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
+| [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
 
 More coming and stay tuned 👀!

{cocoindex-0.1.70.dist-info → cocoindex-0.1.71.dist-info}/RECORD

@@ -1,18 +1,18 @@
-cocoindex-0.1.70.dist-info/METADATA,sha256=4mFNelKh4RZNnPXiqDr1KOQoyBpL6ysW0ff7M5-MMIQ,11426
-cocoindex-0.1.70.dist-info/WHEEL,sha256=auo2gA2SV-bvS4ssY6DIG7dtu3SpmB5FcBkwIwO6YZk,96
-cocoindex-0.1.70.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.1.70.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-cocoindex/__init__.py,sha256=gN0v1r7nktqXGcVld4s1fmdCG9l1J_xRO56Jxu_yXOQ,2193
-cocoindex/_engine.cp311-win_amd64.pyd,sha256=UOvbowU48GTZmnbGHlokQgu5cG3-djm7B1Ycykxpkp8,70170624
+cocoindex-0.1.71.dist-info/METADATA,sha256=32XlzYVA-jBC7LDrr2llh0Sw0UzTSWhFitCFXiQ-eVY,11515
+cocoindex-0.1.71.dist-info/WHEEL,sha256=auo2gA2SV-bvS4ssY6DIG7dtu3SpmB5FcBkwIwO6YZk,96
+cocoindex-0.1.71.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.1.71.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+cocoindex/__init__.py,sha256=1u3kotLS9MDAj4jY4jN5rRuu39B3DH_wL2LAS2VFC5Y,2204
+cocoindex/_engine.cp311-win_amd64.pyd,sha256=6aShU6E3rOtgN7us5bV2h1VEOMSfvGmp2SQclwVMXkU,70170624
 cocoindex/auth_registry.py,sha256=Qq1IVZb-7K4luRrQSDlOPbISnGEZ4kIDsrCU8H2ARw0,1529
 cocoindex/cli.py,sha256=zjZv7EH-ZLoO1-3Ua2U7Yt3n2A_T7pN2NCuv7FB8WA0,23097
-cocoindex/convert.py,sha256=iz7lYydzh1ClMdXl2sc8HiidPp186YY4qXVI133g5u0,13487
+cocoindex/convert.py,sha256=xZXkzZiUQktrGYY0GYGMeK2xi5cTyUN55g2FkA5F80w,16251
 cocoindex/flow.py,sha256=On0J6-3_3uPg-2TjzU3yans_4t_9vchrkH5BpGkbLMo,37203
-cocoindex/functions.py,sha256=pt4Zs_ybz1fBPrwLbgTxHN4yL41SX4IrNGeB5tahzno,3262
+cocoindex/functions.py,sha256=ERtlJJ_D5xNSL4nE6FiqMseX4aAuqsTG6RvJVBC_LPE,3210
 cocoindex/index.py,sha256=GrqTm1rLwICQ8hadtNvJAxVg7GWMvtMmFcbiNtNzmP0,569
 cocoindex/lib.py,sha256=cZosix4nwROvod4QJOwCzrm6U1CVy_wKMMk7sDDG_Z0,849
 cocoindex/llm.py,sha256=TDUyTxW7ooFmpA_jDNezABhReXksOXN8F8zY9aiuNTw,709
-cocoindex/op.py,sha256=h1bp56NEVxCRrOjzyta1h52u6d9Vol_Qau9Pv1sUlVE,12141
+cocoindex/op.py,sha256=6RRlzNf3rq5q1TNxNB8nv9OOee22dHyk9LZ2fD-K20s,13522
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=6mE-jR1Kh5c4GETDvBgwjXZq69TK5rh1qNpaseRDZnw,1117
 cocoindex/setting.py,sha256=E_5Z_MqDt6vaMcqPNA67QB7h_36OBcNu50EN3WjNDts,4898
@@ -20,9 +20,9 @@ cocoindex/setup.py,sha256=KbJvmeFu0NbeoH-5iDmHZP86f26HIId8kHmGUNZAePI,3160
 cocoindex/sources.py,sha256=DEEfJGrz0eG9dFF-sjn7ddbebHfqPpbyN5KXYagbZ50,2152
 cocoindex/targets.py,sha256=7FfG9kuEf5KTXtLwXMFaPFIut3PsIbpb3XIEjjeF7Bg,2931
 cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cocoindex/tests/test_convert.py,sha256=zgWJlhnx8QCnvAD92U2BbFchQgYmFv3hbX532RqVMkk,46793
+cocoindex/tests/test_convert.py,sha256=yVWlR2QMfwdzSXBVIi0eFeQ8k9Xvu8vLGgBhzwS1-ng,50430
 cocoindex/tests/test_optional_database.py,sha256=dnzmTgaJf37D3q8fQsjP5UDER6FYETaUokDnFBMLtIk,8755
 cocoindex/tests/test_typing.py,sha256=YHQonf49SPxic8wnZC_5UmltwS5eAa5VNTH0aVwccBQ,15373
 cocoindex/typing.py,sha256=wVjfugSYNXK9H11pPjXci2X2EyvS5eoILAQ3a0H8VLs,13656
 cocoindex/utils.py,sha256=U3W39zD2uZpXX8v84tJD7sRmbC5ar3z_ljAP1cJrYXI,618
-cocoindex-0.1.70.dist-info/RECORD,,
+cocoindex-0.1.71.dist-info/RECORD,,