cocoindex 0.1.69__tar.gz → 0.1.71__tar.gz

{cocoindex-0.1.69 → cocoindex-0.1.71}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.69"
+version = "0.1.71"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.69 → cocoindex-0.1.71}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.69"
+version = "0.1.71"
 edition = "2024"
 rust-version = "1.88"
 

{cocoindex-0.1.69 → cocoindex-0.1.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.69
+Version: 0.1.71
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -52,18 +52,18 @@ Ultra performant data transformation framework for AI, with core engine written
 ⭐ Drop a star to help us grow!
 
 <div align="center">
-    
+
 <!-- Keep these links. Translations will automatically update with the README. -->
-[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
-[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
-[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
-[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
-[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
-[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
-[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
-[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) |
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) |
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) |
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) |
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) |
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) |
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) |
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) |
 [中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
-    
+
 </div>
 
 </br>
@@ -208,6 +208,7 @@ It defines an index flow like this:
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
 | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
+| [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
 
 More coming and stay tuned 👀!

{cocoindex-0.1.69 → cocoindex-0.1.71}/README.md

@@ -27,18 +27,18 @@ Ultra performant data transformation framework for AI, with core engine written
 ⭐ Drop a star to help us grow!
 
 <div align="center">
-    
+
 <!-- Keep these links. Translations will automatically update with the README. -->
-[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
-[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
-[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
-[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
-[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
-[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
-[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
-[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) |
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) |
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) |
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) |
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) |
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) |
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) |
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) |
 [中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
-    
+
 </div>
 
 </br>
@@ -183,6 +183,7 @@ It defines an index flow like this:
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
 | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
+| [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
 
 More coming and stay tuned 👀!

{cocoindex-0.1.69 → cocoindex-0.1.71}/docs/docs/core/custom_function.mdx

@@ -148,6 +148,17 @@ Custom functions take the following additional parameters:
     When the version is changed, the function will be re-executed even if cache is enabled.
     It's required to be set if `cache` is `True`.
 
+*   `arg_relationship: tuple[ArgRelationship, str]`: It specifies the relationship between an input argument and the output,
+    e.g. `(ArgRelationship.CHUNKS_BASE_TEXT, "content")` means the output is chunks for the text represented by the
+    input argument with name `content`.
+    This provides metadata for tools, e.g. CocoInsight.
+    Currently the following attributes are supported:
+
+    *   `ArgRelationship.CHUNKS_BASE_TEXT`:
+        The output is chunks for the text represented by the input argument. In this case, the output is expected to be a *Table*, whose each row represents a text chunk, and the first column has type *Range*, representing the range of the text chunk.
+    *   `ArgRelationship.EMBEDDING_ORIGIN_TEXT`: The output is embedding vector for the text represented by the input argument. The output is expected to be a *Vector*.
+    *   `ArgRelationship.RECTS_BASE_IMAGE`: The output is rectangles for the image represented by the input argument. The output is expected to be a *Table*, whose each row represents a rectangle, and the first column has type *Struct*, with fields `min_x`, `min_y`, `max_x`, `max_y` to represent the coordinates of the rectangle.
+
 For example:
 
 <Tabs>

{cocoindex-0.1.69 → cocoindex-0.1.71}/docs/docs/core/data_types.mdx

@@ -86,7 +86,7 @@ Optionally, it can have a fixed dimension. Noted as *Vector[Type]* or *Vector[Ty
 
 It supports the following Python types:
 
-*   `cocoindex.Vector[T]` or `cocoindex.Vector[T, typing.Literal[Dim]]`, e.g. `cocoindex.Vector[cocoindex.Float32]` or `cocoindex.Vector[cocoindex.Float32, 384]`
+*   `cocoindex.Vector[T]` or `cocoindex.Vector[T, typing.Literal[Dim]]`, e.g. `cocoindex.Vector[cocoindex.Float32]` or `cocoindex.Vector[cocoindex.Float32, typing.Literal[384]]`
     *   The underlying Python type is `numpy.typing.NDArray[T]` where `T` is a numpy numeric type (`numpy.int64`, `numpy.float32` or `numpy.float64`), or `list[T]` otherwise
 *   `numpy.typing.NDArray[T]` where `T` is a numpy numeric type
 *   `list[T]`

{cocoindex-0.1.69 → cocoindex-0.1.71}/docs/docs/core/flow_def.mdx

@@ -416,22 +416,28 @@ flow_builder.declare(
 ### Auth Registry
 
 CocoIndex manages an auth registry. It's an in-memory key-value store, mainly to store authentication information for a backend.
+It's usually used for targets, where key stability is important for backend cleanup.
 
-Operation spec is the default way to configure a persistent backend. But it has the following limitations:
+Operation spec is the default way to configure sources, functions and targets. But it has the following limitations:
 
 *   The spec isn't supposed to contain secret information, and it's frequently shown in various places, e.g. `cocoindex show`.
-*   Once an operation is removed after flow definition code change, the spec is also gone.
-    But we still need to be able to drop the backend (e.g. a table) when [setup / drop flow](/docs/core/flow_methods#setup--drop-flow).
+*   For targets, once an operation is removed after flow definition code change, the spec is also gone.
+    But we still need to be able to drop the persistent backend (e.g. a table) when [setup / drop flow](/docs/core/flow_methods#setup--drop-flow).
 
-Auth registry is introduced to solve the problems above. It works as follows:
+Auth registry is introduced to solve the problems above.
 
-*   You can create new **auth entry** by a key and a value.
-*   You can references the entry by the key, and pass it as part of spec for certain operations. e.g. `Neo4j` takes `connection` field in the form of auth entry reference.
+
+#### Auth Entry
+
+An auth entry is an entry in the auth registry with an explicit key.
+
+*   You can create new *auth entry* by a key and a value.
+*   You can reference the entry by the key, and pass it as part of spec for certain operations. e.g. `Neo4j` takes `connection` field in the form of auth entry reference.
 
 <Tabs>
 <TabItem value="python" label="Python" default>
 
-You can add an auth entry by `cocoindex.add_auth_entry()` function, which returns a `cocoindex.AuthEntryReference`:
+You can add an auth entry by `cocoindex.add_auth_entry()` function, which returns a `cocoindex.AuthEntryReference[T]`:
 
 ```python
 my_graph_conn = cocoindex.add_auth_entry(
@@ -445,7 +451,7 @@ my_graph_conn = cocoindex.add_auth_entry(
 
 Then reference it when building a spec that takes an auth entry:
 
-*   You can either reference by the `AuthEntryReference` object directly:
+*   You can either reference by the `AuthEntryReference[T]` object directly:
 
     ```python
     demo_collector.export(
@@ -472,3 +478,28 @@ Note that CocoIndex backends use the key of an auth entry to identify the backen
 
 *   If a key is no longer referenced in any operation spec, keep it until the next flow setup / drop action,
     so that CocoIndex will be able to clean up the backends.
+
+#### Transient Auth Entry
+
+A transient auth entry is an entry in the auth registry with an automatically generated key.
+It's usually used for sources and functions, where key stability is not important.
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+You can create a new *transient auth entry* by `cocoindex.add_transient_auth_entry()` function, which returns a `cocoindex.TransientAuthEntryReference[T]`, and pass it to a source or function spec that takes it, e.g.
+
+```python
+flow_builder.add_source(
+    cocoindex.sources.AzureBlob(
+        ...
+        sas_token=cocoindex.add_transient_auth_entry("...")
+    )
+)
+```
+
+
+</TabItem>
+</Tabs>
+
+Whenever a `TransientAuthEntryReference[T]` is expected, you can also pass a `AuthEntryReference[T]` instead, as `AuthEntryReference[T]` is a subtype of `TransientAuthEntryReference[T]`.

{cocoindex-0.1.69 → cocoindex-0.1.71}/docs/docs/ops/sources.md

@@ -170,22 +170,33 @@ These are actions you need to take:
 
 #### Authentication
 
-We use Azure’s **Default Credential** system (DefaultAzureCredential) for secure and flexible authentication.
-This allows you to connect to Azure services without putting any secrets in the code or flow spec.
-It automatically chooses the best authentication method based on your environment:
-
-*   On your local machine: uses your Azure CLI login (`az login`) or environment variables.
-
-    ```sh
-    az login
-    # Optional: Set a default subscription if you have more than one
-    az account set --subscription "<YOUR_SUBSCRIPTION_NAME_OR_ID>"
-    ```
-*   In Azure (VM, App Service, AKS, etc.): uses the resource’s Managed Identity.
-*   In automated environments: supports Service Principals via environment variables
-    *   `AZURE_CLIENT_ID`
-    *   `AZURE_TENANT_ID`
-    *   `AZURE_CLIENT_SECRET`
+We support the following authentication methods:
+
+*   Shared access signature (SAS) tokens.
+    You can generate it from the Azure Portal in the settings for a specific container.
+    You need to provide at least *List* and *Read* permissions when generating the SAS token.
+    It's a query string in the form of
+    `sp=rl&st=2025-07-20T09:33:00Z&se=2025-07-19T09:48:53Z&sv=2024-11-04&sr=c&sig=i3FDjsadfklj3%23adsfkk`.
+
+*   Storage account access key. You can find it in the Azure Portal in the settings for a specific storage account.
+
+*   Default credential. When none of the above is provided, it will use the default credential.
+
+    This allows you to connect to Azure services without putting any secrets in the code or flow spec.
+    It automatically chooses the best authentication method based on your environment:
+
+    *   On your local machine: uses your Azure CLI login (`az login`) or environment variables.
+
+        ```sh
+        az login
+        # Optional: Set a default subscription if you have more than one
+        az account set --subscription "<YOUR_SUBSCRIPTION_NAME_OR_ID>"
+        ```
+    *   In Azure (VM, App Service, AKS, etc.): uses the resource’s Managed Identity.
+    *   In automated environments: supports Service Principals via environment variables
+        *   `AZURE_CLIENT_ID`
+        *   `AZURE_TENANT_ID`
+        *   `AZURE_CLIENT_SECRET`
 
 You can refer to [this doc](https://learn.microsoft.com/en-us/azure/developer/python/sdk/authentication/overview) for more details.
 
@@ -202,6 +213,8 @@ The spec takes the following fields:
 *   `excluded_patterns` (`list[str]`, optional): a list of glob patterns to exclude files, e.g. `["*.tmp", "**/*.log"]`.
     Any file or directory matching these patterns will be excluded even if they match `included_patterns`.
     If not specified, no files will be excluded.
+*   `sas_token` (`cocoindex.TransientAuthEntryReference[str]`, optional): a SAS token for authentication.
+*   `account_access_key` (`cocoindex.TransientAuthEntryReference[str]`, optional): an account access key for authentication.
 
     :::info
 

{cocoindex-0.1.69 → cocoindex-0.1.71}/examples/amazon_s3_embedding/main.py

@@ -102,12 +102,6 @@ def _main() -> None:
 
     amazon_s3_text_embedding_flow.setup()
     with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow) as updater:
-        while True:
-            updates = updater.next_status_updates()
-            print(f"Updates: {updates}")
-            if not updates.active_sources:
-                break
-
         # Run queries in a loop to demonstrate the query capabilities.
         while True:
             query = input("Enter search query (or Enter to quit): ")

cocoindex-0.1.71/examples/face_recognition/README.md

@@ -0,0 +1,51 @@
+# Recognize faces in images and build embedding index
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+
+In this example, we will recognize faces in images and build embedding index.
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+## Steps
+### Indexing Flow
+
+1. We will ingest a list of images.
+2. For each image, we:
+   - Extract faces from the image.
+   - Compute embeddings for each face.
+3. We will export to the following tables in Postgres with PGVector:
+   - Filename, rect, embedding for each face.
+
+
+## Prerequisite
+
+1.  [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+2.  dependencies:
+
+    ```bash
+    pip install -e .
+    ```
+
+## Run
+
+Update index, which will also setup the tables at the first time:
+
+```bash
+cocoindex update --setup main.py
+```
+
+You can also run the command with `-L`, which will watch for file changes and update the index automatically.
+
+```bash
+cocoindex update --setup -L main.py
+```
+
+## CocoInsight
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline. It just connects to your local CocoIndex server, with zero pipeline data retention. Run following command to start CocoInsight:
+
+```
+cocoindex server -ci main.py
+```
+
+Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

cocoindex-0.1.71/examples/face_recognition/main.py

@@ -0,0 +1,115 @@
+import cocoindex
+import io
+import dataclasses
+import datetime
+import typing
+
+import face_recognition
+from PIL import Image
+import numpy as np
+
+
+@dataclasses.dataclass
+class ImageRect:
+    top: int
+    left: int
+    bottom: int
+    right: int
+
+
+@dataclasses.dataclass
+class FaceBase:
+    """A face in an image."""
+
+    rect: ImageRect
+    image: bytes
+
+
+MAX_IMAGE_WIDTH = 1280
+
+
+@cocoindex.op.function(cache=True, behavior_version=1, gpu=True)
+def extract_faces(content: bytes) -> list[FaceBase]:
+    """Extract the first pages of a PDF."""
+    orig_img = Image.open(io.BytesIO(content)).convert("RGB")
+
+    # The model is too slow on large images, so we resize them if too large.
+    if orig_img.width > MAX_IMAGE_WIDTH:
+        ratio = orig_img.width * 1.0 / MAX_IMAGE_WIDTH
+        img = orig_img.resize(
+            (MAX_IMAGE_WIDTH, int(orig_img.height / ratio)),
+            resample=Image.Resampling.BICUBIC,
+        )
+    else:
+        ratio = 1.0
+        img = orig_img
+
+    # Extract face locations.
+    locs = face_recognition.face_locations(np.array(img), model="cnn")
+
+    faces: list[FaceBase] = []
+    for top, right, bottom, left in locs:
+        rect = ImageRect(
+            left=int(left * ratio),
+            top=int(top * ratio),
+            right=int(right * ratio),
+            bottom=int(bottom * ratio),
+        )
+
+        # Crop the face and save it as a PNG.
+        buf = io.BytesIO()
+        orig_img.crop((rect.left, rect.top, rect.right, rect.bottom)).save(
+            buf, format="PNG"
+        )
+        face = buf.getvalue()
+        faces.append(FaceBase(rect, face))
+
+    return faces
+
+
+@cocoindex.op.function(cache=True, behavior_version=1, gpu=True)
+def extract_face_embedding(
+    face: bytes,
+) -> cocoindex.Vector[cocoindex.Float32, typing.Literal[128]]:
+    """Extract the embedding of a face."""
+    img = Image.open(io.BytesIO(face)).convert("RGB")
+    embedding = face_recognition.face_encodings(
+        np.array(img),
+        known_face_locations=[(0, img.width - 1, img.height - 1, 0)],
+    )[0]
+    return embedding
+
+
+@cocoindex.flow_def(name="FaceRecognition")
+def face_recognition_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+) -> None:
+    """
+    Define an example flow that embeds files into a vector database.
+    """
+    data_scope["images"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="images", binary=True),
+        refresh_interval=datetime.timedelta(seconds=10),
+    )
+
+    face_embeddings = data_scope.add_collector()
+
+    with data_scope["images"].row() as image:
+        # Extract faces
+        image["faces"] = image["content"].transform(extract_faces)
+
+        with image["faces"].row() as face:
+            face["embedding"] = face["image"].transform(extract_face_embedding)
+
+            # Collect embeddings
+            face_embeddings.collect(
+                filename=image["filename"],
+                rect=face["rect"],
+                embedding=face["embedding"],
+            )
+
+    face_embeddings.export(
+        "face_embeddings",
+        cocoindex.targets.Postgres(),
+        primary_key_fields=["filename", "rect"],
+    )

cocoindex-0.1.71/examples/face_recognition/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "cocoindex-face-recognition-example"
+version = "0.1.0"
+description = "Build index for papers with both metadata and content embeddings"
+requires-python = ">=3.11"
+dependencies = [
+    "cocoindex>=0.1.67",
+    "face-recognition>=1.3.0",
+    "pillow>=10.0.0",
+    "numpy>=1.26.0",
+]
+
+[tool.setuptools]
+packages = []

cocoindex-0.1.71/examples/text_embedding_qdrant/.env

@@ -0,0 +1,2 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

{cocoindex-0.1.69 → cocoindex-0.1.71}/python/cocoindex/__init__.py

@@ -6,7 +6,7 @@ from . import functions, sources, targets, cli, utils
 
 from . import targets as storages  # Deprecated: Use targets instead
 
-from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
+from .auth_registry import AuthEntryReference, add_auth_entry, add_transient_auth_entry
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
@@ -38,10 +38,12 @@ __all__ = [
     "targets",
     "storages",
     "cli",
+    "op",
     "utils",
     # Auth registry
     "AuthEntryReference",
     "add_auth_entry",
+    "add_transient_auth_entry",
     "ref_auth_entry",
     # Flow
     "FlowBuilder",

cocoindex-0.1.71/python/cocoindex/auth_registry.py

@@ -0,0 +1,51 @@
+"""
+Auth registry is used to register and reference auth entries.
+"""
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+import threading
+
+from . import _engine  # type: ignore
+from .convert import dump_engine_object
+
+T = TypeVar("T")
+
+# Global atomic counter for generating unique auth entry keys
+_counter_lock = threading.Lock()
+_auth_key_counter = 0
+
+
+def _generate_auth_key() -> str:
+    """Generate a unique auth entry key using a global atomic counter."""
+    global _auth_key_counter  # pylint: disable=global-statement
+    with _counter_lock:
+        _auth_key_counter += 1
+        return f"__auth_{_auth_key_counter}"
+
+
+@dataclass
+class TransientAuthEntryReference(Generic[T]):
+    """Reference an auth entry, may or may not have a stable key."""
+
+    key: str
+
+
+class AuthEntryReference(TransientAuthEntryReference[T]):
+    """Reference an auth entry, with a key stable across ."""
+
+
+def add_transient_auth_entry(value: T) -> TransientAuthEntryReference[T]:
+    """Add an auth entry to the registry. Returns its reference."""
+    return add_auth_entry(_generate_auth_key(), value)
+
+
+def add_auth_entry(key: str, value: T) -> AuthEntryReference[T]:
+    """Add an auth entry to the registry. Returns its reference."""
+    _engine.add_auth_entry(key, dump_engine_object(value))
+    return AuthEntryReference(key)
+
+
+def ref_auth_entry(key: str) -> AuthEntryReference[T]:
+    """Reference an auth entry by its key."""
+    return AuthEntryReference(key)

{cocoindex-0.1.69 → cocoindex-0.1.71}/python/cocoindex/convert.py

@@ -92,10 +92,14 @@ def make_engine_value_decoder(
         if src_type_kind == "Struct":
             return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
         if src_type_kind in TABLE_TYPES:
-            raise ValueError(
-                f"Missing type annotation for `{''.join(field_path)}`."
-                f"It's required for {src_type_kind} type."
-            )
+            if src_type_kind == "LTable":
+                return _make_engine_ltable_to_list_dict_decoder(
+                    field_path, src_type["row"]["fields"]
+                )
+            elif src_type_kind == "KTable":
+                return _make_engine_ktable_to_dict_dict_decoder(
+                    field_path, src_type["row"]["fields"]
+                )
         return lambda value: value
 
     # Handle struct -> dict binding for explicit dict annotations
@@ -340,6 +344,77 @@ def _make_engine_struct_to_dict_decoder(
     return decode_to_dict
 
 
+def _make_engine_ltable_to_list_dict_decoder(
+    field_path: list[str],
+    src_fields: list[dict[str, Any]],
+) -> Callable[[list[Any] | None], list[dict[str, Any]] | None]:
+    """Make a decoder from engine LTable values to a list of dicts."""
+
+    # Create a decoder for each row (struct) to dict
+    row_decoder = _make_engine_struct_to_dict_decoder(field_path, src_fields)
+
+    def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None:
+        if values is None:
+            return None
+        result = []
+        for i, row_values in enumerate(values):
+            decoded_row = row_decoder(row_values)
+            if decoded_row is None:
+                raise ValueError(
+                    f"LTable row at index {i} decoded to None, which is not allowed."
+                )
+            result.append(decoded_row)
+        return result
+
+    return decode_to_list_dict
+
+
+def _make_engine_ktable_to_dict_dict_decoder(
+    field_path: list[str],
+    src_fields: list[dict[str, Any]],
+) -> Callable[[list[Any] | None], dict[Any, dict[str, Any]] | None]:
+    """Make a decoder from engine KTable values to a dict of dicts."""
+
+    if not src_fields:
+        raise ValueError("KTable must have at least one field for the key")
+
+    # First field is the key, remaining fields are the value
+    key_field_schema = src_fields[0]
+    value_fields_schema = src_fields[1:]
+
+    # Create decoders
+    field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+    key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any)
+    field_path.pop()
+
+    value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema)
+
+    def decode_to_dict_dict(
+        values: list[Any] | None,
+    ) -> dict[Any, dict[str, Any]] | None:
+        if values is None:
+            return None
+        result = {}
+        for row_values in values:
+            if not row_values:
+                raise ValueError("KTable row must have at least 1 value (the key)")
+            key = key_decoder(row_values[0])
+            if len(row_values) == 1:
+                value: dict[str, Any] = {}
+            else:
+                tmp = value_decoder(row_values[1:])
+                if tmp is None:
+                    value = {}
+                else:
+                    value = tmp
+            if isinstance(key, dict):
+                key = tuple(key.values())
+            result[key] = value
+        return result
+
+    return decode_to_dict_dict
+
+
 def dump_engine_object(v: Any) -> Any:
     """Recursively dump an object for engine. Engine side uses `Pythonized` to catch."""
     if v is None: