cocoindex 0.1.36__cp311-cp311-macosx_11_0_arm64.whl → 0.1.38__cp311-cp311-macosx_11_0_arm64.whl

cocoindex/cli.py

@@ -16,7 +16,7 @@ def cli():
 @cli.command()
 @click.option(
     "-a", "--all", "show_all", is_flag=True, show_default=True, default=False,
-    help="Also show all flows with persisted setup, even if not defined in the current process.")
+    help="Also show all flows with persisted setup under the current app namespace, even if not defined in the current process.")
 def ls(show_all: bool):
     """
     List all flows.
@@ -65,7 +65,7 @@ def show(flow_name: str | None, color: bool, verbose: bool):
 
     console.print()
     table = Table(
-        title=f"Schema for Flow: {flow.name}",
+        title=f"Schema for Flow: {flow.full_name}",
         show_header=True,
         header_style="bold magenta"
     )
@@ -108,7 +108,7 @@ def drop(flow_name: tuple[str, ...], drop_all: bool):
     if drop_all:
         flow_names = flow_names_with_setup()
     elif len(flow_name) == 0:
-        flow_names = [fl.name for fl in flow.flows()]
+        flow_names = flow.flow_names()
     else:
         flow_names = list(flow_name)
     setup_status = drop_setup(flow_names)
@@ -160,7 +160,7 @@ def evaluate(flow_name: str | None, output_dir: str | None, cache: bool = True):
     """
     fl = _flow_by_name(flow_name)
     if output_dir is None:
-        output_dir = f"eval_{fl.name}_{datetime.datetime.now().strftime('%y%m%d_%H%M%S')}"
+        output_dir = f"eval_{setting.get_app_namespace(trailing_delimiter='_')}{flow_name}_{datetime.datetime.now().strftime('%y%m%d_%H%M%S')}"
     options = flow.EvaluateAndDumpOptions(output_dir=output_dir, use_cache=cache)
     fl.evaluate_and_dump(options)
 
@@ -212,11 +212,12 @@ def server(address: str | None, live_update: bool, quiet: bool, cors_origin: str
 
     lib.start_server(server_settings)
 
+    if COCOINDEX_HOST in cors_origins:
+        click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
+
     if live_update:
         options = flow.FlowLiveUpdaterOptions(live_mode=True, print_stats=not quiet)
         flow.update_all_flows(options)
-    if COCOINDEX_HOST in cors_origins:
-        click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
     input("Press Enter to stop...")
 
 

cocoindex/convert.py

@@ -8,13 +8,15 @@ import uuid
 
 from enum import Enum
 from typing import Any, Callable, get_origin
-from .typing import analyze_type_info, encode_enriched_type, TABLE_TYPES, KEY_FIELD_NAME
+from .typing import analyze_type_info, encode_enriched_type, is_namedtuple_type, TABLE_TYPES, KEY_FIELD_NAME
 
 
 def encode_engine_value(value: Any) -> Any:
     """Encode a Python value to an engine value."""
     if dataclasses.is_dataclass(value):
         return [encode_engine_value(getattr(value, f.name)) for f in dataclasses.fields(value)]
+    if is_namedtuple_type(type(value)):
+        return [encode_engine_value(getattr(value, name)) for name in value._fields]
     if isinstance(value, (list, tuple)):
         return [encode_engine_value(v) for v in value]
     if isinstance(value, dict):
@@ -55,16 +57,16 @@ def make_engine_value_decoder(
             f"Type mismatch for `{''.join(field_path)}`: "
             f"passed in {src_type_kind}, declared {dst_annotation} ({dst_type_info.kind})")
 
-    if dst_type_info.dataclass_type is not None:
+    if dst_type_info.struct_type is not None:
         return _make_engine_struct_value_decoder(
-            field_path, src_type['fields'], dst_type_info.dataclass_type)
+            field_path, src_type['fields'], dst_type_info.struct_type)
 
     if src_type_kind in TABLE_TYPES:
         field_path.append('[*]')
         elem_type_info = analyze_type_info(dst_type_info.elem_type)
-        if elem_type_info.dataclass_type is None:
+        if elem_type_info.struct_type is None:
             raise ValueError(f"Type mismatch for `{''.join(field_path)}`: "
-                            f"declared `{dst_type_info.kind}`, a dataclass type expected")
+                            f"declared `{dst_type_info.kind}`, a dataclass or NamedTuple type expected")
         engine_fields_schema = src_type['row']['fields']
         if elem_type_info.key_type is not None:
             key_field_schema = engine_fields_schema[0]
@@ -73,14 +75,14 @@ def make_engine_value_decoder(
                 field_path, key_field_schema['type'], elem_type_info.key_type)
             field_path.pop()
             value_decoder = _make_engine_struct_value_decoder(
-                field_path, engine_fields_schema[1:], elem_type_info.dataclass_type)
+                field_path, engine_fields_schema[1:], elem_type_info.struct_type)
             def decode(value):
                 if value is None:
                     return None
                 return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
         else:
             elem_decoder = _make_engine_struct_value_decoder(
-                field_path, engine_fields_schema, elem_type_info.dataclass_type)
+                field_path, engine_fields_schema, elem_type_info.struct_type)
             def decode(value):
                 if value is None:
                     return None
@@ -96,11 +98,31 @@ def make_engine_value_decoder(
 def _make_engine_struct_value_decoder(
         field_path: list[str],
         src_fields: list[dict[str, Any]],
-        dst_dataclass_type: type,
+        dst_struct_type: type,
     ) -> Callable[[list], Any]:
     """Make a decoder from an engine field values to a Python value."""
 
     src_name_to_idx = {f['name']: i for i, f in enumerate(src_fields)}
+    
+    is_dataclass = dataclasses.is_dataclass(dst_struct_type)
+    is_namedtuple = is_namedtuple_type(dst_struct_type)
+    
+    if is_dataclass:
+        parameters = inspect.signature(dst_struct_type).parameters
+    elif is_namedtuple:
+        defaults = getattr(dst_struct_type, '_field_defaults', {})
+        parameters = {
+            name: inspect.Parameter(
+                name=name,
+                kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                default=defaults.get(name, inspect.Parameter.empty),
+                annotation=dst_struct_type.__annotations__.get(name, inspect.Parameter.empty)
+            )
+            for name in dst_struct_type._fields
+        }
+    else:
+        raise ValueError(f"Unsupported struct type: {dst_struct_type}")
+
     def make_closure_for_value(name: str, param: inspect.Parameter) -> Callable[[list], Any]:
         src_idx = src_name_to_idx.get(name)
         if src_idx is not None:
@@ -108,7 +130,7 @@ def _make_engine_struct_value_decoder(
             field_decoder = make_engine_value_decoder(
                 field_path, src_fields[src_idx]['type'], param.annotation)
             field_path.pop()
-            return lambda values: field_decoder(values[src_idx])
+            return lambda values: field_decoder(values[src_idx]) if len(values) > src_idx else param.default
 
         default_value = param.default
         if default_value is inspect.Parameter.empty:
@@ -119,9 +141,9 @@ def _make_engine_struct_value_decoder(
 
     field_value_decoder = [
         make_closure_for_value(name, param)
-        for (name, param) in inspect.signature(dst_dataclass_type).parameters.items()]
+        for (name, param) in parameters.items()]
 
-    return lambda values: dst_dataclass_type(
+    return lambda values: dst_struct_type(
         *(decoder(values) for decoder in field_value_decoder))
 
 def dump_engine_object(v: Any) -> Any:

cocoindex/flow.py

@@ -19,6 +19,7 @@ from rich.tree import Tree
 from . import _engine
 from . import index
 from . import op
+from . import setting
 from .convert import dump_engine_object
 from .typing import encode_enriched_type
 from .runtime import execution_context
@@ -310,7 +311,7 @@ class _FlowBuilderState:
 
     def __init__(self, /, name: str | None = None):
         flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
-        self.engine_flow_builder = _engine.FlowBuilder(flow_name)
+        self.engine_flow_builder = _engine.FlowBuilder(get_full_flow_name(flow_name))
         self.field_name_builder = _NameBuilder()
 
     def get_data_slice(self, v: Any) -> _engine.DataSlice:
@@ -481,7 +482,7 @@ class Flow:
         Render the flow spec as a styled rich Tree with hierarchical structure.
         """
         spec = self._get_spec(verbose=verbose)
-        tree = Tree(f"Flow: {self.name}", style="cyan")
+        tree = Tree(f"Flow: {self.full_name}", style="cyan")
 
         def build_tree(label: str, lines: list):
             node = Tree(label, style="bold magenta" if lines else "cyan")
@@ -508,9 +509,9 @@ class Flow:
         return repr(self._lazy_engine_flow())
 
     @property
-    def name(self) -> str:
+    def full_name(self) -> str:
         """
-        Get the name of the flow.
+        Get the full name of the flow.
         """
         return self._lazy_engine_flow().name()
 
@@ -566,8 +567,16 @@ def _create_lazy_flow(name: str | None, fl_def: Callable[[FlowBuilder, DataScope
 _flows_lock = Lock()
 _flows: dict[str, Flow] = {}
 
+def get_full_flow_name(name: str) -> str:
+    """
+    Get the full name of a flow.
+    """
+    return f"{setting.get_app_namespace(trailing_delimiter='.')}{name}"
+
 def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
     """Add a flow definition to the cocoindex library."""
+    if not all(c.isalnum() or c == '_' for c in name):
+        raise ValueError(f"Flow name '{name}' contains invalid characters. Only alphanumeric characters and underscores are allowed.")
     with _flows_lock:
         if name in _flows:
             raise KeyError(f"Flow with name {name} already exists")
@@ -587,12 +596,12 @@ def flow_names() -> list[str]:
     with _flows_lock:
         return list(_flows.keys())
 
-def flows() -> list[Flow]:
+def flows() -> dict[str, Flow]:
     """
     Get all flows.
     """
     with _flows_lock:
-        return list(_flows.values())
+        return dict(_flows)
 
 def flow_by_name(name: str) -> Flow:
     """
@@ -605,14 +614,13 @@ def ensure_all_flows_built() -> None:
     """
     Ensure all flows are built.
     """
-    for fl in flows():
-        fl.internal_flow()
+    execution_context.run(ensure_all_flows_built_async())
 
 async def ensure_all_flows_built_async() -> None:
     """
     Ensure all flows are built.
     """
-    for fl in flows():
+    for fl in flows().values():
         await fl.internal_flow_async()
 
 def update_all_flows(options: FlowLiveUpdaterOptions) -> dict[str, _engine.IndexUpdateInfo]:
@@ -626,13 +634,13 @@ async def update_all_flows_async(options: FlowLiveUpdaterOptions) -> dict[str, _
     Update all flows.
     """
     await ensure_all_flows_built_async()
-    async def _update_flow(fl: Flow) -> _engine.IndexUpdateInfo:
+    async def _update_flow(name: str, fl: Flow) -> tuple[str, _engine.IndexUpdateInfo]:
         async with FlowLiveUpdater(fl, options) as updater:
             await updater.wait_async()
-            return updater.update_stats()
+            return (name, updater.update_stats())
     fls = flows()
-    all_stats = await asyncio.gather(*(_update_flow(fl) for fl in fls))
-    return {fl.name: stats for fl, stats in zip(fls, all_stats)}
+    all_stats = await asyncio.gather(*(_update_flow(name, fl) for (name, fl) in fls.items()))
+    return dict(all_stats)
 
 _transient_flow_name_builder = _NameBuilder()
 class TransientFlow:

cocoindex/lib.py

@@ -15,6 +15,7 @@ from .convert import dump_engine_object
 def init(settings: setting.Settings):
     """Initialize the cocoindex library."""
     _engine.init(dump_engine_object(settings))
+    setting.set_app_namespace(settings.app_namespace)
 
 
 def start_server(settings: setting.ServerSettings):

cocoindex/setting.py

@@ -6,6 +6,25 @@ import os
 from typing import Callable, Self, Any, overload
 from dataclasses import dataclass
 
+_app_namespace: str = ''
+
+def get_app_namespace(*, trailing_delimiter: str | None = None) -> str:
+    """Get the application namespace. Append the `trailing_delimiter` if not empty."""
+    if _app_namespace == '' or trailing_delimiter is None:
+        return _app_namespace
+    return f'{_app_namespace}{trailing_delimiter}'
+
+def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
+    """Split the full name into the application namespace and the rest."""
+    parts = full_name.split(delimiter, 1)
+    if len(parts) == 1:
+        return '', parts[0]
+    return (parts[0], parts[1])
+
+def set_app_namespace(app_namespace: str):
+    """Set the application namespace."""
+    global _app_namespace  # pylint: disable=global-statement
+    _app_namespace = app_namespace
 
 @dataclass
 class DatabaseConnectionSpec:
@@ -30,6 +49,7 @@ def _load_field(target: dict[str, Any], name: str, env_name: str, required: bool
 class Settings:
     """Settings for the cocoindex library."""
     database: DatabaseConnectionSpec
+    app_namespace: str
 
     @classmethod
     def from_env(cls) -> Self:
@@ -40,7 +60,10 @@ class Settings:
         _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
         _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
         database = DatabaseConnectionSpec(**db_kwargs)
-        return cls(database=database)
+
+        app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", '')
+
+        return cls(database=database, app_namespace=app_namespace)
 
 @dataclass
 class ServerSettings:

cocoindex/setup.py

@@ -1,4 +1,5 @@
 from . import flow
+from . import setting
 from . import _engine
 
 def sync_setup() -> _engine.SetupStatus:
@@ -7,10 +8,15 @@ def sync_setup() -> _engine.SetupStatus:
 
 def drop_setup(flow_names: list[str]) -> _engine.SetupStatus:
     flow.ensure_all_flows_built()
-    return _engine.drop_setup(flow_names)
+    return _engine.drop_setup([flow.get_full_flow_name(name) for name in flow_names])
 
 def flow_names_with_setup() -> list[str]:
-    return _engine.flow_names_with_setup()
+    result = []
+    for name in _engine.flow_names_with_setup():
+        app_namespace, name = setting.split_app_namespace(name, '.')
+        if app_namespace == setting.get_app_namespace():
+            result.append(name)
+    return result
 
 def apply_setup_changes(setup_status: _engine.SetupStatus):
     _engine.apply_setup_changes(setup_status)

cocoindex/sources.py

@@ -28,3 +28,16 @@ class GoogleDrive(op.SourceSpec):
     root_folder_ids: list[str]
     binary: bool = False
     recent_changes_poll_interval: datetime.timedelta | None = None
+
+
+class AmazonS3(op.SourceSpec):
+    """Import data from an Amazon S3 bucket. Supports optional prefix and file filtering by glob patterns."""
+
+    _op_category = op.OpCategory.SOURCE
+
+    bucket_name: str
+    prefix: str | None = None
+    binary: bool = False
+    included_patterns: list[str] | None = None
+    excluded_patterns: list[str] | None = None
+    sqs_queue_url: str | None = None

cocoindex/tests/test_convert.py

@@ -1,11 +1,12 @@
 import uuid
 import datetime
 from dataclasses import dataclass, make_dataclass
+from typing import NamedTuple, Literal
 import pytest
 import cocoindex
 from cocoindex.typing import encode_enriched_type
 from cocoindex.convert import encode_engine_value, make_engine_value_decoder
-from typing import Literal
+
 @dataclass
 class Order:
     order_id: str
@@ -33,6 +34,17 @@ class NestedStruct:
     orders: list[Order]
     count: int = 0
 
+class OrderNamedTuple(NamedTuple):
+    order_id: str
+    name: str
+    price: float
+    extra_field: str = "default_extra"
+
+class CustomerNamedTuple(NamedTuple):
+    name: str
+    order: OrderNamedTuple
+    tags: list[Tag] | None = None
+
 def build_engine_value_decoder(engine_type_in_py, python_type=None):
     """
     Helper to build a converter for the given engine-side type (as represented in Python).
@@ -62,10 +74,16 @@ def test_encode_engine_value_date_time_types():
 def test_encode_engine_value_struct():
     order = Order(order_id="O123", name="mixed nuts", price=25.0)
     assert encode_engine_value(order) == ["O123", "mixed nuts", 25.0, "default_extra"]
+    
+    order_nt = OrderNamedTuple(order_id="O123", name="mixed nuts", price=25.0)
+    assert encode_engine_value(order_nt) == ["O123", "mixed nuts", 25.0, "default_extra"]
 
 def test_encode_engine_value_list_of_structs():
     orders = [Order("O1", "item1", 10.0), Order("O2", "item2", 20.0)]
     assert encode_engine_value(orders) == [["O1", "item1", 10.0, "default_extra"], ["O2", "item2", 20.0, "default_extra"]]
+    
+    orders_nt = [OrderNamedTuple("O1", "item1", 10.0), OrderNamedTuple("O2", "item2", 20.0)]
+    assert encode_engine_value(orders_nt) == [["O1", "item1", 10.0, "default_extra"], ["O2", "item2", 20.0, "default_extra"]]
 
 def test_encode_engine_value_struct_with_list():
     basket = Basket(items=["apple", "banana"])
@@ -74,6 +92,9 @@ def test_encode_engine_value_struct_with_list():
 def test_encode_engine_value_nested_struct():
     customer = Customer(name="Alice", order=Order("O1", "item1", 10.0))
     assert encode_engine_value(customer) == ["Alice", ["O1", "item1", 10.0, "default_extra"], None]
+    
+    customer_nt = CustomerNamedTuple(name="Alice", order=OrderNamedTuple("O1", "item1", 10.0))
+    assert encode_engine_value(customer_nt) == ["Alice", ["O1", "item1", 10.0, "default_extra"], None]
 
 def test_encode_engine_value_empty_list():
     assert encode_engine_value([]) == []
@@ -103,20 +124,34 @@ def test_make_engine_value_decoder_basic_types():
 @pytest.mark.parametrize(
     "data_type, engine_val, expected",
     [
-        # All fields match
+        # All fields match (dataclass)
         (Order, ["O123", "mixed nuts", 25.0, "default_extra"], Order("O123", "mixed nuts", 25.0, "default_extra")),
+        # All fields match (NamedTuple)
+        (OrderNamedTuple, ["O123", "mixed nuts", 25.0, "default_extra"], OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra")),
         # Extra field in engine value (should ignore extra)
         (Order, ["O123", "mixed nuts", 25.0, "default_extra", "unexpected"], Order("O123", "mixed nuts", 25.0, "default_extra")),
+        (OrderNamedTuple, ["O123", "mixed nuts", 25.0, "default_extra", "unexpected"], OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra")),
         # Fewer fields in engine value (should fill with default)
         (Order, ["O123", "mixed nuts", 0.0, "default_extra"], Order("O123", "mixed nuts", 0.0, "default_extra")),
+        (OrderNamedTuple, ["O123", "mixed nuts", 0.0, "default_extra"], OrderNamedTuple("O123", "mixed nuts", 0.0, "default_extra")),
         # More fields in engine value (should ignore extra)
         (Order, ["O123", "mixed nuts", 25.0, "unexpected"], Order("O123", "mixed nuts", 25.0, "unexpected")),
+        (OrderNamedTuple, ["O123", "mixed nuts", 25.0, "unexpected"], OrderNamedTuple("O123", "mixed nuts", 25.0, "unexpected")),
         # Truly extra field (should ignore the fifth field)
         (Order, ["O123", "mixed nuts", 25.0, "default_extra", "ignored"], Order("O123", "mixed nuts", 25.0, "default_extra")),
+        (OrderNamedTuple, ["O123", "mixed nuts", 25.0, "default_extra", "ignored"], OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra")),
         # Missing optional field in engine value (tags=None)
         (Customer, ["Alice", ["O1", "item1", 10.0, "default_extra"], None], Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), None)),
+        (CustomerNamedTuple, ["Alice", ["O1", "item1", 10.0, "default_extra"], None], CustomerNamedTuple("Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), None)),
         # Extra field in engine value for Customer (should ignore)
         (Customer, ["Alice", ["O1", "item1", 10.0, "default_extra"], [["vip"]], "extra"], Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), [Tag("vip")])),
+        (CustomerNamedTuple, ["Alice", ["O1", "item1", 10.0, "default_extra"], [["vip"]], "extra"], CustomerNamedTuple("Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), [Tag("vip")])),
+        # Missing optional field with default
+        (Order, ["O123", "mixed nuts", 25.0], Order("O123", "mixed nuts", 25.0, "default_extra")),
+        (OrderNamedTuple, ["O123", "mixed nuts", 25.0], OrderNamedTuple("O123", "mixed nuts", 25.0, "default_extra")),
+        # Partial optional fields
+        (Customer, ["Alice", ["O1", "item1", 10.0]], Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), None)),
+        (CustomerNamedTuple, ["Alice", ["O1", "item1", 10.0]], CustomerNamedTuple("Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), None)),
     ]
 )
 def test_struct_decoder_cases(data_type, engine_val, expected):
@@ -124,17 +159,27 @@ def test_struct_decoder_cases(data_type, engine_val, expected):
     assert decoder(engine_val) == expected
 
 def test_make_engine_value_decoder_collections():
-    # List of structs
+    # List of structs (dataclass)
     decoder = build_engine_value_decoder(list[Order])
     engine_val = [
         ["O1", "item1", 10.0, "default_extra"],
         ["O2", "item2", 20.0, "default_extra"]
     ]
     assert decoder(engine_val) == [Order("O1", "item1", 10.0, "default_extra"), Order("O2", "item2", 20.0, "default_extra")]
+    
+    # List of structs (NamedTuple)
+    decoder = build_engine_value_decoder(list[OrderNamedTuple])
+    assert decoder(engine_val) == [OrderNamedTuple("O1", "item1", 10.0, "default_extra"), OrderNamedTuple("O2", "item2", 20.0, "default_extra")]
+    
     # Struct with list field
     decoder = build_engine_value_decoder(Customer)
     engine_val = ["Alice", ["O1", "item1", 10.0, "default_extra"], [["vip"], ["premium"]]]
     assert decoder(engine_val) == Customer("Alice", Order("O1", "item1", 10.0, "default_extra"), [Tag("vip"), Tag("premium")])
+    
+    # NamedTuple with list field
+    decoder = build_engine_value_decoder(CustomerNamedTuple)
+    assert decoder(engine_val) == CustomerNamedTuple("Alice", OrderNamedTuple("O1", "item1", 10.0, "default_extra"), [Tag("vip"), Tag("premium")])
+    
     # Struct with struct field
     decoder = build_engine_value_decoder(NestedStruct)
     engine_val = [
@@ -239,6 +284,13 @@ def test_roundtrip_ltable():
     assert encoded == [["O1", "item1", 10.0, "default_extra"], ["O2", "item2", 20.0, "default_extra"]]
     decoded = build_engine_value_decoder(t)(encoded)
     assert decoded == value
+    
+    t_nt = list[OrderNamedTuple]
+    value_nt = [OrderNamedTuple("O1", "item1", 10.0), OrderNamedTuple("O2", "item2", 20.0)]
+    encoded = encode_engine_value(value_nt)
+    assert encoded == [["O1", "item1", 10.0, "default_extra"], ["O2", "item2", 20.0, "default_extra"]]
+    decoded = build_engine_value_decoder(t_nt)(encoded)
+    assert decoded == value_nt
 
 def test_roundtrip_ktable_str_key():
     t = dict[str, Order]
@@ -247,6 +299,13 @@ def test_roundtrip_ktable_str_key():
     assert encoded == [["K1", "O1", "item1", 10.0, "default_extra"], ["K2", "O2", "item2", 20.0, "default_extra"]]
     decoded = build_engine_value_decoder(t)(encoded)
     assert decoded == value
+    
+    t_nt = dict[str, OrderNamedTuple]
+    value_nt = {"K1": OrderNamedTuple("O1", "item1", 10.0), "K2": OrderNamedTuple("O2", "item2", 20.0)}
+    encoded = encode_engine_value(value_nt)
+    assert encoded == [["K1", "O1", "item1", 10.0, "default_extra"], ["K2", "O2", "item2", 20.0, "default_extra"]]
+    decoded = build_engine_value_decoder(t_nt)(encoded)
+    assert decoded == value_nt
 
 def test_roundtrip_ktable_struct_key():
     @dataclass(frozen=True)
@@ -261,6 +320,14 @@ def test_roundtrip_ktable_struct_key():
                        [["B", 4], "O2", "item2", 20.0, "default_extra"]]
     decoded = build_engine_value_decoder(t)(encoded)
     assert decoded == value
+    
+    t_nt = dict[OrderKey, OrderNamedTuple]
+    value_nt = {OrderKey("A", 3): OrderNamedTuple("O1", "item1", 10.0), OrderKey("B", 4): OrderNamedTuple("O2", "item2", 20.0)}
+    encoded = encode_engine_value(value_nt)
+    assert encoded == [[["A", 3], "O1", "item1", 10.0, "default_extra"],
+                       [["B", 4], "O2", "item2", 20.0, "default_extra"]]
+    decoded = build_engine_value_decoder(t_nt)(encoded)
+    assert decoded == value_nt
 
 IntVectorType = cocoindex.Vector[int, Literal[5]]
 def test_vector_as_vector() -> None:

cocoindex/typing.py

@@ -56,8 +56,11 @@ KEY_FIELD_NAME = '_key'
 
 ElementType = type | tuple[type, type]
 
+def is_namedtuple_type(t) -> bool:
+    return isinstance(t, type) and issubclass(t, tuple) and hasattr(t, "_fields")
+
 def _is_struct_type(t) -> bool:
-    return isinstance(t, type) and dataclasses.is_dataclass(t)
+    return isinstance(t, type) and (dataclasses.is_dataclass(t) or is_namedtuple_type(t))
 
 @dataclasses.dataclass
 class AnalyzedTypeInfo:
@@ -69,7 +72,7 @@ class AnalyzedTypeInfo:
     elem_type: ElementType | None   # For Vector and Table
 
     key_type: type | None           # For element of KTable
-    dataclass_type: type | None     # For Struct
+    struct_type: type | None        # For Struct, a dataclass or namedtuple
 
     attrs: dict[str, Any] | None
     nullable: bool = False
@@ -117,15 +120,16 @@ def analyze_type_info(t) -> AnalyzedTypeInfo:
         elif isinstance(attr, TypeKind):
             kind = attr.kind
 
-    dataclass_type = None
+    struct_type = None
     elem_type = None
     key_type = None
     if _is_struct_type(t):
+        struct_type = t
+
         if kind is None:
             kind = 'Struct'
         elif kind != 'Struct':
             raise ValueError(f"Unexpected type kind for struct: {kind}")
-        dataclass_type = t
     elif base_type is collections.abc.Sequence or base_type is list:
         args = typing.get_args(t)
         elem_type = args[0]
@@ -167,36 +171,50 @@ def analyze_type_info(t) -> AnalyzedTypeInfo:
         else:
             raise ValueError(f"type unsupported yet: {t}")
 
-    return AnalyzedTypeInfo(kind=kind, vector_info=vector_info,
-                            elem_type=elem_type, key_type=key_type, dataclass_type=dataclass_type,
-                            attrs=attrs, nullable=nullable)
-
-def _encode_fields_schema(dataclass_type: type, key_type: type | None = None) -> list[dict[str, Any]]:
+    return AnalyzedTypeInfo(
+        kind=kind,
+        vector_info=vector_info,
+        elem_type=elem_type,
+        key_type=key_type,
+        struct_type=struct_type,
+        attrs=attrs,
+        nullable=nullable,
+    )
+
+def _encode_fields_schema(struct_type: type, key_type: type | None = None) -> list[dict[str, Any]]:
     result = []
     def add_field(name: str, t) -> None:
         try:
             type_info = encode_enriched_type_info(analyze_type_info(t))
         except ValueError as e:
-            e.add_note(f"Failed to encode annotation for field - "
-                       f"{dataclass_type.__name__}.{name}: {t}")
+            e.add_note(
+                f"Failed to encode annotation for field - "
+                f"{struct_type.__name__}.{name}: {t}"
+            )
             raise
         type_info['name'] = name
         result.append(type_info)
 
     if key_type is not None:
         add_field(KEY_FIELD_NAME, key_type)
-    for field in dataclasses.fields(dataclass_type):
-        add_field(field.name, field.type)
+
+    if dataclasses.is_dataclass(struct_type):
+        for field in dataclasses.fields(struct_type):
+            add_field(field.name, field.type)
+    elif is_namedtuple_type(struct_type):
+        for name, field_type in struct_type.__annotations__.items():
+            add_field(name, field_type)
+
     return result
 
 def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
     encoded_type: dict[str, Any] = { 'kind': type_info.kind }
 
     if type_info.kind == 'Struct':
-        if type_info.dataclass_type is None:
-            raise ValueError("Struct type must have a dataclass type")
-        encoded_type['fields'] = _encode_fields_schema(type_info.dataclass_type, type_info.key_type)
-        if doc := inspect.getdoc(type_info.dataclass_type):
+        if type_info.struct_type is None:
+            raise ValueError("Struct type must have a dataclass or namedtuple type")
+        encoded_type['fields'] = _encode_fields_schema(type_info.struct_type, type_info.key_type)
+        if doc := inspect.getdoc(type_info.struct_type):
             encoded_type['description'] = doc
 
     elif type_info.kind == 'Vector':

{cocoindex-0.1.36.dist-info → cocoindex-0.1.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.36
+Version: 0.1.38
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
@@ -148,12 +148,13 @@ It defines an index flow like this:
 | [Code Embedding](examples/code_embedding) | Index code embeddings for semantic search |
 | [PDF Embedding](examples/pdf_embedding) | Parse PDF and index text embeddings for semantic search |
 | [Manuals LLM Extraction](examples/manuals_llm_extraction) | Extract structured information from a manual using LLM |
+| [Amazon S3 Embedding](examples/amazon_s3_embedding) | Index text documents from Amazon S3 |
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
 | [Product_Taxonomy_Knowledge_Graph](examples/product_taxonomy_knowledge_graph) | Build knowledge graph for product recommendations | 
-| [Image Search with Vision API](examples/image_search_example) | Generates detailed captions for images using a vision model, embeds them, enables semantic search via FastAPI and served on a React frontend.|
+| [Image Search with Vision API](examples/image_search_example) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 
 More coming and stay tuned 👀!
 

cocoindex-0.1.38.dist-info/RECORD

@@ -0,0 +1,25 @@
+cocoindex-0.1.38.dist-info/METADATA,sha256=d1qjWo7V_MjeUiBe4zIOyqCyg8gM7-DaUDbKPSIf3_Q,9793
+cocoindex-0.1.38.dist-info/WHEEL,sha256=p_tvkyHH2UmMBrR2Gemb1ahXJMM2SXUIsCLrWZgJvB8,104
+cocoindex-0.1.38.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex/__init__.py,sha256=LpB0VjGvkD1beio8R9RCT6PI3eU0keV-3sBL45fHTQE,690
+cocoindex/_engine.cpython-311-darwin.so,sha256=Al34X1flIWqzdu6cu_UjO34tUA0SHYm4xuSKymvq2-0,56753664
+cocoindex/auth_registry.py,sha256=NsALZ3SKsDG9cPdrlTlalIqUvgbgFOaFGAbWJNedtJE,692
+cocoindex/cli.py,sha256=Ac3ybnQW-HGVGJeUwIOHd1qhjs0KC5wCsemWuyouEfU,8999
+cocoindex/convert.py,sha256=tRY-QBeeFMFwCYiRk7a0_tuDqopw8iqBpg_Aswcq9JQ,6864
+cocoindex/flow.py,sha256=r1GnRIthmkniJEsAxGsYlDXbcf7ydMwZy1qJEKzwtqc,23814
+cocoindex/functions.py,sha256=F79dNmGE127LaU67kF5Oqtf_tIzebFQH7MkyceMX4-s,1830
+cocoindex/index.py,sha256=LssEOuZi6AqhwKtZM3QFeQpa9T-0ELi8G5DsrYKECvc,534
+cocoindex/lib.py,sha256=OqTMuOHicdyX9PRA7fmTzznK8HZMrzxpUDbqxAEF--Q,2383
+cocoindex/llm.py,sha256=_3rtahuKcqcEHPkFSwhXOSrekZyGxVApPoYtlU_chcA,348
+cocoindex/op.py,sha256=OGYRYl7gPa7X7iSU30iTrCzvqRBu7jQqfvN4vjG__dA,10730
+cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cocoindex/query.py,sha256=8_3Lb_EVjZtl2ZyJNZGX16LoKXEd-PL8OjY-zs9GQeA,3205
+cocoindex/runtime.py,sha256=jqRnWkkIlAhE04gi4y0Y5bzuq9FX4j0aVNU-nengLJk,980
+cocoindex/setting.py,sha256=AaIMclEktbBgK7Cks2D8LfS1cskf8UUcbSb6UBLdoSs,3260
+cocoindex/setup.py,sha256=ErNtX08NfFOFKehp5qGUvCx8Wiz9f3gmzvfBhAqrQyI,745
+cocoindex/sources.py,sha256=7lpwYLsFCRfbURKf79Vu0JZZoXjAYY0DxNHzUb-VHBY,1327
+cocoindex/storages.py,sha256=MFMsfyOCYMggTWeWrOi82miqOXQmiUuqq828x5htBr0,2207
+cocoindex/tests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+cocoindex/tests/test_convert.py,sha256=7jc--I3frrg7DB5MPr4JFzE7DSCznJuWyHdlDLQJ_fM,15516
+cocoindex/typing.py,sha256=369ABRtnpbaVSQVIBc2ZDutXW8jUmncvNJd9CHEWT3Q,8962
+cocoindex-0.1.38.dist-info/RECORD,,

{cocoindex-0.1.36.dist-info → cocoindex-0.1.38.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.8.4)
+Generator: maturin (1.8.6)
 Root-Is-Purelib: false
 Tag: cp311-cp311-macosx_11_0_arm64

cocoindex-0.1.36.dist-info/RECORD

@@ -1,25 +0,0 @@
-cocoindex-0.1.36.dist-info/METADATA,sha256=YH6UMfZ-m_ako8_9VXinagNN6kD6iVEh94Pv9Ua-C3U,9686
-cocoindex-0.1.36.dist-info/WHEEL,sha256=M0oGXcMDUVEBxvyDRZ1SJRlU2WxAfG7DBwXO4GUZt1Q,104
-cocoindex-0.1.36.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-cocoindex/__init__.py,sha256=LpB0VjGvkD1beio8R9RCT6PI3eU0keV-3sBL45fHTQE,690
-cocoindex/_engine.cpython-311-darwin.so,sha256=OD7YGKys5qje2tDaT8QmDRRSkLAPe7hJeQa5SGqhcC0,49748544
-cocoindex/auth_registry.py,sha256=NsALZ3SKsDG9cPdrlTlalIqUvgbgFOaFGAbWJNedtJE,692
-cocoindex/cli.py,sha256=QdZjgnABuDQfy6JiAxeAJiQMI5FNT9FQGLiYAUtLMw8,8923
-cocoindex/convert.py,sha256=mBUTa_Ag39_ut-yE_jc1wqS3zLjtOm6QKet-bqJ-RWc,5947
-cocoindex/flow.py,sha256=MZZ0Uf0ObAzR1yIjUecRgA-U0t__95eoLBK_DxwwLnk,23375
-cocoindex/functions.py,sha256=F79dNmGE127LaU67kF5Oqtf_tIzebFQH7MkyceMX4-s,1830
-cocoindex/index.py,sha256=LssEOuZi6AqhwKtZM3QFeQpa9T-0ELi8G5DsrYKECvc,534
-cocoindex/lib.py,sha256=812GB8Z-2PyjG73Odvw5jtNBLnoeU9aOh9s2ZnETKa8,2329
-cocoindex/llm.py,sha256=_3rtahuKcqcEHPkFSwhXOSrekZyGxVApPoYtlU_chcA,348
-cocoindex/op.py,sha256=OGYRYl7gPa7X7iSU30iTrCzvqRBu7jQqfvN4vjG__dA,10730
-cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cocoindex/query.py,sha256=8_3Lb_EVjZtl2ZyJNZGX16LoKXEd-PL8OjY-zs9GQeA,3205
-cocoindex/runtime.py,sha256=jqRnWkkIlAhE04gi4y0Y5bzuq9FX4j0aVNU-nengLJk,980
-cocoindex/setting.py,sha256=pms1blwlXIOqZIpye-rfiwzqYUCAC8oEL7mQM5A160g,2356
-cocoindex/setup.py,sha256=AQLbtBLuJX066IANS7BGp20246mAGQ_4Z0W6MVJcQzY,481
-cocoindex/sources.py,sha256=wZFU8lwSXjyofJR-syySH9fTyPnBlAPJ6-1hQNX8fGA,936
-cocoindex/storages.py,sha256=MFMsfyOCYMggTWeWrOi82miqOXQmiUuqq828x5htBr0,2207
-cocoindex/tests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-cocoindex/tests/test_convert.py,sha256=WPRKp0jv_uSEM81RGWEAmsax-J-FtXt90mZ0yEnvGLs,11236
-cocoindex/typing.py,sha256=BI2vPw4Iu4S3aznNJQrfM2LZU_weGYASTXF1W3ZWh_Y,8568
-cocoindex-0.1.36.dist-info/RECORD,,