cocoindex 0.1.83__cp312-cp312-win_amd64.whl → 0.2.1__cp312-cp312-win_amd64.whl

cocoindex/convert.py

@@ -14,7 +14,6 @@ from typing import Any, Callable, Mapping, Type, get_origin
 import numpy as np
 
 from .typing import (
-    KEY_FIELD_NAME,
     TABLE_TYPES,
     AnalyzedAnyType,
     AnalyzedBasicType,
@@ -28,7 +27,6 @@ from .typing import (
     encode_enriched_type,
     is_namedtuple_type,
     is_numpy_number_type,
-    is_struct_type,
 )
 
 
@@ -88,38 +86,35 @@ def make_engine_value_encoder(type_info: AnalyzedTypeInfo) -> Callable[[Any], An
 
             return encode_struct_list
 
-    if isinstance(variant, AnalyzedDictType):
-        if not variant.value_type:
-            return lambda value: value
+        # Otherwise it's a vector, falling into basic type in the engine.
 
+    if isinstance(variant, AnalyzedDictType):
         value_type_info = analyze_type_info(variant.value_type)
-        if isinstance(value_type_info.variant, AnalyzedStructType):
-
-            def encode_struct_dict(value: Any) -> Any:
-                if not isinstance(value, dict):
-                    return value
-                if not value:
-                    return []
-
-                sample_key, sample_val = next(iter(value.items()))
-                key_type, val_type = type(sample_key), type(sample_val)
-
-                # Handle KTable case
-                if value and is_struct_type(val_type):
-                    key_encoder = (
-                        make_engine_value_encoder(analyze_type_info(key_type))
-                        if is_struct_type(key_type)
-                        else make_engine_value_encoder(ANY_TYPE_INFO)
-                    )
-                    value_encoder = make_engine_value_encoder(
-                        analyze_type_info(val_type)
-                    )
-                    return [
-                        [key_encoder(k)] + value_encoder(v) for k, v in value.items()
-                    ]
-                return {key_encoder(k): value_encoder(v) for k, v in value.items()}
+        if not isinstance(value_type_info.variant, AnalyzedStructType):
+            raise ValueError(
+                f"Value type for dict is required to be a struct (e.g. dataclass or NamedTuple), got {variant.value_type}. "
+                f"If you want a free-formed dict, use `cocoindex.Json` instead."
+            )
+        value_encoder = make_engine_value_encoder(value_type_info)
+
+        key_type_info = analyze_type_info(variant.key_type)
+        key_encoder = make_engine_value_encoder(key_type_info)
+        if isinstance(key_type_info.variant, AnalyzedBasicType):
 
-            return encode_struct_dict
+            def encode_row(k: Any, v: Any) -> Any:
+                return [key_encoder(k)] + value_encoder(v)
+
+        else:
+
+            def encode_row(k: Any, v: Any) -> Any:
+                return key_encoder(k) + value_encoder(v)
+
+        def encode_struct_dict(value: Any) -> Any:
+            if not value:
+                return []
+            return [encode_row(k, v) for k, v in value.items()]
+
+        return encode_struct_dict
 
     if isinstance(variant, AnalyzedStructType):
         struct_type = variant.struct_type
@@ -132,8 +127,8 @@ def make_engine_value_encoder(type_info: AnalyzedTypeInfo) -> Callable[[Any], An
             field_names = [f.name for f in fields]
 
             def encode_dataclass(value: Any) -> Any:
-                if not dataclasses.is_dataclass(value):
-                    return value
+                if value is None:
+                    return None
                 return [
                     encoder(getattr(value, name))
                     for encoder, name in zip(field_encoders, field_names)
@@ -154,8 +149,8 @@ def make_engine_value_encoder(type_info: AnalyzedTypeInfo) -> Callable[[Any], An
             ]
 
             def encode_namedtuple(value: Any) -> Any:
-                if not is_namedtuple_type(type(value)):
-                    return value
+                if value is None:
+                    return None
                 return [
                     encoder(getattr(value, name))
                     for encoder, name in zip(field_encoders, field_names)
@@ -248,25 +243,47 @@ def make_engine_value_decoder(
                         f"declared `{dst_type_info.core_type}`, a dict type expected"
                     )
 
-                key_field_schema = engine_fields_schema[0]
-                field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-                key_decoder = make_engine_value_decoder(
-                    field_path,
-                    key_field_schema["type"],
-                    analyze_type_info(key_type),
-                    for_key=True,
-                )
-                field_path.pop()
+                num_key_parts = src_type.get("num_key_parts", 1)
+                key_type_info = analyze_type_info(key_type)
+                key_decoder: Callable[..., Any] | None = None
+                if (
+                    isinstance(
+                        key_type_info.variant, (AnalyzedBasicType, AnalyzedAnyType)
+                    )
+                    and num_key_parts == 1
+                ):
+                    single_key_decoder = make_engine_value_decoder(
+                        field_path,
+                        engine_fields_schema[0]["type"],
+                        key_type_info,
+                        for_key=True,
+                    )
+
+                    def key_decoder(value: list[Any]) -> Any:
+                        return single_key_decoder(value[0])
+
+                else:
+                    key_decoder = make_engine_struct_decoder(
+                        field_path,
+                        engine_fields_schema[0:num_key_parts],
+                        key_type_info,
+                        for_key=True,
+                    )
                 value_decoder = make_engine_struct_decoder(
                     field_path,
-                    engine_fields_schema[1:],
+                    engine_fields_schema[num_key_parts:],
                     analyze_type_info(value_type),
                 )
 
                 def decode(value: Any) -> Any | None:
                     if value is None:
                         return None
-                    return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
+                    return {
+                        key_decoder(v[0:num_key_parts]): value_decoder(
+                            v[num_key_parts:]
+                        )
+                        for v in value
+                    }
 
         return decode
 

cocoindex/setting.py

@@ -44,8 +44,8 @@ class DatabaseConnectionSpec:
     url: str
     user: str | None = None
     password: str | None = None
-    max_connections: int = 64
-    min_connections: int = 16
+    max_connections: int = 25
+    min_connections: int = 5
 
 
 @dataclass

cocoindex/subprocess_exec.py

@@ -22,6 +22,7 @@ import time
 from .user_app_loader import load_user_app
 from .runtime import execution_context
 import logging
+import multiprocessing as mp
 
 WATCHDOG_INTERVAL_SECONDS = 10.0
 
@@ -43,6 +44,7 @@ def _get_pool() -> ProcessPoolExecutor:
                 max_workers=1,
                 initializer=_subprocess_init,
                 initargs=(_user_apps, os.getpid()),
+                mp_context=mp.get_context("spawn"),
             )
         return _pool
 
@@ -69,6 +71,7 @@ def _restart_pool(old_pool: ProcessPoolExecutor | None = None) -> None:
             max_workers=1,
             initializer=_subprocess_init,
             initargs=(_user_apps, os.getpid()),
+            mp_context=mp.get_context("spawn"),
         )
         if prev_pool is not None:
             # Best-effort shutdown of previous pool; letting exceptions bubble up
@@ -124,8 +127,19 @@ def _start_parent_watchdog(
 
 def _subprocess_init(user_apps: list[str], parent_pid: int) -> None:
     _start_parent_watchdog(parent_pid)
+
+    # In case any user app is already in this subprocess, e.g. the subprocess is forked, we need to avoid loading it again.
+    with _pool_lock:
+        already_loaded_apps = set(_user_apps)
+
+    loaded_apps = []
     for app_target in user_apps:
-        load_user_app(app_target)
+        if app_target not in already_loaded_apps:
+            load_user_app(app_target)
+            loaded_apps.append(app_target)
+
+    with _pool_lock:
+        _user_apps.extend(loaded_apps)
 
 
 class _OnceResult:

cocoindex/typing.py

@@ -330,35 +330,50 @@ def analyze_type_info(t: Any) -> AnalyzedTypeInfo:
 
 def _encode_struct_schema(
     struct_type: type, key_type: type | None = None
-) -> dict[str, Any]:
+) -> tuple[dict[str, Any], int | None]:
     fields = []
 
-    def add_field(name: str, t: Any) -> None:
+    def add_field(name: str, analyzed_type: AnalyzedTypeInfo) -> None:
         try:
-            type_info = encode_enriched_type_info(analyze_type_info(t))
+            type_info = encode_enriched_type_info(analyzed_type)
         except ValueError as e:
             e.add_note(
                 f"Failed to encode annotation for field - "
-                f"{struct_type.__name__}.{name}: {t}"
+                f"{struct_type.__name__}.{name}: {analyzed_type.core_type}"
             )
             raise
         type_info["name"] = name
         fields.append(type_info)
 
+    def add_fields_from_struct(struct_type: type) -> None:
+        if dataclasses.is_dataclass(struct_type):
+            for field in dataclasses.fields(struct_type):
+                add_field(field.name, analyze_type_info(field.type))
+        elif is_namedtuple_type(struct_type):
+            for name, field_type in struct_type.__annotations__.items():
+                add_field(name, analyze_type_info(field_type))
+        else:
+            raise ValueError(f"Unsupported struct type: {struct_type}")
+
+    result: dict[str, Any] = {}
+    num_key_parts = None
     if key_type is not None:
-        add_field(KEY_FIELD_NAME, key_type)
+        key_type_info = analyze_type_info(key_type)
+        if isinstance(key_type_info.variant, AnalyzedBasicType):
+            add_field(KEY_FIELD_NAME, key_type_info)
+            num_key_parts = 1
+        elif isinstance(key_type_info.variant, AnalyzedStructType):
+            add_fields_from_struct(key_type_info.variant.struct_type)
+            num_key_parts = len(fields)
+        else:
+            raise ValueError(f"Unsupported key type: {key_type}")
 
-    if dataclasses.is_dataclass(struct_type):
-        for field in dataclasses.fields(struct_type):
-            add_field(field.name, field.type)
-    elif is_namedtuple_type(struct_type):
-        for name, field_type in struct_type.__annotations__.items():
-            add_field(name, field_type)
+    add_fields_from_struct(struct_type)
 
-    result: dict[str, Any] = {"fields": fields}
+    result["fields"] = fields
     if doc := inspect.getdoc(struct_type):
         result["description"] = doc
-    return result
+    return result, num_key_parts
 
 
 def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
@@ -374,7 +389,7 @@ def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
         return {"kind": variant.kind}
 
     if isinstance(variant, AnalyzedStructType):
-        encoded_type = _encode_struct_schema(variant.struct_type)
+        encoded_type, _ = _encode_struct_schema(variant.struct_type)
         encoded_type["kind"] = "Struct"
         return encoded_type
 
@@ -384,10 +399,8 @@ def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
         if isinstance(elem_type_info.variant, AnalyzedStructType):
             if variant.vector_info is not None:
                 raise ValueError("LTable type must not have a vector info")
-            return {
-                "kind": "LTable",
-                "row": _encode_struct_schema(elem_type_info.variant.struct_type),
-            }
+            row_type, _ = _encode_struct_schema(elem_type_info.variant.struct_type)
+            return {"kind": "LTable", "row": row_type}
         else:
             vector_info = variant.vector_info
             return {
@@ -402,12 +415,14 @@ def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
             raise ValueError(
                 f"KTable value must have a Struct type, got {value_type_info.core_type}"
             )
+        row_type, num_key_parts = _encode_struct_schema(
+            value_type_info.variant.struct_type,
+            variant.key_type,
+        )
         return {
             "kind": "KTable",
-            "row": _encode_struct_schema(
-                value_type_info.variant.struct_type,
-                variant.key_type,
-            ),
+            "row": row_type,
+            "num_key_parts": num_key_parts,
         }
 
     if isinstance(variant, AnalyzedUnionType):

{cocoindex-0.1.83.dist-info → cocoindex-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.83
+Version: 0.2.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.1.83.dist-info → cocoindex-0.2.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-cocoindex-0.1.83.dist-info/METADATA,sha256=hdRQSWNsnB6dUFa1F8ikl2fE9dnIVNv288ldby5XbNU,12404
-cocoindex-0.1.83.dist-info/WHEEL,sha256=HF3aUMilrtO42xS_fBzOVaPE8OtiNjP_RotLatII7HM,96
-cocoindex-0.1.83.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.1.83.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+cocoindex-0.2.1.dist-info/METADATA,sha256=m9RVQtYVruDkXdRHm0GdTqZYRSnNF5R_bBIcUqZoIC8,12403
+cocoindex-0.2.1.dist-info/WHEEL,sha256=4hYCffp0RsSVQAuv2PMtXQ9QS7YSHeZi4PrSg-wi2q0,96
+cocoindex-0.2.1.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.2.1.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 cocoindex/__init__.py,sha256=5zwuS_X-n7QAE5uBSufqXp77OW8KVVD8E5t_6koDLRc,2293
-cocoindex/_engine.cp312-win_amd64.pyd,sha256=FbhWKFdn6QGQZ0dsgXNsuYKWubdEKro9GQU4yqxtjcg,71575552
+cocoindex/_engine.cp312-win_amd64.pyd,sha256=l1eJWMLVZz7ihhGO5iqFvnK8sLGzjI3vOIZRBhNtZzc,71615488
 cocoindex/auth_registry.py,sha256=Qq1IVZb-7K4luRrQSDlOPbISnGEZ4kIDsrCU8H2ARw0,1529
 cocoindex/cli.py,sha256=VgeVgO5z2l6tv7XWS3q-sRZ8cYe9r7izUYUR3NfdgJA,21372
-cocoindex/convert.py,sha256=IqrOswncP99pOYecCtOKpp1pQEQHyxE13KKfLGSN9ZQ,22066
+cocoindex/convert.py,sha256=k1NT3VoERVS2HV1rLDFyCk6rG1YgHXDnIvrI-lqFdUE,22681
 cocoindex/flow.py,sha256=6AadyuaQsujkjV1zcXM-DiYr29uVdGXII4tIuNGDo_k,37440
 cocoindex/functions.py,sha256=CtiwTVW6g4BtO5_EvVcij7Si4Bx-axnM1hsdU43aM4g,12617
 cocoindex/index.py,sha256=GrqTm1rLwICQ8hadtNvJAxVg7GWMvtMmFcbiNtNzmP0,569
@@ -15,10 +15,10 @@ cocoindex/llm.py,sha256=JM5EPup7FZojynCI0rg4gnMBYmdPZpaHJbVBz3f7BAI,897
 cocoindex/op.py,sha256=1uOwE1zzMs7FrMSpQMJPUI8YtnXXk3-tlEJ_ahPFq5A,22884
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=6mE-jR1Kh5c4GETDvBgwjXZq69TK5rh1qNpaseRDZnw,1117
-cocoindex/setting.py,sha256=048VfKRly9TzLElJlL01tUxySGtRbsc9m68yIHvMzMU,5464
+cocoindex/setting.py,sha256=obq1EiFlSJvidQZPVVC9V5ZUZAzYnyYcSm_6XUVDkUE,5463
 cocoindex/setup.py,sha256=KbJvmeFu0NbeoH-5iDmHZP86f26HIId8kHmGUNZAePI,3160
 cocoindex/sources.py,sha256=mVZhyVTyHzFy7z3lceYqgguygADQkIL42nYK6zKSKKQ,2868
-cocoindex/subprocess_exec.py,sha256=KDV7xdFUhKYqHZdiJqirNTl7DbJQUXLvga0Q1urE6XA,8143
+cocoindex/subprocess_exec.py,sha256=ase_F4ivmyShCIJcygRl9Uz8BKthpOLq-TUfyqrUB9A,8658
 cocoindex/targets.py,sha256=7FfG9kuEf5KTXtLwXMFaPFIut3PsIbpb3XIEjjeF7Bg,2931
 cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/tests/test_convert.py,sha256=kngLt9p2BXo3D7_OZNnnSS1w_Ew8966IuaDpHcfd9Go,51750
@@ -26,8 +26,8 @@ cocoindex/tests/test_optional_database.py,sha256=dnzmTgaJf37D3q8fQsjP5UDER6FYETa
 cocoindex/tests/test_transform_flow.py,sha256=DDtxENZHh_2mt0vDU4T1Oaq-jVLJDwaKpkMg83_9NfE,5324
 cocoindex/tests/test_typing.py,sha256=WqR1M11dSVYXZj1DnXnOvwH6VNQpiv5lCEO-FqA_by0,12820
 cocoindex/tests/test_validation.py,sha256=I4wr8lAMAjmy5xgG5N_OJKveXt8XIa96MsQTXhw5AnA,4677
-cocoindex/typing.py,sha256=gMNJIpGGe-SiXlihDQ-Dw2YdebQvOyG-bWovR-veO6g,13817
+cocoindex/typing.py,sha256=bkKcp4G_xQODwQn92sfeNi-kXSWnbtlwTpFbrMGCnj4,14673
 cocoindex/user_app_loader.py,sha256=jKNyCq5Osl4dMevlDNloGuwCfDscxw5o0m9_OqrHDN8,1965
 cocoindex/utils.py,sha256=U3W39zD2uZpXX8v84tJD7sRmbC5ar3z_ljAP1cJrYXI,618
 cocoindex/validation.py,sha256=4ZjsW-SZT8X_TEEhEE6QG6D-8Oq_TkPAhTqP0mdFYSE,3194
-cocoindex-0.1.83.dist-info/RECORD,,
+cocoindex-0.2.1.dist-info/RECORD,,

{cocoindex-0.1.83.dist-info → cocoindex-0.2.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.9.3)
+Generator: maturin (1.9.4)
 Root-Is-Purelib: false
 Tag: cp312-cp312-win_amd64