cocoindex 0.2.0__cp313-cp313-win_amd64.whl → 0.2.2__cp313-cp313-win_amd64.whl

cocoindex/flow.py

@@ -120,9 +120,10 @@ def _transform_helper(
     else:
         raise ValueError("transform() can only be called on a CocoIndex function")
 
-    return _create_data_slice(
-        flow_builder_state,
-        lambda target_scope, name: flow_builder_state.engine_flow_builder.transform(
+    def _create_data_slice_inner(
+        target_scope: _engine.DataScopeRef | None, name: str | None
+    ) -> _engine.DataSlice:
+        result = flow_builder_state.engine_flow_builder.transform(
             kind,
             dump_engine_object(spec),
             transform_args,
@@ -130,7 +131,12 @@ def _transform_helper(
             flow_builder_state.field_name_builder.build_name(
                 name, prefix=_to_snake_case(_spec_kind(fn_spec)) + "_"
             ),
-        ),
+        )
+        return result
+
+    return _create_data_slice(
+        flow_builder_state,
+        _create_data_slice_inner,
         name,
     )
 
@@ -166,6 +172,7 @@ class _DataSliceState:
     def engine_data_slice(self) -> _engine.DataSlice:
         """
         Get the internal DataSlice.
+        This can be blocking.
         """
         if self._lazy_lock is None:
             if self._data_slice is None:
@@ -179,6 +186,13 @@ class _DataSliceState:
                     self._data_slice = self._data_slice_creator(None)
                 return self._data_slice
 
+    async def engine_data_slice_async(self) -> _engine.DataSlice:
+        """
+        Get the internal DataSlice.
+        This can be blocking.
+        """
+        return await asyncio.to_thread(lambda: self.engine_data_slice)
+
     def attach_to_scope(self, scope: _engine.DataScopeRef, field_name: str) -> None:
         """
         Attach the current data slice (if not yet attached) to the given scope.
@@ -795,9 +809,8 @@ class Flow:
         """
         Setup persistent backends of the flow. The async version.
         """
-        await make_setup_bundle([self]).describe_and_apply_async(
-            report_to_stdout=report_to_stdout
-        )
+        bundle = await make_setup_bundle_async([self])
+        await bundle.describe_and_apply_async(report_to_stdout=report_to_stdout)
 
     def drop(self, report_to_stdout: bool = False) -> None:
         """
@@ -814,9 +827,8 @@ class Flow:
         """
         Drop persistent backends of the flow. The async version.
         """
-        await make_drop_bundle([self]).describe_and_apply_async(
-            report_to_stdout=report_to_stdout
-        )
+        bundle = await make_drop_bundle_async([self])
+        await bundle.describe_and_apply_async(report_to_stdout=report_to_stdout)
 
     def close(self) -> None:
         """
@@ -1071,19 +1083,16 @@ class TransformFlow(Generic[T]):
                 _DataSliceState(flow_builder_state, engine_ds)
             )
 
-        output = self._flow_fn(**kwargs)
-        flow_builder_state.engine_flow_builder.set_direct_output(
-            _data_slice_state(output).engine_data_slice
-        )
+        output = await asyncio.to_thread(lambda: self._flow_fn(**kwargs))
+        output_data_slice = await _data_slice_state(output).engine_data_slice_async()
+
+        flow_builder_state.engine_flow_builder.set_direct_output(output_data_slice)
         engine_flow = (
             await flow_builder_state.engine_flow_builder.build_transient_flow_async(
                 execution_context.event_loop
             )
         )
-
-        engine_return_type = (
-            _data_slice_state(output).engine_data_slice.data_type().schema()
-        )
+        engine_return_type = output_data_slice.data_type().schema()
         python_return_type: type[T] | None = _get_data_slice_annotation_type(
             inspect.signature(self._flow_fn).return_annotation
         )
@@ -1142,28 +1151,42 @@ def transform_flow() -> Callable[[Callable[..., DataSlice[T]]], TransformFlow[T]
     return _transform_flow_wrapper
 
 
-def make_setup_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
+async def make_setup_bundle_async(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
     """
     Make a bundle to setup flows with the given names.
     """
     full_names = []
     for fl in flow_iter:
-        fl.internal_flow()
+        await fl.internal_flow_async()
         full_names.append(fl.full_name)
     return SetupChangeBundle(_engine.make_setup_bundle(full_names))
 
 
-def make_drop_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
+def make_setup_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
+    """
+    Make a bundle to setup flows with the given names.
+    """
+    return execution_context.run(make_setup_bundle_async(flow_iter))
+
+
+async def make_drop_bundle_async(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
     """
     Make a bundle to drop flows with the given names.
     """
     full_names = []
     for fl in flow_iter:
-        fl.internal_flow()
+        await fl.internal_flow_async()
         full_names.append(fl.full_name)
     return SetupChangeBundle(_engine.make_drop_bundle(full_names))
 
 
+def make_drop_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
+    """
+    Make a bundle to drop flows with the given names.
+    """
+    return execution_context.run(make_drop_bundle_async(flow_iter))
+
+
 def setup_all_flows(report_to_stdout: bool = False) -> None:
     """
     Setup all flows registered in the current process.

cocoindex/setting.py

@@ -44,8 +44,8 @@ class DatabaseConnectionSpec:
     url: str
     user: str | None = None
     password: str | None = None
-    max_connections: int = 64
-    min_connections: int = 16
+    max_connections: int = 25
+    min_connections: int = 5
 
 
 @dataclass

cocoindex/subprocess_exec.py

@@ -22,6 +22,7 @@ import time
 from .user_app_loader import load_user_app
 from .runtime import execution_context
 import logging
+import multiprocessing as mp
 
 WATCHDOG_INTERVAL_SECONDS = 10.0
 
@@ -43,6 +44,7 @@ def _get_pool() -> ProcessPoolExecutor:
                 max_workers=1,
                 initializer=_subprocess_init,
                 initargs=(_user_apps, os.getpid()),
+                mp_context=mp.get_context("spawn"),
             )
         return _pool
 
@@ -69,6 +71,7 @@ def _restart_pool(old_pool: ProcessPoolExecutor | None = None) -> None:
             max_workers=1,
             initializer=_subprocess_init,
             initargs=(_user_apps, os.getpid()),
+            mp_context=mp.get_context("spawn"),
         )
         if prev_pool is not None:
             # Best-effort shutdown of previous pool; letting exceptions bubble up
@@ -124,8 +127,19 @@ def _start_parent_watchdog(
 
 def _subprocess_init(user_apps: list[str], parent_pid: int) -> None:
     _start_parent_watchdog(parent_pid)
+
+    # In case any user app is already in this subprocess, e.g. the subprocess is forked, we need to avoid loading it again.
+    with _pool_lock:
+        already_loaded_apps = set(_user_apps)
+
+    loaded_apps = []
     for app_target in user_apps:
-        load_user_app(app_target)
+        if app_target not in already_loaded_apps:
+            load_user_app(app_target)
+            loaded_apps.append(app_target)
+
+    with _pool_lock:
+        _user_apps.extend(loaded_apps)
 
 
 class _OnceResult:

cocoindex/tests/test_transform_flow.py

@@ -166,6 +166,25 @@ class GpuAppendSuffixExecutor:
         return f"{text}{self.spec.suffix}"
 
 
+class GpuAppendSuffixWithAnalyzePrepare(cocoindex.op.FunctionSpec):
+    suffix: str
+
+
+@cocoindex.op.executor_class(gpu=True)
+class GpuAppendSuffixWithAnalyzePrepareExecutor:
+    spec: GpuAppendSuffixWithAnalyzePrepare
+    suffix: str
+
+    def analyze(self) -> Any:
+        return str
+
+    def prepare(self) -> None:
+        self.suffix = self.spec.suffix
+
+    def __call__(self, text: str) -> str:
+        return f"{text}{self.suffix}"
+
+
 def test_gpu_function() -> None:
     @cocoindex.transform_flow()
     def transform_flow(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[str]:
@@ -174,3 +193,15 @@ def test_gpu_function() -> None:
     result = transform_flow.eval("Hello")
     expected = "Hello world!"
     assert result == expected, f"Expected {expected}, got {result}"
+
+    @cocoindex.transform_flow()
+    def transform_flow_with_analyze_prepare(
+        text: cocoindex.DataSlice[str],
+    ) -> cocoindex.DataSlice[str]:
+        return text.transform(gpu_append_world).transform(
+            GpuAppendSuffixWithAnalyzePrepare(suffix="!!")
+        )
+
+    result = transform_flow_with_analyze_prepare.eval("Hello")
+    expected = "Hello world!!"
+    assert result == expected, f"Expected {expected}, got {result}"

{cocoindex-0.2.0.dist-info → cocoindex-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.0
+Version: 0.2.2
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -84,7 +84,7 @@ CocoIndex makes it effortless to transform data with AI, and keep source data an
 </br>
 
 <p align="center">
-<img width="4187" height="1883" alt="CocoIndex Features" src="https://github.com/user-attachments/assets/6147673f-0daf-4313-a0c7-7c4205ba2e31" />
+<img alt="CocoIndex Features" src="https://cocoindex.io/images/venn2.svg" />
 </p>
 
 </br>

{cocoindex-0.2.0.dist-info → cocoindex-0.2.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
-cocoindex-0.2.0.dist-info/METADATA,sha256=u3xlaek3vJchi1ki3C0wm03J9SNiifOMK6WFmquDvwI,12403
-cocoindex-0.2.0.dist-info/WHEEL,sha256=K7foeVF-x_RZTycPKa1uE1HH2bAWe3AiJbihrXn5Hhc,96
-cocoindex-0.2.0.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.2.0.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+cocoindex-0.2.2.dist-info/METADATA,sha256=fTrnBYaHaBQXAZOTR6165dE9McaS-ePB_H6SsF41dUs,12334
+cocoindex-0.2.2.dist-info/WHEEL,sha256=K7foeVF-x_RZTycPKa1uE1HH2bAWe3AiJbihrXn5Hhc,96
+cocoindex-0.2.2.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.2.2.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 cocoindex/__init__.py,sha256=5zwuS_X-n7QAE5uBSufqXp77OW8KVVD8E5t_6koDLRc,2293
-cocoindex/_engine.cp313-win_amd64.pyd,sha256=HQPzr9nNVz0dlpAaXdHRKDpMcdHlNj4J_gwfonnvPFw,71612928
+cocoindex/_engine.cp313-win_amd64.pyd,sha256=5XNMzSZdXaFSYrKeVORWQAM_u93uVuSImkW7SEeQl00,71629824
 cocoindex/auth_registry.py,sha256=Qq1IVZb-7K4luRrQSDlOPbISnGEZ4kIDsrCU8H2ARw0,1529
 cocoindex/cli.py,sha256=VgeVgO5z2l6tv7XWS3q-sRZ8cYe9r7izUYUR3NfdgJA,21372
 cocoindex/convert.py,sha256=k1NT3VoERVS2HV1rLDFyCk6rG1YgHXDnIvrI-lqFdUE,22681
-cocoindex/flow.py,sha256=6AadyuaQsujkjV1zcXM-DiYr29uVdGXII4tIuNGDo_k,37440
+cocoindex/flow.py,sha256=YGSASrMNtQzX9S42XfOpspEBFayj16OiD3q5bTz1y98,38409
 cocoindex/functions.py,sha256=CtiwTVW6g4BtO5_EvVcij7Si4Bx-axnM1hsdU43aM4g,12617
 cocoindex/index.py,sha256=GrqTm1rLwICQ8hadtNvJAxVg7GWMvtMmFcbiNtNzmP0,569
 cocoindex/lib.py,sha256=cZosix4nwROvod4QJOwCzrm6U1CVy_wKMMk7sDDG_Z0,849
@@ -15,19 +15,19 @@ cocoindex/llm.py,sha256=JM5EPup7FZojynCI0rg4gnMBYmdPZpaHJbVBz3f7BAI,897
 cocoindex/op.py,sha256=1uOwE1zzMs7FrMSpQMJPUI8YtnXXk3-tlEJ_ahPFq5A,22884
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=6mE-jR1Kh5c4GETDvBgwjXZq69TK5rh1qNpaseRDZnw,1117
-cocoindex/setting.py,sha256=048VfKRly9TzLElJlL01tUxySGtRbsc9m68yIHvMzMU,5464
+cocoindex/setting.py,sha256=obq1EiFlSJvidQZPVVC9V5ZUZAzYnyYcSm_6XUVDkUE,5463
 cocoindex/setup.py,sha256=KbJvmeFu0NbeoH-5iDmHZP86f26HIId8kHmGUNZAePI,3160
 cocoindex/sources.py,sha256=mVZhyVTyHzFy7z3lceYqgguygADQkIL42nYK6zKSKKQ,2868
-cocoindex/subprocess_exec.py,sha256=KDV7xdFUhKYqHZdiJqirNTl7DbJQUXLvga0Q1urE6XA,8143
+cocoindex/subprocess_exec.py,sha256=ase_F4ivmyShCIJcygRl9Uz8BKthpOLq-TUfyqrUB9A,8658
 cocoindex/targets.py,sha256=7FfG9kuEf5KTXtLwXMFaPFIut3PsIbpb3XIEjjeF7Bg,2931
 cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/tests/test_convert.py,sha256=kngLt9p2BXo3D7_OZNnnSS1w_Ew8966IuaDpHcfd9Go,51750
 cocoindex/tests/test_optional_database.py,sha256=dnzmTgaJf37D3q8fQsjP5UDER6FYETaUokDnFBMLtIk,8755
-cocoindex/tests/test_transform_flow.py,sha256=DDtxENZHh_2mt0vDU4T1Oaq-jVLJDwaKpkMg83_9NfE,5324
+cocoindex/tests/test_transform_flow.py,sha256=DxM-7_kWeU-QzOpH77Vd5Jehbbq00xCBBgRK7mRn0kI,6237
 cocoindex/tests/test_typing.py,sha256=WqR1M11dSVYXZj1DnXnOvwH6VNQpiv5lCEO-FqA_by0,12820
 cocoindex/tests/test_validation.py,sha256=I4wr8lAMAjmy5xgG5N_OJKveXt8XIa96MsQTXhw5AnA,4677
 cocoindex/typing.py,sha256=bkKcp4G_xQODwQn92sfeNi-kXSWnbtlwTpFbrMGCnj4,14673
 cocoindex/user_app_loader.py,sha256=jKNyCq5Osl4dMevlDNloGuwCfDscxw5o0m9_OqrHDN8,1965
 cocoindex/utils.py,sha256=U3W39zD2uZpXX8v84tJD7sRmbC5ar3z_ljAP1cJrYXI,618
 cocoindex/validation.py,sha256=4ZjsW-SZT8X_TEEhEE6QG6D-8Oq_TkPAhTqP0mdFYSE,3194
-cocoindex-0.2.0.dist-info/RECORD,,
+cocoindex-0.2.2.dist-info/RECORD,,