cocoindex 0.1.64__cp311-cp311-manylinux_2_28_aarch64.whl → 0.1.65__cp311-cp311-manylinux_2_28_aarch64.whl

cocoindex/__init__.py

@@ -11,6 +11,7 @@ from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
 from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions
+from .flow import add_flow_def, remove_flow
 from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
 from .lib import init, start_server, stop
 from .llm import LlmSpec, LlmApiType
@@ -52,6 +53,8 @@ __all__ = [
     "GeneratedField",
     "FlowLiveUpdater",
     "FlowLiveUpdaterOptions",
+    "add_flow_def",
+    "remove_flow",
     "update_all_flows_async",
     "setup_all_flows",
     "drop_all_flows",

cocoindex/flow.py

@@ -624,7 +624,7 @@ class Flow:
 
     _name: str
     _full_name: str
-    _lazy_engine_flow: Callable[[], _engine.Flow]
+    _lazy_engine_flow: Callable[[], _engine.Flow] | None
 
     def __init__(
         self, name: str, full_name: str, engine_flow_creator: Callable[[], _engine.Flow]
@@ -664,18 +664,18 @@ class Flow:
         return tree
 
     def _get_spec(self, verbose: bool = False) -> _engine.RenderedSpec:
-        return self._lazy_engine_flow().get_spec(
+        return self.internal_flow().get_spec(
             output_mode="verbose" if verbose else "concise"
         )
 
     def _get_schema(self) -> list[tuple[str, str, str]]:
-        return cast(list[tuple[str, str, str]], self._lazy_engine_flow().get_schema())
+        return cast(list[tuple[str, str, str]], self.internal_flow().get_schema())
 
     def __str__(self) -> str:
         return str(self._get_spec())
 
     def __repr__(self) -> str:
-        return repr(self._lazy_engine_flow())
+        return repr(self.internal_flow())
 
     @property
     def name(self) -> str:
@@ -715,12 +715,14 @@ class Flow:
         """
         Evaluate the flow and dump flow outputs to files.
         """
-        return self._lazy_engine_flow().evaluate_and_dump(dump_engine_object(options))
+        return self.internal_flow().evaluate_and_dump(dump_engine_object(options))
 
     def internal_flow(self) -> _engine.Flow:
         """
         Get the engine flow.
         """
+        if self._lazy_engine_flow is None:
+            raise RuntimeError(f"Flow {self.full_name} is already removed")
         return self._lazy_engine_flow()
 
     async def internal_flow_async(self) -> _engine.Flow:
@@ -731,13 +733,13 @@ class Flow:
 
     def setup(self, report_to_stdout: bool = False) -> None:
         """
-        Setup the flow.
+        Setup persistent backends of the flow.
         """
         execution_context.run(self.setup_async(report_to_stdout=report_to_stdout))
 
     async def setup_async(self, report_to_stdout: bool = False) -> None:
         """
-        Setup the flow. The async version.
+        Setup persistent backends of the flow. The async version.
         """
         await make_setup_bundle([self]).describe_and_apply_async(
             report_to_stdout=report_to_stdout
@@ -745,13 +747,18 @@ class Flow:
 
     def drop(self, report_to_stdout: bool = False) -> None:
         """
-        Drop the flow.
+        Drop persistent backends of the flow.
+
+        The current instance is still valid after it's called.
+        For example, you can still call `setup()` after it, to setup the persistent backends again.
+
+        Call `cocoindex.remove_flow()` if you want to remove the flow from the current process.
         """
         execution_context.run(self.drop_async(report_to_stdout=report_to_stdout))
 
     async def drop_async(self, report_to_stdout: bool = False) -> None:
         """
-        Drop the flow. The async version.
+        Drop persistent backends of the flow. The async version.
         """
         await make_drop_bundle([self]).describe_and_apply_async(
             report_to_stdout=report_to_stdout
@@ -805,6 +812,19 @@ def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) ->
     return fl
 
 
+def remove_flow(fl: Flow) -> None:
+    """
+    Remove a flow from the current process to free up resources.
+    After it's called, methods of the flow should no longer be called.
+
+    This will NOT touch the persistent backends of the flow.
+    """
+    _engine.remove_flow_context(fl.full_name)
+    fl._lazy_engine_flow = None  # pylint: disable=protected-access
+    with _flows_lock:
+        del _flows[fl.name]
+
+
 def flow_def(
     name: str | None = None,
 ) -> Callable[[Callable[[FlowBuilder, DataScope], None]], Flow]:

cocoindex/setting.py

@@ -44,12 +44,12 @@ class DatabaseConnectionSpec:
 
 
 @dataclass
-class DefaultExecutionOptions:
-    """Default execution options."""
+class GlobalExecutionOptions:
+    """Global execution options."""
 
-    # The maximum number of concurrent inflight requests.
-    source_max_inflight_rows: int | None = 256
-    source_max_inflight_bytes: int | None = 1024 * 1024 * 1024
+    # The maximum number of concurrent inflight requests, shared among all sources from all flows.
+    source_max_inflight_rows: int | None = None
+    source_max_inflight_bytes: int | None = None
 
 
 def _load_field(
@@ -81,7 +81,7 @@ class Settings:
 
     database: DatabaseConnectionSpec | None = None
     app_namespace: str = ""
-    default_execution_options: DefaultExecutionOptions | None = None
+    global_execution_options: GlobalExecutionOptions | None = None
 
     @classmethod
     def from_env(cls) -> Self:
@@ -110,14 +110,14 @@ class Settings:
             "COCOINDEX_SOURCE_MAX_INFLIGHT_BYTES",
             parse=int,
         )
-        default_execution_options = DefaultExecutionOptions(**exec_kwargs)
+        global_execution_options = GlobalExecutionOptions(**exec_kwargs)
 
         app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", "")
 
         return cls(
             database=database,
             app_namespace=app_namespace,
-            default_execution_options=default_execution_options,
+            global_execution_options=global_execution_options,
         )
 
 

{cocoindex-0.1.64.dist-info → cocoindex-0.1.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.64
+Version: 0.1.65
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.1.64.dist-info → cocoindex-0.1.65.dist-info}/RECORD

@@ -1,13 +1,13 @@
-cocoindex-0.1.64.dist-info/METADATA,sha256=ByEFBlXrGkbESv0twsq2eLycjhTdkQKnE_EzQVIpWgQ,10136
-cocoindex-0.1.64.dist-info/WHEEL,sha256=tYU0YwShGV5a1IBa9K6b40tOXRGPW5TH008p3c7dncU,109
-cocoindex-0.1.64.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.1.64.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-cocoindex/__init__.py,sha256=hDjehCjxRabFCW0RTt00JxnSAJIn9HeVoK4OjFbETsk,1910
-cocoindex/_engine.cpython-311-aarch64-linux-gnu.so,sha256=a6r52tacUnzRqxo_W26iQ7fbR3qdCMb8umOaJ6-TLUs,61287640
+cocoindex-0.1.65.dist-info/METADATA,sha256=q_r8liwwqP1fcDFl0sNkTjty5ZZt6xbGEUFcpOOOR04,10136
+cocoindex-0.1.65.dist-info/WHEEL,sha256=tYU0YwShGV5a1IBa9K6b40tOXRGPW5TH008p3c7dncU,109
+cocoindex-0.1.65.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.1.65.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex/__init__.py,sha256=TXK1FRqfm9DeDD7MycgDEfNutu8utM-dARQSurD98vc,1993
+cocoindex/_engine.cpython-311-aarch64-linux-gnu.so,sha256=16C-8InM-eD_9sopRNgpuuipujEuv4ISa8JGpjlog4A,61386504
 cocoindex/auth_registry.py,sha256=1XqO7ibjmBBd8i11XSJTvTgdz8p1ptW-ZpuSgo_5zzk,716
 cocoindex/cli.py,sha256=-gp639JSyQN6YjnhGqCakIzYoSSqXxQMbxbkcYGP0QY,22359
 cocoindex/convert.py,sha256=qE1Ut_tAwX4wA4WqaWxpyj80-1t6WZ8Oi5_L9Mw5g4k,11393
-cocoindex/flow.py,sha256=Rb3ImrFa-TKYZXZnfcTj4ePUZXqKgJCzZR4OYHP6rlk,34207
+cocoindex/flow.py,sha256=MFPtfJBVTjQ56d7vUn2LvtY30Vg4q2rY6nqvjjJL1kQ,35085
 cocoindex/functions.py,sha256=IBwvdPpGR-S5mk53HvHpT2GVs15MI9wQznxgOdxA0ac,3202
 cocoindex/index.py,sha256=j93B9jEvvLXHtpzKWL88SY6wCGEoPgpsQhEGHlyYGFg,540
 cocoindex/lib.py,sha256=f--9dAYd84CZosbDZqNW0oGbBLsY3dXiUTR1VrfQ_QY,817
@@ -15,7 +15,7 @@ cocoindex/llm.py,sha256=0ri8ZRg9_Zf2gyC5xuQ1Kq6kdZUO8r-A5WLnxit5S_4,448
 cocoindex/op.py,sha256=r_Usx7Jqh49Cck3tsYLx2vLRNUZArkQP_g7bIID6LPU,11809
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=bAdHYaXFWiiUWyAgzmKTeaAaRR0D_AmaqVCIdPO-v00,1056
-cocoindex/setting.py,sha256=FMNjer3YVmVyxLuGt6_DJ6vA1QH1mIo7oH0R51OLnk4,4714
+cocoindex/setting.py,sha256=ADuv7RaWd9k-m3V0Cfy2jmaCt6DupJCviWdOm0CTiVw,4734
 cocoindex/setup.py,sha256=7uIHKN4FOCuoidPXcKyGTrkqpkl9luL49-6UcnMxYzw,3068
 cocoindex/sources.py,sha256=JCnOhv1w4o28e03i7yvo4ESicWYAhckkBg5bQlxNH4U,1330
 cocoindex/targets.py,sha256=Nfh_tpFd1goTnS_cxBjIs4j9zl3Z4Z1JomAQ1dl3Sic,2796
@@ -25,4 +25,4 @@ cocoindex/tests/test_optional_database.py,sha256=snAmkNa6wtOSaxoZE1HgjvL5v_ylitt
 cocoindex/tests/test_typing.py,sha256=t6UCYShcfonTfjBlGRWPiFGMZ8DGFfABXo6idekPoJE,14757
 cocoindex/typing.py,sha256=qQ-nSdkHzu8pSxfuR5sGGfoE8nCKqCDb0D9jbmxVt4M,12635
 cocoindex/utils.py,sha256=hUhX-XV6XGCtJSEIpBOuDv6VvqImwPlgBxztBTw7u0U,598
-cocoindex-0.1.64.dist-info/RECORD,,
+cocoindex-0.1.65.dist-info/RECORD,,