cocoindex 0.1.38__tar.gz → 0.1.40__tar.gz

{cocoindex-0.1.38 → cocoindex-0.1.40}/Cargo.lock

@@ -993,7 +993,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.38"
+version = "0.1.40"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.38 → cocoindex-0.1.40}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.38"
+version = "0.1.40"
 edition = "2024"
 
 [profile.release]

{cocoindex-0.1.38 → cocoindex-0.1.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.38
+Version: 0.1.40
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
@@ -153,7 +153,7 @@ It defines an index flow like this:
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
-| [Product_Taxonomy_Knowledge_Graph](examples/product_taxonomy_knowledge_graph) | Build knowledge graph for product recommendations | 
+| [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database| 
 | [Image Search with Vision API](examples/image_search_example) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 
 More coming and stay tuned 👀!

{cocoindex-0.1.38 → cocoindex-0.1.40}/README.md

@@ -137,7 +137,7 @@ It defines an index flow like this:
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
-| [Product_Taxonomy_Knowledge_Graph](examples/product_taxonomy_knowledge_graph) | Build knowledge graph for product recommendations | 
+| [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database| 
 | [Image Search with Vision API](examples/image_search_example) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 
 More coming and stay tuned 👀!

{cocoindex-0.1.38 → cocoindex-0.1.40}/docs/docs/core/flow_def.mdx

@@ -146,8 +146,9 @@ def demo_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataSco
 
 :::info
 
-In live update mode, for each refresh, CocoIndex will traverse the data source to figure out the changes,
+In live update mode, for each refresh, CocoIndex will list rows in the data source to figure out the changes based on metadata such as last modified time,
 and only perform transformations on changed source keys.
+If nothing changed during the last refresh cycle, only list operations will be performed, which is usually cheap for most data sources.
 
 :::
 
@@ -311,6 +312,38 @@ Following metrics are supported:
 
 ## Miscellaneous
 
+### Getting App Namespace
+
+You can use the [`app_namespace` setting](initialization#app-namespace) or `COCOINDEX_APP_NAMESPACE` environment variable to specify the app namespace,
+to organize flows across different environments (e.g., dev, staging, production), team members, etc.
+
+In the code, You can call `flow.get_app_namespace()` to get the app namespace, and use it to name certain backends. It takes the following arguments:
+
+*   `trailing_delimiter` (optional): a string to append to the app namespace when it's not empty.
+
+e.g. when the current app namespace is `Staging`, `flow.get_app_namespace(trailing_delimiter='.')` will return `Staging.`.
+
+For example,
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+```python
+doc_embeddings.export(
+    "doc_embeddings",
+    cocoindex.storages.Qdrant(
+        collection_name=cocoindex.get_app_namespace(trailing_delimiter='__') + "doc_embeddings",
+        ...
+    ),
+    ...
+)
+```
+
+</TabItem>
+</Tabs>
+
+It will use `Staging__doc_embeddings` as the collection name if the current app namespace is `Staging`, and use `doc_embeddings` if the app namespace is empty.
+
 ### Target Declarations
 
 Most time a target storage is created by calling `export()` method on a collector, and this `export()` call comes with configurations needed for the target storage, e.g. options for storage indexes.

{cocoindex-0.1.38 → cocoindex-0.1.40}/docs/docs/core/flow_methods.mdx

@@ -105,7 +105,7 @@ A data source may enable one or multiple *change capture mechanisms*:
 *   Configured with a [refresh interval](flow_def#refresh-interval), which is generally applicable to all data sources.
 
 *   Specific data sources also provide their specific change capture mechanisms.
-    For example, [`GoogleDrive` source](../ops/sources#googledrive) allows polling recent modified files.
+    For example, [`AmazonS3` source](../ops/sources/#amazons3) watches S3 bucket's change events, and [`GoogleDrive` source](../ops/sources#googledrive) allows polling recent modified files.
     See documentations for specific data sources.
 
 Change capture mechanisms enable CocoIndex to continuously capture changes from the source data and update the target data accordingly, under live update mode.

{cocoindex-0.1.38 → cocoindex-0.1.40}/docs/docs/core/initialization.mdx

@@ -88,9 +88,10 @@ if __name__ == "__main__":
 
 ### App Namespace
 
-The `app_namespace` field helps organize flows across different environments (e.g., testing, production) or teams. When set, it prefixes flow names with the namespace.
+The `app_namespace` field helps organize flows across different environments (e.g., dev, staging, production), team members, etc. When set, it prefixes flow names with the namespace.
 
-For example, if the namespace is "Staging", for a flow with name specified as `Flow1` in code, the full name of the flow will be `Staging.Flow1`.
+For example, if the namespace is `Staging`, for a flow with name specified as `Flow1` in code, the full name of the flow will be `Staging.Flow1`.
+You can also get the current app namespace by calling `cocoindex.get_app_namespace()` (see [Getting App Namespace](flow_def#getting-app-namespace) for more details).
 
 If not set, all flows are in a default unnamed namespace.
 

{cocoindex-0.1.38 → cocoindex-0.1.40}/docs/docs/ops/storages.md

@@ -40,7 +40,7 @@ The spec takes the following fields:
     See [DatabaseConnectionSpec](../core/initialization#databaseconnectionspec) for its specific fields.
     If not provided, will use the same database as the [internal storage](/docs/core/basics#internal-storage).
 
-*   `table_name` (type: `str`, optional): The name of the table to store to. If unspecified, will generate a new automatically. We recommend specifying a name explicitly if you want to directly query the table. It can be omitted if you want to use CocoIndex's query handlers to query the table.
+*   `table_name` (type: `str`, optional): The name of the table to store to. If unspecified, will use the table name `[${AppNamespace}__]${FlowName}__${TargetName}`, e.g. `DemoFlow__doc_embeddings` or `Staging__DemoFlow__doc_embeddings`.
 
 ### Qdrant
 

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/amazon_s3_embedding/pyproject.toml

@@ -3,4 +3,7 @@ name = "amazon-s3-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Amazon S3 files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/code_embedding/pyproject.toml

@@ -3,4 +3,7 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,4 +3,7 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,4 +3,7 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/manuals_llm_extraction/pyproject.toml

@@ -4,7 +4,10 @@ version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.35",
+    "cocoindex>=0.1.39",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,10 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.35",
+    "cocoindex>=0.1.39",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38/examples/product_taxonomy_knowledge_graph → cocoindex-0.1.40/examples/product_recommendation}/README.md

@@ -1,6 +1,8 @@
-# Build Real-Time Product Recommendation based on LLM Taxonomy Extraction and Knowledge Graph
+# Build Real-Time Recommendation Engine with LLM and Graph Database
 
-We will process a list of products and use LLM to extract the taxonomy and complimentary taxonomy for each product.
+We will build a real-time product recommendation engine with LLM and graph database. In particular, we will use LLM to understand the category (taxonomy) of a product. In addition, we will use LLM to enumerate the complementary products - users are likely to buy together with the current product (pencil and notebook). 
+
+We will use Graph to explore the relationships between products that can be further used for product recommendations or labeling.
 
 Please drop [CocoIndex on Github](https://github.com/cocoindex-io/cocoindex) a star to support us and stay tuned for more updates. Thank you so much 🥥🤗. [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
 

{cocoindex-0.1.38/examples/product_taxonomy_knowledge_graph → cocoindex-0.1.40/examples/product_recommendation}/pyproject.toml

@@ -3,4 +3,7 @@ name = "cocoindex-ecommerce-taxonomy"
 version = "0.1.0"
 description = "Simple example for CocoIndex: extract taxonomy from e-commerce products and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1", "jinja2>=3.1.6"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1", "jinja2>=3.1.6"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/text_embedding/main.py

@@ -2,6 +2,7 @@ from dotenv import load_dotenv
 
 import cocoindex
 
+@cocoindex.transform_flow()
 def text_to_embedding(text: cocoindex.DataSlice) -> cocoindex.DataSlice:
     """
     Embed the text using a SentenceTransformer model.

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/text_embedding/pyproject.toml

@@ -3,4 +3,7 @@ name = "text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

cocoindex-0.1.40/examples/text_embedding_qdrant/.env

@@ -0,0 +1,2 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

{cocoindex-0.1.38 → cocoindex-0.1.40}/examples/text_embedding_qdrant/pyproject.toml

@@ -3,4 +3,7 @@ name = "text-embedding-qdrant"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+
+[tool.setuptools]
+packages = []

{cocoindex-0.1.38 → cocoindex-0.1.40}/python/cocoindex/__init__.py

@@ -1,14 +1,15 @@
 """
 Cocoindex is a framework for building and running indexing pipelines.
 """
-from . import functions, query, sources, storages, cli
-from .flow import FlowBuilder, DataScope, DataSlice, Flow, flow_def
+from . import functions, query, sources, storages, cli, utils
+from .flow import FlowBuilder, DataScope, DataSlice, Flow, flow_def, transform_flow
 from .flow import EvaluateAndDumpOptions, GeneratedField
 from .flow import update_all_flows_async, FlowLiveUpdater, FlowLiveUpdaterOptions
 from .llm import LlmSpec, LlmApiType
 from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
 from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
 from .lib import *
-from .setting import *
+from .setting import DatabaseConnectionSpec, Settings, ServerSettings
+from .setting import get_app_namespace
 from ._engine import OpArgSchema
 from .typing import Float32, Float64, LocalDateTime, OffsetDateTime, Range, Vector, Json

{cocoindex-0.1.38 → cocoindex-0.1.40}/python/cocoindex/convert.py

@@ -44,7 +44,7 @@ def make_engine_value_decoder(
 
     src_type_kind = src_type['kind']
 
-    if dst_annotation is inspect.Parameter.empty:
+    if dst_annotation is None or dst_annotation is inspect.Parameter.empty or dst_annotation is Any:
         if src_type_kind == 'Struct' or src_type_kind in TABLE_TYPES:
             raise ValueError(f"Missing type annotation for `{''.join(field_path)}`."
                              f"It's required for {src_type_kind} type.")

{cocoindex-0.1.38 → cocoindex-0.1.40}/python/cocoindex/flow.py

@@ -8,8 +8,9 @@ import asyncio
 import re
 import inspect
 import datetime
+import functools
 
-from typing import Any, Callable, Sequence, TypeVar
+from typing import Any, Callable, Sequence, TypeVar, Generic, get_args, get_origin, Type, NamedTuple
 from threading import Lock
 from enum import Enum
 from dataclasses import dataclass
@@ -20,7 +21,7 @@ from . import _engine
 from . import index
 from . import op
 from . import setting
-from .convert import dump_engine_object
+from .convert import dump_engine_object, encode_engine_value, make_engine_value_decoder
 from .typing import encode_enriched_type
 from .runtime import execution_context
 
@@ -123,7 +124,7 @@ class _DataSliceState:
         # TODO: We'll support this by an identity transformer or "aliasing" in the future.
         raise ValueError("DataSlice is already attached to a field")
 
-class DataSlice:
+class DataSlice(Generic[T]):
     """A data slice represents a slice of data in a flow. It's readonly."""
 
     _state: _DataSliceState
@@ -183,11 +184,11 @@ class DataSlice:
                         name, prefix=_to_snake_case(_spec_kind(fn_spec))+'_'),
                 ))
 
-    def call(self, func: Callable[[DataSlice], T]) -> T:
+    def call(self, func: Callable[[DataSlice], T], *args, **kwargs) -> T:
         """
         Call a function with the data slice.
         """
-        return func(self)
+        return func(self, *args, **kwargs)
 
 def _data_slice_state(data_slice: DataSlice) -> _DataSliceState:
     return data_slice._state  # pylint: disable=protected-access
@@ -309,9 +310,8 @@ class _FlowBuilderState:
     engine_flow_builder: _engine.FlowBuilder
     field_name_builder: _NameBuilder
 
-    def __init__(self, /, name: str | None = None):
-        flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
-        self.engine_flow_builder = _engine.FlowBuilder(get_full_flow_name(flow_name))
+    def __init__(self, full_name: str):
+        self.engine_flow_builder = _engine.FlowBuilder(full_name)
         self.field_name_builder = _NameBuilder()
 
     def get_data_slice(self, v: Any) -> _engine.DataSlice:
@@ -463,9 +463,13 @@ class Flow:
     """
     A flow describes an indexing pipeline.
     """
+    _name: str
+    _full_name: str
     _lazy_engine_flow: Callable[[], _engine.Flow]
 
-    def __init__(self, engine_flow_creator: Callable[[], _engine.Flow]):
+    def __init__(self, name: str, full_name: str, engine_flow_creator: Callable[[], _engine.Flow]):
+        self._name = name
+        self._full_name = full_name
         engine_flow = None
         lock = Lock()
         def _lazy_engine_flow() -> _engine.Flow:
@@ -496,7 +500,7 @@ class Flow:
             tree.children.append(section_node)
         return tree
 
-    def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]:
+    def _get_spec(self, verbose: bool = False) -> _engine.RenderedSpec:
         return self._lazy_engine_flow().get_spec(output_mode="verbose" if verbose else "concise")
     
     def _get_schema(self) -> list[tuple[str, str, str]]:
@@ -508,12 +512,19 @@ class Flow:
     def __repr__(self):
         return repr(self._lazy_engine_flow())
 
+    @property
+    def name(self) -> str:
+        """
+        Get the name of the flow.
+        """
+        return self._name
+
     @property
     def full_name(self) -> str:
         """
         Get the full name of the flow.
         """
-        return self._lazy_engine_flow().name()
+        return self._full_name
 
     def update(self) -> _engine.IndexUpdateInfo:
         """
@@ -554,14 +565,16 @@ def _create_lazy_flow(name: str | None, fl_def: Callable[[FlowBuilder, DataScope
     Create a flow without really building it yet.
     The flow will be built the first time when it's really needed.
     """
+    flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
+    flow_full_name = get_full_flow_name(flow_name)
     def _create_engine_flow() -> _engine.Flow:
-        flow_builder_state = _FlowBuilderState(name=name)
+        flow_builder_state = _FlowBuilderState(flow_full_name)
         root_scope = DataScope(
             flow_builder_state, flow_builder_state.engine_flow_builder.root_scope())
         fl_def(FlowBuilder(flow_builder_state), root_scope)
         return flow_builder_state.engine_flow_builder.build_flow(execution_context.event_loop)
 
-    return Flow(_create_engine_flow)
+    return Flow(flow_name, flow_full_name, _create_engine_flow)
 
 
 _flows_lock = Lock()
@@ -642,27 +655,67 @@ async def update_all_flows_async(options: FlowLiveUpdaterOptions) -> dict[str, _
     all_stats = await asyncio.gather(*(_update_flow(name, fl) for (name, fl) in fls.items()))
     return dict(all_stats)
 
-_transient_flow_name_builder = _NameBuilder()
-class TransientFlow:
+def _get_data_slice_annotation_type(data_slice_type: Type[DataSlice[T]]) -> Type[T] | None:
+    type_args = get_args(data_slice_type)
+    if data_slice_type is DataSlice:
+        return None
+    if get_origin(data_slice_type) != DataSlice or len(type_args) != 1:
+        raise ValueError(f"Expect a DataSlice[T] type, but got {data_slice_type}")
+    return type_args[0]
+
+_transform_flow_name_builder = _NameBuilder()
+
+class TransformFlowInfo(NamedTuple):
+    engine_flow: _engine.TransientFlow
+    result_decoder: Callable[[Any], T]
+
+class TransformFlow(Generic[T]):
     """
     A transient transformation flow that transforms in-memory data.
     """
-    _engine_flow: _engine.TransientFlow
+    _flow_fn: Callable[..., DataSlice[T]]
+    _flow_name: str
+    _flow_arg_types: list[Any]
+    _param_names: list[str]
+
+    _lazy_lock: asyncio.Lock
+    _lazy_flow_info: TransformFlowInfo | None = None
 
     def __init__(
-            self, flow_fn: Callable[..., DataSlice],
+            self, flow_fn: Callable[..., DataSlice[T]],
             flow_arg_types: Sequence[Any], /, name: str | None = None):
+        self._flow_fn = flow_fn
+        self._flow_name = _transform_flow_name_builder.build_name(name, prefix="_transform_flow_")
+        self._flow_arg_types = list(flow_arg_types)
+        self._lazy_lock = asyncio.Lock()
+
+    def __call__(self, *args, **kwargs) -> DataSlice[T]:
+        return self._flow_fn(*args, **kwargs)
 
-        flow_builder_state = _FlowBuilderState(
-            name=_transient_flow_name_builder.build_name(name, prefix="_transient_flow_"))
-        sig = inspect.signature(flow_fn)
-        if len(sig.parameters) != len(flow_arg_types):
+    @property
+    def _flow_info(self) -> TransformFlowInfo:
+        if self._lazy_flow_info is not None:
+            return self._lazy_flow_info
+        return execution_context.run(self._flow_info_async())
+
+    async def _flow_info_async(self) -> TransformFlowInfo:
+        if self._lazy_flow_info is not None:
+            return self._lazy_flow_info
+        async with self._lazy_lock:
+            if self._lazy_flow_info is None:
+                self._lazy_flow_info = await self._build_flow_info_async()
+            return self._lazy_flow_info
+
+    async def _build_flow_info_async(self) -> TransformFlowInfo:
+        flow_builder_state = _FlowBuilderState(self._flow_name)
+        sig = inspect.signature(self._flow_fn)
+        if len(sig.parameters) != len(self._flow_arg_types):
             raise ValueError(
                 f"Number of parameters in the flow function ({len(sig.parameters)}) "
-                "does not match the number of argument types ({len(flow_arg_types)})")
+                f"does not match the number of argument types ({len(self._flow_arg_types)})")
 
         kwargs: dict[str, DataSlice] = {}
-        for (param_name, param), param_type in zip(sig.parameters.items(), flow_arg_types):
+        for (param_name, param), param_type in zip(sig.parameters.items(), self._flow_arg_types):
             if param.kind not in (inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                   inspect.Parameter.KEYWORD_ONLY):
                 raise ValueError(f"Parameter {param_name} is not a parameter can be passed by name")
@@ -670,20 +723,68 @@ class TransientFlow:
                 param_name, encode_enriched_type(param_type))
             kwargs[param_name] = DataSlice(_DataSliceState(flow_builder_state, engine_ds))
 
-        output = flow_fn(**kwargs)
+        output = self._flow_fn(**kwargs)
         flow_builder_state.engine_flow_builder.set_direct_output(
             _data_slice_state(output).engine_data_slice)
-        self._engine_flow = flow_builder_state.engine_flow_builder.build_transient_flow(
-            execution_context.event_loop)
+        engine_flow = await flow_builder_state.engine_flow_builder.build_transient_flow_async(execution_context.event_loop)
+        self._param_names = list(sig.parameters.keys())
+
+        engine_return_type = _data_slice_state(output).engine_data_slice.data_type().schema()
+        python_return_type = _get_data_slice_annotation_type(sig.return_annotation)
+        result_decoder = make_engine_value_decoder([], engine_return_type['type'], python_return_type)
+
+        return TransformFlowInfo(engine_flow, result_decoder)
 
     def __str__(self):
-        return str(self._engine_flow)
+        return str(self._flow_info.engine_flow)
 
     def __repr__(self):
-        return repr(self._engine_flow)
+        return repr(self._flow_info.engine_flow)
 
     def internal_flow(self) -> _engine.TransientFlow:
         """
         Get the internal flow.
         """
-        return self._engine_flow
+        return self._flow_info.engine_flow
+
+    def eval(self, *args, **kwargs) -> T:
+        """
+        Evaluate the transform flow.
+        """
+        return execution_context.run(self.eval_async(*args, **kwargs))
+
+    async def eval_async(self, *args, **kwargs) -> T:
+        """
+        Evaluate the transform flow.
+        """
+        flow_info = await self._flow_info_async()
+        params = []
+        for i, arg in enumerate(self._param_names):
+            if i < len(args):
+                params.append(encode_engine_value(args[i]))
+            elif arg in kwargs:
+                params.append(encode_engine_value(kwargs[arg]))
+            else:
+                raise ValueError(f"Parameter {arg} is not provided")
+        engine_result = await flow_info.engine_flow.evaluate_async(params)
+        return flow_info.result_decoder(engine_result)
+
+
+def transform_flow() -> Callable[[Callable[..., DataSlice[T]]], TransformFlow[T]]:
+    """
+    A decorator to wrap the transform function.
+    """
+    def _transform_flow_wrapper(fn: Callable[..., DataSlice[T]]):
+        sig = inspect.signature(fn)
+        arg_types = []
+        for (param_name, param) in sig.parameters.items():
+            if param.kind not in (inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                                  inspect.Parameter.KEYWORD_ONLY):
+                raise ValueError(f"Parameter {param_name} is not a parameter can be passed by name")
+            arg_types.append(_get_data_slice_annotation_type(param.annotation))
+
+        _transform_flow = TransformFlow(fn, arg_types)
+        functools.update_wrapper(_transform_flow, fn)
+        return _transform_flow
+
+    return _transform_flow_wrapper

{cocoindex-0.1.38 → cocoindex-0.1.40}/python/cocoindex/op.py

@@ -100,8 +100,8 @@ def _register_op_factory(
             return op_args.behavior_version
 
     class _WrappedClass(executor_cls, _Fallback):
-        _args_converters: list[Callable[[Any], Any]]
-        _kwargs_converters: dict[str, Callable[[str, Any], Any]]
+        _args_decoders: list[Callable[[Any], Any]]
+        _kwargs_decoders: dict[str, Callable[[str, Any], Any]]
         _acall: Callable
 
         def __init__(self, spec):
@@ -109,17 +109,17 @@ def _register_op_factory(
             self.spec = spec
             self._acall = _to_async_call(super().__call__)
 
-        def analyze(self, *args, **kwargs):
+        def analyze(self, *args: _engine.OpArgSchema, **kwargs: _engine.OpArgSchema):
             """
             Analyze the spec and arguments. In this phase, argument types should be validated.
             It should return the expected result type for the current op.
             """
-            self._args_converters = []
-            self._kwargs_converters = {}
+            self._args_decoders = []
+            self._kwargs_decoders = {}
 
             # Match arguments with parameters.
             next_param_idx = 0
-            for arg in  args:
+            for arg in args:
                 if next_param_idx >= len(expected_args):
                     raise ValueError(
                         f"Too many arguments passed in: {len(args)} > {len(expected_args)}")
@@ -128,7 +128,7 @@ def _register_op_factory(
                     inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.VAR_KEYWORD):
                     raise ValueError(
                         f"Too many positional arguments passed in: {len(args)} > {next_param_idx}")
-                self._args_converters.append(
+                self._args_decoders.append(
                     make_engine_value_decoder(
                         [arg_name], arg.value_type['type'], arg_param.annotation))
                 if arg_param.kind != inspect.Parameter.VAR_POSITIONAL:
@@ -146,7 +146,7 @@ def _register_op_factory(
                 if expected_arg is None:
                     raise ValueError(f"Unexpected keyword argument passed in: {kwarg_name}")
                 arg_param = expected_arg[1]
-                self._kwargs_converters[kwarg_name] = make_engine_value_decoder(
+                self._kwargs_decoders[kwarg_name] = make_engine_value_decoder(
                     [kwarg_name], kwarg.value_type['type'], arg_param.annotation)
 
             missing_args = [name for (name, arg) in expected_kwargs
@@ -174,8 +174,8 @@ def _register_op_factory(
                 await _to_async_call(setup_method)()
 
         async def __call__(self, *args, **kwargs):
-            converted_args = (converter(arg) for converter, arg in zip(self._args_converters, args))
-            converted_kwargs = {arg_name: self._kwargs_converters[arg_name](arg)
+            decoded_args = (decoder(arg) for decoder, arg in zip(self._args_decoders, args))
+            decoded_kwargs = {arg_name: self._kwargs_decoders[arg_name](arg)
                                 for arg_name, arg in kwargs.items()}
 
             if op_args.gpu:
@@ -185,9 +185,9 @@ def _register_op_factory(
                 # For now, we use a lock to ensure only one task is executed at a time.
                 # TODO: Implement multi-processing dispatching.
                 async with _gpu_dispatch_lock:
-                    output = await self._acall(*converted_args, **converted_kwargs)
+                    output = await self._acall(*decoded_args, **decoded_kwargs)
             else:
-                output = await self._acall(*converted_args, **converted_kwargs)
+                output = await self._acall(*decoded_args, **decoded_kwargs)
             return encode_engine_value(output)
 
     _WrappedClass.__name__ = executor_cls.__name__

{cocoindex-0.1.38 → cocoindex-0.1.40}/python/cocoindex/query.py

@@ -50,7 +50,7 @@ class SimpleSemanticsQueryHandler:
                     if engine_handler is None:
                         engine_handler = _engine.SimpleSemanticsQueryHandler(
                             flow.internal_flow(), target_name,
-                            fl.TransientFlow(query_transform_flow, [str]).internal_flow(),
+                            fl.TransformFlow(query_transform_flow, [str]).internal_flow(),
                             default_similarity_metric.value)
                         engine_handler.register_query_handler(name)
             return engine_handler