cocoindex 0.1.74__tar.gz → 0.1.76__tar.gz

{cocoindex-0.1.74 → cocoindex-0.1.76}/.github/workflows/_doc_release.yml

@@ -11,7 +11,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 18
           cache: yarn
           cache-dependency-path: docs/yarn.lock
       - uses: webfactory/ssh-agent@v0.5.0

{cocoindex-0.1.74 → cocoindex-0.1.76}/.github/workflows/_test.yml

@@ -43,7 +43,7 @@ jobs:
         - name: Install Python toolchains
           run: |
             source .venv/bin/activate
-            pip install maturin pytest mypy
+            pip install maturin mypy pytest pytest-asyncio
         - name: Python build
           run: |
             source .venv/bin/activate

{cocoindex-0.1.74 → cocoindex-0.1.76}/.github/workflows/docs.yml

@@ -19,7 +19,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 18
           cache: yarn
           cache-dependency-path: docs/yarn.lock
       - name: Install dependencies

{cocoindex-0.1.74 → cocoindex-0.1.76}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.74"
+version = "0.1.76"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.74 → cocoindex-0.1.76}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.74"
+version = "0.1.76"
 edition = "2024"
 rust-version = "1.88"
 

{cocoindex-0.1.74 → cocoindex-0.1.76}/PKG-INFO

@@ -1,24 +1,28 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.74
+Version: 0.1.76
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: watchfiles>=1.1.0
 Requires-Dist: numpy>=1.23.2
 Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: pytest-asyncio ; extra == 'dev'
 Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: mypy ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
+Requires-Dist: colpali-engine ; extra == 'colpali'
 Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'all'
+Requires-Dist: colpali-engine ; extra == 'all'
 Provides-Extra: dev
 Provides-Extra: embeddings
+Provides-Extra: colpali
 Provides-Extra: all
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Author-email: CocoIndex <cocoindex.io@gmail.com>
-License: Apache-2.0
+License-Expression: Apache-2.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Homepage, https://cocoindex.io/
@@ -210,6 +214,7 @@ It defines an index flow like this:
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
+| [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
 
 More coming and stay tuned 👀!
 

{cocoindex-0.1.74 → cocoindex-0.1.76}/README.md

@@ -185,6 +185,7 @@ It defines an index flow like this:
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
+| [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
 
 More coming and stay tuned 👀!
 

{cocoindex-0.1.74 → cocoindex-0.1.76}/docs/docs/core/flow_def.mdx

@@ -33,23 +33,23 @@ It takes two arguments:
 *   `flow_builder`: a `FlowBuilder` object to help build the flow.
 *   `data_scope`: a `DataScope` object, representing the top-level data scope. Any data created by the flow should be added to it.
 
-Alternatively, for more flexibility (e.g. you want to do this conditionally or generate dynamic name), you can explicitly call the `cocoindex.add_flow_def()` method:
+Alternatively, for more flexibility (e.g. you want to do this conditionally or generate dynamic name), you can explicitly call the `cocoindex.open_flow()` method:
 
 ```python
 def demo_flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
     ...
 
 # Add the flow definition to the flow registry.
-demo_flow = cocoindex.add_flow_def("DemoFlow", demo_flow_def)
+demo_flow = cocoindex.open_flow("DemoFlow", demo_flow_def)
 ```
 
 In both cases, `demo_flow` will be an object with `cocoindex.Flow` class type.
 See [Flow Running](/docs/core/flow_methods) for more details on it.
 
-Sometimes you no longer want to keep states of the flow in memory. We provide a `cocoindex.remove_flow()` method for this purpose:
+Sometimes you no longer want to keep states of the flow in memory. We provide a `close()` method for this purpose:
 
 ```python
-cocoindex.remove_flow(demo_flow)
+demo_flow.close()
 ```
 
 After it's called, `demo_flow` becomes an invalid object, and you should not call any methods of it.

{cocoindex-0.1.74 → cocoindex-0.1.76}/docs/docs/core/flow_methods.mdx

@@ -1,13 +1,13 @@
 ---
-title: Run a Flow
+title: Operate a Flow
 toc_max_heading_level: 4
-description: Run a CocoIndex Flow, including build / update data in the target and evaluate the flow without changing the target.
+description: Operate a CocoIndex Flow, including build / update data in the target and evaluate the flow without changing the target.
 ---
 
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Run a CocoIndex Flow
+# Operate a CocoIndex Flow
 
 After a flow is defined as discussed in [Flow Definition](/docs/core/flow_def), you can start to transform data with it.
 
@@ -39,7 +39,7 @@ It creates a `demo_flow` object in `cocoindex.Flow` type.
 For a flow, its persistent backends need to be ready before it can run, including:
 
 *   [Internal storage](/docs/core/basics#internal-storage) for CocoIndex.
-*   Backend entities for targets exported by the flow, e.g. a table (in relational databases), a collection (in some vector databases), etc.
+*   Backend resources for targets exported by the flow, e.g. a table (in relational databases), a collection (in some vector databases), etc.
 
 The desired state of the backends for a flow is derived based on the flow definition itself.
 CocoIndex supports two types of actions to manage the persistent backends automatically:
@@ -104,7 +104,7 @@ cocoindex.drop_all_flows(report_to_stdout=True)
 
 After dropping the flow, the in-memory `cocoindex.Flow` instance is still valid, and you can call setup methods on it again.
 
-If you want to remove the flow from the current process, you can call `cocoindex.remove_flow(demo_flow)` to do so (see [related doc](/docs/core/flow_def#entry-point)).
+If you want to remove the flow from the current process, you can call `demo_flow.close()` to do so (see [related doc](/docs/core/flow_def#entry-point)).
 
 :::
 

{cocoindex-0.1.74 → cocoindex-0.1.76}/docs/docs/custom_ops/custom_targets.mdx

@@ -114,13 +114,6 @@ This method should be implemented to:
 - Update configuration when a target spec changes
 - Clean up resources when a target is removed (`current` is `None`)
 
-:::note Best practice: Keep all actions idempotent
-
-Ideally this method should be idempotent, i.e. when calling this with the same arguments multiple times, the effect should remain the same.
-For example, if the target is a directory, it should be a no-op if we try to create it (`previous` is `None`) when the directory already exists, and also a no-op if we try to delete it (`current` is `None`) when the directory does not exist.
-
-:::
-
 #### `describe(key) -> str` (Optional)
 
 Returns a human-readable description of the target for logging and debugging purposes.
@@ -165,6 +158,18 @@ def prepare(spec: CustomTarget) -> PreparedCustomTarget:
 
 If not provided, the original spec will be passed directly to `mutate`.
 
+## Best Practices
+
+### Idempotency of Methods with Side Effects
+
+`apply_setup_change()` and `mutate()` are the two methods that are expected to produce side effects.
+We expect them to be idempotent, i.e. when calling them with the same arguments multiple times, the effect should remain the same.
+
+For example,
+- For `apply_setup_change()`, if the target is a directory, it should be a no-op if we try to create it (`previous` is `None`) when the directory already exists, and also a no-op if we try to delete it (`current` is `None`) when the directory does not exist.
+- For `mutate()`, if a mutation is a deletion, it should be a no-op if the row does not exist.
+
+This is to make sure when the system if left in an intermediate state, e.g. interrupted in the middle between a change is made and CocoIndex notes down the change is completed, the targets can still be gracefully rolled forward to the desired states after the system is resumed.
 
 ## Examples
 

{cocoindex-0.1.74 → cocoindex-0.1.76}/docs/docs/ops/functions.md

@@ -188,3 +188,53 @@ Input data:
 *   `text` (*Str*, required): The text to embed.
 
 Return: *Vector[Float32, N]*, where *N* is the dimension of the embedding vector determined by the model.
+
+## ColPaliEmbedImage
+
+`ColPaliEmbedImage` embeds images using the ColPali multimodal model.
+
+ColPali (Contextual Late-interaction over Patches) uses late interaction between image patch embeddings and text token embeddings for retrieval.
+
+:::note Optional Dependency Required
+
+This function requires the `colpali-engine` library, which is an optional dependency. Install CocoIndex with:
+
+```bash
+pip install 'cocoindex[colpali]'
+```
+:::
+
+The spec takes the following fields:
+
+*   `model` (`str`): The ColPali model name to use (e.g., "vidore/colpali-v1.2")
+
+Input data:
+
+*   `img_bytes` (*Bytes*): The image data in bytes format.
+
+Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable patches and fixed hidden dimension.
+
+## ColPaliEmbedQuery
+
+`ColPaliEmbedQuery` embeds text queries using the ColPali multimodal model.
+
+This produces query embeddings compatible with ColPali image embeddings for late interaction scoring (MaxSim).
+
+:::note Optional Dependency Required
+
+This function requires the `colpali-engine` library, which is an optional dependency. Install CocoIndex with:
+
+```bash
+pip install 'cocoindex[colpali]'
+```
+:::
+
+The spec takes the following fields:
+
+*   `model` (`str`): The ColPali model name to use (e.g., "vidore/colpali-v1.2")
+
+Input data:
+
+*   `query` (*Str*): The text query to embed.
+
+Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable tokens and fixed hidden dimension.

cocoindex-0.1.76/docs/docs/tutorials/manage_flow_dynamically.mdx

@@ -0,0 +1,302 @@
+---
+title: Manage Flows Dynamically
+description: "Learn how to dynamically manage multiple flow instances in CocoIndex. Create parameterized data indexing pipelines, handle persistent resources, perform updates, and implement memory-efficient caching for scalable applications."
+---
+
+# Manage Flows Dynamically
+
+You write a function, a.k.a. *flow definition*, to define indexing logic.
+Sometimes you want to reuse the same flow definition for multiple *flow instances* (a.k.a. *flow*), e.g. each takes input from different sources, exports to different targets, and even with slightly different parameters for transformation logic.
+
+## States of a flow instance
+
+A flow instance has states from two aspects:
+
+*   *In-process object*, of type `cocoindex.Flow`.
+*   *Persistent resource*, including states in the [internal storage](/docs/core/basics#internal-storage) and backend resources that are owned by the flow instance.
+
+A flow instance is ultimately a persistent resource. Its in-process object is a handle to operate on it. Consider file handles and database connections.
+CocoIndex provides APIs to *open* and *close* flow instances, and *setup* and *drop* the persistent resource.
+
+## Parameterize the flow definition
+
+In the example from the [Quickstart Guide](/docs/getting_started/quickstart), we decorate the flow definition function with a `@cocoindex.flow_def(name="DemoFlow")` decorator:
+
+```python title="Example in Quickstart Guide"
+@cocoindex.flow_def(name="TextEmbedding")
+def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    ...
+```
+
+This immediately creates the in-process object of the flow instance, using the given function as the flow definition.
+This is a shortcut of:
+
+```python
+def _text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    ...
+
+text_embedding_flow = cocoindex.open_flow("TextEmbedding", _text_embedding_flow)
+```
+
+Here, `cocoindex.open_flow()` is the function that creates the in-process object of the flow instance, with the given name and flow definition function.
+You can directly call it dynamically with flow name created programmatically.
+
+Oftentimes, you also want to parameterize the flow definition function.
+For example, we may have a dataclass like this to hold the parameters of the flow:
+
+```python
+@dataclass
+class TextEmbeddingFlowParameters:
+    source_path: str
+    target_table_name: str
+```
+
+And consider we have a registry of parameters for all flow instances somewhere.
+For simplicity, we use a hardcoded `dict` here, and provide a simple function to get the parameters for a given flow name.
+In reality, the source of truth may come from a configuration file, a database, etc., and the function can be replaced by your own implementation.
+
+```python
+FLOW_PARAMETERS: dict[str, TextEmbeddingFlowParameters] = {
+    "foo": TextEmbeddingFlowParameters(source_path="/path/to/foo", target_table_name="foo_embeddings"),
+    "bar": TextEmbeddingFlowParameters(source_path="/path/to/bar", target_table_name="bar_embeddings"),
+}
+
+def get_flow_parameters(name: str) -> TextEmbeddingFlowParameters:
+    return FLOW_PARAMETERS[name]
+```
+
+Then you can have a function that returns the flow definition function for the given parameters:
+
+```python
+def text_embedding_flow_def(params: TextEmbeddingFlowParameters):
+    def _flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+        # Add a data source to read files from the specified directory
+        data_scope["documents"] = flow_builder.add_source(
+            cocoindex.sources.LocalFile(path=params.source_path))
+
+        doc_embeddings = data_scope.add_collector()
+        ...
+
+        # Export the collected data to a Postgres table, with the specified table name.
+        doc_embeddings.export(
+            "doc_embeddings",
+            cocoindex.targets.Postgres(table_name=params.target_table_name),
+            primary_key_fields=["filename", "location"],
+        )
+
+    return _flow_def
+```
+
+With this, you can open flow instances dynamically with its parameters:
+
+```python
+text_embedding_flows: dict[str, cocoindex.Flow] = {}
+
+def get_text_embedding_flow(name: str) -> cocoindex.Flow:
+    flow = text_embedding_flows.get(name)
+
+    if flow is None:
+        params = get_flow_parameters(name)
+        flow = text_embedding_flows[name] = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
+
+    return flow
+```
+
+## Operate on the flow instances
+
+### Setup the persistent resource
+
+After you instantiated and open flow instances dynamically, before you can perform any data updates, you need to make sure the persistent resource is ready.
+You can use the `setup()` method, e.g. modify the above code to:
+
+```python
+text_embedding_flows: dict[str, cocoindex.Flow] = {}
+
+def get_text_embedding_flow(name: str) -> cocoindex.Flow:
+    flow = text_embedding_flows.get(name)
+
+    if flow is None:
+        params = get_flow_parameters(name)
+        flow = text_embedding_flows[name] = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
+        flow.setup(report_to_stdout=True)
+
+    return flow
+```
+
+`setup()` method synchronizes the persistent resource to a state that is consistent with the in-process object. For example,
+*   If the persistent resource is not there yet, it will create the backend resources for new targets.
+*   If your flow definition changed and a new target has been added since the last time of setup, it will create the backend resources for new targets.
+*   If an existing target is removed from the flow definition, it will drop the backend resources for the removed target.
+*   If nothing changed since the last time of setup, it will be a no-op. i.e. the `setup()` method is idempotent.
+
+`setup()` takes a `report_to_stdout` parameter to control whether to print the setup progress to the standard output.
+
+`setup()` takes care of all scenarios and makes sure the persistent resource is in the right state.
+It's generally safe to call it after you open a flow instance, even if you don't know whether the persistent resource already exists.
+
+
+### Perform data updates
+
+After you make sure the persistent resource is ready, you can perform data updates using the flow.
+
+The `update()` method updates the target defined by the flow.
+
+```python
+flow.update()
+```
+
+This performs a one-time data update. After the function returns, the target is up-to-date as of the moment when the function is called. For example, we can call `update()` to update the target after the flow is setup:
+
+```python
+def update_text_embedding_index(name: str):
+    flow = get_text_embedding_flow(name)
+    flow.update()
+```
+
+You can also do a live update.
+See the [Live Updates](/docs/tutorials/live_updates) tutorial for more details.
+
+
+### Close the flow object
+
+Sometimes you don't want to hold the in-process object forever.
+You can free up the memory resources by closing the flow instances with the `close()` method.
+
+For example, the `dict` we managed above behaves like a cache to hold the flow instances.
+If a specific flow isn't used for a while, we may close it.
+The `TTLCache` from [`cachetools`](https://pypi.org/project/cachetools/) package provides exactly this functionality.
+We can rewrite the above code a little bit.
+First, we bring in necessary imports:
+
+```python
+from cachetools import cached, TTLCache
+```
+
+Then we define our own version of `TTLCache` to make it call the `close()` method when the flow instance is evicted from the cache:
+
+```python
+class MyTTLCache(TTLCache):
+    def popitem(self):
+        # Close the flow instance when it is evicted from the cache
+        key, flow = super().popitem()
+        flow.close()
+        return key, flow
+```
+
+With this, we can modify our `get_text_embedding_flow()` function to use `MyTTLCache` to cache the flow instances, instead of managing our own `dict`:
+
+```python
+@cached(cache=MyTTLCache(maxsize=20, ttl=600))
+def get_text_embedding_flow(name: str) -> cocoindex.Flow:
+    params = get_flow_parameters(name)
+    flow = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
+    flow.setup(report_to_stdout=True)
+    return flow
+```
+
+The `@cached()` decorator from `cachetools` package automatically manages the cache for us (and it also offers thread safety!).
+Once a flow is not touched for 10 minutes, it will call the `popitem()` method, which will close the in-memory flow object.
+
+
+### Drop the persistent resource
+
+Occasionally, you may want to drop the persistent resource of a flow.
+The `drop()` method is for this purpose.
+
+```python
+def drop_text_embedding_index(name: str):
+    flow = get_text_embedding_flow(name)
+    flow.drop()
+```
+
+This will drop the persistent resource of the flow.
+The in-memory flow object is still alive, and can be reused until it's closed.
+For example, you can still call `setup()` again.
+
+## Put it all together
+
+```python
+import cocoindex
+from cachetools import cached, TTLCache
+from dataclasses import dataclass
+
+@dataclass
+class TextEmbeddingFlowParameters:
+    source_path: str
+    target_table_name: str
+
+FLOW_PARAMETERS: dict[str, TextEmbeddingFlowParameters] = {
+    "foo": TextEmbeddingFlowParameters(source_path="/path/to/foo", target_table_name="foo_embeddings"),
+    "bar": TextEmbeddingFlowParameters(source_path="/path/to/bar", target_table_name="bar_embeddings"),
+}
+
+# Placeholder to get the parameters for a given flow name. You can replace this with your own implementation.
+def get_flow_parameters(name: str) -> TextEmbeddingFlowParameters:
+    return FLOW_PARAMETERS[name]
+
+
+def text_embedding_flow_def(params: TextEmbeddingFlowParameters):
+    def _flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+        # Add a data source to read files from a directory
+        data_scope["documents"] = flow_builder.add_source(
+            cocoindex.sources.LocalFile(path=params.source_path))
+
+        doc_embeddings = data_scope.add_collector()
+        ...
+
+        # Export the collected data to a Postgres table, with the specified table name.
+        doc_embeddings.export(
+            "doc_embeddings",
+            cocoindex.targets.Postgres(table_name=params.target_table_name),
+            primary_key_fields=["filename", "location"],
+        )
+
+    return _flow_def
+
+class MyTTLCache(TTLCache):
+    def popitem(self):
+        # Close the flow instance when it is evicted from the cache
+        key, flow = super().popitem()
+        flow.close()
+        return key, flow
+
+@cached(cache=MyTTLCache(maxsize=20, ttl=600))
+def get_text_embedding_flow(name: str) -> cocoindex.Flow:
+    params = get_flow_parameters(name)
+    flow = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
+    flow.setup(report_to_stdout=True)
+    return flow
+
+def update_text_embedding_index(name: str):
+    flow = get_text_embedding_flow(name)
+    flow.update()
+
+def drop_text_embedding_index(name: str):
+    flow = get_text_embedding_flow(name)
+    flow.drop()
+```
+
+This provides a skeleton.
+With this, you can trigger `update_text_embedding_index()` and `drop_text_embedding_index()` from your application, e.g. a web server API.
+
+## Takeaways
+
+From this tutorial, we walked through major flow management / operation APIs provided by CocoIndex. These APIs can be categorized into three aspects:
+
+| Aspect | APIs | Description |
+|--------|------|-------------|
+| Life of in-process flow object | `open_flow()`, `Flow.close()` | Create and destroy the in-memory handle to operate on flow instances |
+| Life of persistent resource | `Flow.setup()`, `Flow.drop()` | Create and destroy the backend resources and internal storage |
+| Data updates | `Flow.update()`, `FlowLiveUpdater` | Execute the indexing logic to update targets. *Requires persistent resource to be up-to-date first.*  |
+
+
+For simplicity, we use an in-memory `dict` as source of truth for the flow parameters.
+You can replace it with your own mechanism, e.g. table from a database, a configuration file, etc.
+You can trigger these APIs from your applications specific to your use case, e.g. from a specific API endpoint of a web server.
+
+## Further readings
+
+You can see the following documents for more details:
+
+*   [CocoIndex Flow Definition: Entry Point](/docs/core/flow_def#entry-point)
+*   [Operate a Flow](/docs/core/flow_methods)

{cocoindex-0.1.74 → cocoindex-0.1.76}/docs/sidebars.ts

@@ -18,6 +18,7 @@ const sidebars: SidebarsConfig = {
       collapsed: false,
       items: [
         'tutorials/live_updates',
+        'tutorials/manage_flow_dynamically',
       ],
     },
     {