cocoindex 0.1.66__tar.gz → 0.1.68__tar.gz

{cocoindex-0.1.66 → cocoindex-0.1.68}/.github/workflows/_test.yml

@@ -47,7 +47,7 @@ jobs:
         - name: Python build
           run: |
             source .venv/bin/activate
-            maturin develop
+            maturin develop -E all
         - name: Python type check (mypy)
           run: |
             source .venv/bin/activate

{cocoindex-0.1.66 → cocoindex-0.1.68}/.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
     hooks:
         - id: maturin-develop
           name: maturin develop
-          entry: maturin develop
+          entry: maturin develop -E all,dev
           language: system
           files: ^(python/|src/|Cargo\.toml|pyproject\.toml)
           pass_filenames: false

{cocoindex-0.1.66 → cocoindex-0.1.68}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.66"
+version = "0.1.68"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.66 → cocoindex-0.1.68}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.66"
+version = "0.1.68"
 edition = "2024"
 rust-version = "1.88"
 

cocoindex-0.1.66/README.md → cocoindex-0.1.68/PKG-INFO

@@ -1,3 +1,28 @@
+Metadata-Version: 2.4
+Name: cocoindex
+Version: 0.1.68
+Requires-Dist: click>=8.1.8
+Requires-Dist: rich>=14.0.0
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: watchfiles>=1.1.0
+Requires-Dist: numpy>=1.23.2
+Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: ruff ; extra == 'dev'
+Requires-Dist: mypy ; extra == 'dev'
+Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
+Requires-Dist: cocoindex[embeddings] ; extra == 'all'
+Provides-Extra: dev
+Provides-Extra: embeddings
+Provides-Extra: all
+License-File: LICENSE
+Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
+Author-email: CocoIndex <cocoindex.io@gmail.com>
+License: Apache-2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://cocoindex.io/
+
 <p align="center">
     <img src="https://cocoindex.io/images/github.svg" alt="CocoIndex">
 </p>
@@ -11,16 +36,36 @@
 [![License](https://img.shields.io/badge/license-Apache%202.0-5B5BD6?logoColor=white)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI version](https://img.shields.io/pypi/v/cocoindex?color=5B5BD6)](https://pypi.org/project/cocoindex/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cocoindex)](https://pypistats.org/packages/cocoindex)
-
 [![CI](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
 [![release](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
+
+</div>
+
+<div align="center">
+    <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>
 
+
 Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box.  Exceptional developer velocity. Production-ready at day 0.
 
 ⭐ Drop a star to help us grow!
 
+<div align="center">
+    
+<!-- Keep these links. Translations will automatically update with the README. -->
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
+    
+</div>
+
 </br>
 
 <p align="center">
@@ -34,7 +79,7 @@ CocoIndex makes it super easy to transform data with AI workloads, and keep sour
 </br>
 
 <p align="center">
-    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='480'>
+    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='400'>
 </p>
 
 </br>
@@ -188,3 +233,4 @@ We are constantly improving, and more features and examples are coming soon. If
 
 ## License
 CocoIndex is Apache 2.0 licensed.
+

cocoindex-0.1.66/PKG-INFO → cocoindex-0.1.68/README.md

@@ -1,26 +1,3 @@
-Metadata-Version: 2.4
-Name: cocoindex
-Version: 0.1.66
-Requires-Dist: click>=8.1.8
-Requires-Dist: rich>=14.0.0
-Requires-Dist: python-dotenv>=1.1.0
-Requires-Dist: watchfiles>=1.1.0
-Requires-Dist: numpy>=1.23.2
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: ruff ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
-Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
-Provides-Extra: test
-Provides-Extra: dev
-Provides-Extra: embeddings
-License-File: LICENSE
-Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
-Author-email: CocoIndex <cocoindex.io@gmail.com>
-License: Apache-2.0
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: Homepage, https://cocoindex.io/
-
 <p align="center">
     <img src="https://cocoindex.io/images/github.svg" alt="CocoIndex">
 </p>
@@ -34,16 +11,36 @@ Project-URL: Homepage, https://cocoindex.io/
 [![License](https://img.shields.io/badge/license-Apache%202.0-5B5BD6?logoColor=white)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI version](https://img.shields.io/pypi/v/cocoindex?color=5B5BD6)](https://pypi.org/project/cocoindex/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cocoindex)](https://pypistats.org/packages/cocoindex)
-
 [![CI](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
 [![release](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
+
+</div>
+
+<div align="center">
+    <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>
 
+
 Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box.  Exceptional developer velocity. Production-ready at day 0.
 
 ⭐ Drop a star to help us grow!
 
+<div align="center">
+    
+<!-- Keep these links. Translations will automatically update with the README. -->
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
+    
+</div>
+
 </br>
 
 <p align="center">
@@ -57,7 +54,7 @@ CocoIndex makes it super easy to transform data with AI workloads, and keep sour
 </br>
 
 <p align="center">
-    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='480'>
+    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='400'>
 </p>
 
 </br>
@@ -211,4 +208,3 @@ We are constantly improving, and more features and examples are coming soon. If
 
 ## License
 CocoIndex is Apache 2.0 licensed.
-

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/docs/about/contributing.md

@@ -46,12 +46,12 @@ Following the steps below to get cocoindex build on latest codebase locally - if
 
 -   Install required tools:
     ```sh
-    pip install maturin mypy pre-commit
+    pip install maturin
     ```
 
 -   Build the library. Run at the root of cocoindex directory:
     ```sh
-    maturin develop
+    maturin develop -E all,dev
     ```
 
 -   Install and enable pre-commit hooks. This ensures all checks run automatically before each commit:

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/docs/core/data_types.mdx

@@ -28,8 +28,6 @@ When you define a [custom function](/docs/core/custom_function), you need to ann
     we use the type annotation as a guidance to construct the Python value.
     Type annotation is optional for basic types and struct types, and required for table types.
 
-*   When you define a [custom function](/docs/core/custom_function), type annotation is required for arguments and return values.
-
 ### Basic Types
 
 #### Primitive Types

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/docs/core/flow_def.mdx

@@ -57,8 +57,7 @@ After it's called, `demo_flow` becomes an invalid object, and you should not cal
 :::note
 
 This only removes states of the flow from the current process, and it won't affect the persistent states.
-
-If you w
+See [Setup / drop flow](/docs/core/flow_methods#setup--drop-flow) if you want to clean up the persistent states.
 
 :::
 

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/docs/core/flow_methods.mdx

@@ -241,13 +241,27 @@ A `FlowLiveUpdater` object supports the following methods:
 
 *   `start()`: Start the updater.
     CocoIndex will continuously capture changes from the source data and update the target data accordingly in background threads managed by the engine.
+
 *   `abort()`: Abort the updater.
+
 *   `wait()`: Wait for the updater to finish. It only unblocks in one of the following cases:
     *   The updater was aborted.
     *   A one time update is done, and live update is not enabled:
         either `live_mode` is `False`, or all data sources have no change capture mechanisms enabled.
+
+*   `next_status_updates()`: Get the next status updates.
+    It blocks until there's a new status updates, including the processing finishes for a bunch of source updates, and live updater stops (aborted, or no more sources to process).
+    You can continuously call this method in a loop to get the latest status updates and react accordingly.
+
+    It returns a `cocoindex.FlowUpdaterStatusUpdates` object, with the following properties:
+    *   `active_sources`: Names of sources that are still active, i.e. not stopped processing. If it's empty, it means the updater is stopped.
+    *   `updated_sources`: Names of sources with updates since last time.
+        You can check this to see which sources have recent updates and get processed.
+
 *   `update_stats()`: It returns the stats of the updater.
 
+This snippets shows the lifecycle of a live updater:
+
 ```python
 my_updater = cocoindex.FlowLiveUpdater(demo_flow)
 # Start the updater.
@@ -256,14 +270,37 @@ my_updater.start()
 # Perform your own logic (e.g. a query loop).
 ...
 
-# Print the update stats.
-print(my_updater.update_stats())
-# Abort the updater.
-my_updater.abort()
+...
 # Wait for the updater to finish.
 my_updater.wait()
+# Print the update stats.
+print(my_updater.update_stats())
 ```
 
+Somewhere (in the same or other threads) you can also continuously call `next_status_updates()` to get the latest status updates and react accordingly, e.g.
+
+```python
+while True:
+    updates = my_updater.next_status_updates()
+
+    for source in updates.updated_sources:
+        # Perform downstream operations on the target of the source.
+        run_your_downstream_operations_for(source)
+
+    # Break the loop if there's no more active sources.
+    if not updates.active_sources:
+        break
+```
+
+:::info
+
+`next_status_updates()` automatically combines multiple status updates if more than one arrives between two calls,
+e.g. your downstream operations may take more time, or you don't need to process too frequently (in which case you can explicitly sleep for a while).
+
+So you don't need to worry about the status updates piling up.
+
+:::
+
 Python SDK also allows you to use the updater as a context manager.
 It will automatically start the updater during the context entry, and abort and wait for the updater to finish automatically when the context is exited.
 The following code is equivalent to the code above (if no early return happens):
@@ -272,7 +309,6 @@ The following code is equivalent to the code above (if no early return happens):
 with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
     # Perform your own logic (e.g. a query loop).
     ...
-    print(my_updater.update_stats())
 ```
 
 CocoIndex also provides asynchronous versions of APIs for blocking operations, including:
@@ -287,20 +323,27 @@ CocoIndex also provides asynchronous versions of APIs for blocking operations, i
     # Perform your own logic (e.g. a query loop).
     ...
 
-    # Print the update stats.
-    print(my_updater.update_stats())
-    # Abort the updater.
-    my_updater.abort()
     # Wait for the updater to finish.
     await my_updater.wait_async()
+    # Print the update stats.
+    print(my_updater.update_stats())
+    ```
+
+*   `next_status_updates_async()`, e.g.
+
+    ```python
+    while True:
+        updates = await my_updater.next_status_updates_async()
+
+        ...
     ```
+
 *   Async context manager, e.g.
 
     ```python
     async with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
         # Perform your own logic (e.g. a query loop).
         ...
-        print(my_updater.update_stats())
     ```
 
 </TabItem>

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/docs/ops/sources.md

@@ -167,7 +167,7 @@ These are actions you need to take:
 *   Create a storage account in the [Azure Portal](https://portal.azure.com/).
 *   Create a container in the storage account.
 *   Upload your files to the container.
-*   Grant the user / identity / service principal (depends on your authentication method, see below) access to the container. At minimum, a **Storage Blob Data Reader** role is needed. See [this doc](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-data-operations-portal) for reference.
+*   Grant the user / identity / service principal (depends on your authentication method, see below) access to the storage account. At minimum, a **Storage Blob Data Reader** role is needed. See [this doc](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-data-operations-portal) for reference.
 
 #### Authentication
 

{cocoindex-0.1.66 → cocoindex-0.1.68}/docs/src/css/custom.css

@@ -66,6 +66,7 @@
 
   code {
     font-size: var(--ifm-code-font-size);
+    border: none;
   }
 
   a {

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/amazon_s3_embedding/main.py

@@ -101,7 +101,13 @@ def _main() -> None:
     pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
 
     amazon_s3_text_embedding_flow.setup()
-    with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow):
+    with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow) as updater:
+        while True:
+            updates = updater.next_status_updates()
+            print(f"Updates: {updates}")
+            if not updates.active_sources:
+                break
+
         # Run queries in a loop to demonstrate the query capabilities.
         while True:
             query = input("Enter search query (or Enter to quit): ")

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/amazon_s3_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "amazon-s3-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Amazon S3 files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/azure_blob_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "azure-blob-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Azure Blob Storage files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/code_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,7 +3,7 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.63"]
+dependencies = ["cocoindex>=0.1.67"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/image_search/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on images."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.63",
+    "cocoindex>=0.1.67",
     "python-dotenv>=1.0.1",
     "fastapi>=0.100.0",
     "torch>=2.0.0",

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/manuals_llm_extraction/pyproject.toml

@@ -3,7 +3,7 @@ name = "manuals-llm-extraction"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.63", "marker-pdf>=1.5.2"]
+dependencies = ["cocoindex>=0.1.67", "marker-pdf>=1.5.2"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/paper_metadata/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Build index for papers with both metadata and content embeddings"
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex[embeddings]>=0.1.63",
+    "cocoindex[embeddings]>=0.1.67",
     "pypdf>=5.7.0",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/patient_intake_extraction/pyproject.toml

@@ -4,8 +4,8 @@ version = "0.1.0"
 description = "Extract structured information from patient intake forms using LLM."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.63",
+    "cocoindex>=0.1.67",
     "python-dotenv>=1.0.1",
     "markitdown>=0.1.2",
-    "openai>=1.68.2"
+    "openai>=1.68.2",
 ]

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex[embeddings]>=0.1.63",
+    "cocoindex[embeddings]>=0.1.67",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/product_recommendation/pyproject.toml

@@ -3,7 +3,7 @@ name = "cocoindex-ecommerce-taxonomy"
 version = "0.1.0"
 description = "Simple example for CocoIndex: extract taxonomy from e-commerce products and build knowledge graph."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.63", "jinja2>=3.1.6"]
+dependencies = ["cocoindex>=0.1.67", "jinja2>=3.1.6"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/text_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex[embeddings]>=0.1.63",
+    "cocoindex[embeddings]>=0.1.67",
     "python-dotenv>=1.0.1",
     "pgvector>=0.4.1",
     "psycopg[binary,pool]",

{cocoindex-0.1.66 → cocoindex-0.1.68}/examples/text_embedding_qdrant/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex[embeddings]>=0.1.63",
+    "cocoindex[embeddings]>=0.1.67",
     "python-dotenv>=1.0.1",
     "qdrant-client>=1.6.0",
 ]

{cocoindex-0.1.66 → cocoindex-0.1.68}/pyproject.toml

@@ -29,9 +29,9 @@ module-name = "cocoindex._engine"
 features = ["pyo3/extension-module"]
 
 [project.optional-dependencies]
-test = ["pytest"]
-dev = ["ruff", "pre-commit"]
+dev = ["pytest", "ruff", "mypy", "pre-commit"]
 embeddings = ["sentence-transformers>=3.3.1"]
+all = ["cocoindex[embeddings]"]
 
 [tool.mypy]
 python_version = "3.11"

{cocoindex-0.1.66 → cocoindex-0.1.68}/python/cocoindex/__init__.py

@@ -10,7 +10,7 @@ from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
-from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions
+from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions, FlowUpdaterStatusUpdates
 from .flow import add_flow_def, remove_flow
 from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
 from .lib import init, start_server, stop
@@ -54,6 +54,7 @@ __all__ = [
     "GeneratedField",
     "FlowLiveUpdater",
     "FlowLiveUpdaterOptions",
+    "FlowUpdaterStatusUpdates",
     "add_flow_def",
     "remove_flow",
     "update_all_flows_async",

{cocoindex-0.1.66 → cocoindex-0.1.68}/python/cocoindex/flow.py

@@ -534,6 +534,18 @@ class FlowLiveUpdaterOptions:
     print_stats: bool = False
 
 
+class FlowUpdaterStatusUpdates(NamedTuple):
+    """
+    Status updates for a flow updater.
+    """
+
+    # Sources that are still active, i.e. not stopped processing.
+    active_sources: list[str]
+
+    # Sources with updates since last time.
+    updated_sources: list[str]
+
+
 class FlowLiveUpdater:
     """
     A live updater for a flow.
@@ -587,7 +599,26 @@ class FlowLiveUpdater:
         """
         Wait for the live updater to finish. Async version.
         """
-        await self._get_engine_live_updater().wait()
+        await self._get_engine_live_updater().wait_async()
+
+    def next_status_updates(self) -> FlowUpdaterStatusUpdates:
+        """
+        Get the next status updates.
+
+        It blocks until there's a new status updates, including the processing finishes for a bunch of source updates,
+        and live updater stops (aborted, or no more sources to process).
+        """
+        return execution_context.run(self.next_status_updates_async())
+
+    async def next_status_updates_async(self) -> FlowUpdaterStatusUpdates:
+        """
+        Get the next status updates. Async version.
+        """
+        updates = await self._get_engine_live_updater().next_status_updates_async()
+        return FlowUpdaterStatusUpdates(
+            active_sources=updates.active_sources,
+            updated_sources=updates.updated_sources,
+        )
 
     def abort(self) -> None:
         """
@@ -879,10 +910,7 @@ def update_all_flows(
     """
     Update all flows.
     """
-    return cast(
-        dict[str, _engine.IndexUpdateInfo],
-        execution_context.run(update_all_flows_async(options)),
-    )
+    return execution_context.run(update_all_flows_async(options))
 
 
 async def update_all_flows_async(
@@ -958,7 +986,7 @@ class TransformFlow(Generic[T]):
     def _flow_info(self) -> TransformFlowInfo:
         if self._lazy_flow_info is not None:
             return self._lazy_flow_info
-        return cast(TransformFlowInfo, execution_context.run(self._flow_info_async()))
+        return execution_context.run(self._flow_info_async())
 
     async def _flow_info_async(self) -> TransformFlowInfo:
         if self._lazy_flow_info is not None:
@@ -1037,7 +1065,7 @@ class TransformFlow(Generic[T]):
         """
         Evaluate the transform flow.
         """
-        return cast(T, execution_context.run(self.eval_async(*args, **kwargs)))
+        return execution_context.run(self.eval_async(*args, **kwargs))
 
     async def eval_async(self, *args: Any, **kwargs: Any) -> T:
         """