cocoindex 0.1.67__tar.gz → 0.1.69__tar.gz

{cocoindex-0.1.67 → cocoindex-0.1.69}/.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
     hooks:
         - id: maturin-develop
           name: maturin develop
-          entry: maturin develop -E all
+          entry: maturin develop -E all,dev
           language: system
           files: ^(python/|src/|Cargo\.toml|pyproject\.toml)
           pass_filenames: false

{cocoindex-0.1.67 → cocoindex-0.1.69}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.67"
+version = "0.1.69"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -1369,6 +1369,7 @@ dependencies = [
  "tree-sitter-java",
  "tree-sitter-javascript",
  "tree-sitter-json",
+ "tree-sitter-kotlin-ng",
  "tree-sitter-language",
  "tree-sitter-md",
  "tree-sitter-pascal",
@@ -5724,6 +5725,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-kotlin-ng"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e800ebbda938acfbf224f4d2c34947a31994b1295ee6e819b65226c7b51b4450"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree-sitter-language"
 version = "0.1.5"

{cocoindex-0.1.67 → cocoindex-0.1.69}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.67"
+version = "0.1.69"
 edition = "2024"
 rust-version = "1.88"
 
@@ -63,6 +63,7 @@ reqwest = { version = "0.12.15", default-features = false, features = [
     "rustls-tls",
 ] }
 async-openai = "0.28.0"
+
 tree-sitter = "0.25.3"
 tree-sitter-language = "0.1.5"
 # Per language tree-sitter parsers
@@ -76,6 +77,8 @@ tree-sitter-html = "0.23.2"
 tree-sitter-java = "0.23.5"
 tree-sitter-javascript = "0.23.1"
 tree-sitter-json = "0.24.8"
+# The other more popular crate tree-sitter-kotlin requires tree-sitter < 0.23 for now
+tree-sitter-kotlin-ng = "1.1.0"
 tree-sitter-md = "0.3.2"
 tree-sitter-pascal = "0.10.0"
 tree-sitter-php = "0.23.11"

{cocoindex-0.1.67 → cocoindex-0.1.69}/PKG-INFO

@@ -1,17 +1,17 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.67
+Version: 0.1.69
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: watchfiles>=1.1.0
 Requires-Dist: numpy>=1.23.2
-Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: ruff ; extra == 'dev'
+Requires-Dist: mypy ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
 Requires-Dist: cocoindex[embeddings] ; extra == 'all'
-Provides-Extra: test
 Provides-Extra: dev
 Provides-Extra: embeddings
 Provides-Extra: all
@@ -36,16 +36,36 @@ Project-URL: Homepage, https://cocoindex.io/
 [![License](https://img.shields.io/badge/license-Apache%202.0-5B5BD6?logoColor=white)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI version](https://img.shields.io/pypi/v/cocoindex?color=5B5BD6)](https://pypi.org/project/cocoindex/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cocoindex)](https://pypistats.org/packages/cocoindex)
-
 [![CI](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
 [![release](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
+
+</div>
+
+<div align="center">
+    <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>
 
+
 Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box.  Exceptional developer velocity. Production-ready at day 0.
 
 ⭐ Drop a star to help us grow!
 
+<div align="center">
+    
+<!-- Keep these links. Translations will automatically update with the README. -->
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
+    
+</div>
+
 </br>
 
 <p align="center">
@@ -59,7 +79,7 @@ CocoIndex makes it super easy to transform data with AI workloads, and keep sour
 </br>
 
 <p align="center">
-    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='480'>
+    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='400'>
 </p>
 
 </br>

{cocoindex-0.1.67 → cocoindex-0.1.69}/README.md

@@ -11,16 +11,36 @@
 [![License](https://img.shields.io/badge/license-Apache%202.0-5B5BD6?logoColor=white)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI version](https://img.shields.io/pypi/v/cocoindex?color=5B5BD6)](https://pypi.org/project/cocoindex/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cocoindex)](https://pypistats.org/packages/cocoindex)
-
 [![CI](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
 [![release](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
+
+</div>
+
+<div align="center">
+    <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>
 
+
 Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box.  Exceptional developer velocity. Production-ready at day 0.
 
 ⭐ Drop a star to help us grow!
 
+<div align="center">
+    
+<!-- Keep these links. Translations will automatically update with the README. -->
+[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) | 
+[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) | 
+[Español](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) | 
+[français](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) | 
+[日本語](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) | 
+[한국어](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) | 
+[Português](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) | 
+[Русский](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) | 
+[中文](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
+    
+</div>
+
 </br>
 
 <p align="center">
@@ -34,7 +54,7 @@ CocoIndex makes it super easy to transform data with AI workloads, and keep sour
 </br>
 
 <p align="center">
-    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='480'>
+    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='400'>
 </p>
 
 </br>

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/about/community.md

@@ -9,7 +9,7 @@ Welcome with a huge coconut hug 🥥⋆｡˚🤗.
 
 We are super excited for community contributions of all kinds - whether it's code improvements, documentation updates, issue reports, feature requests on [GitHub](https://github.com/cocoindex-io/cocoindex), and discussions in our [Discord](https://discord.com/invite/zpA9S2DR7s).
 
-We would love to fostering an inclusive, welcoming, and supportive environment. Contributing to CocoIndex should feel collaborative, friendly and enjoyable for everyone. Together, we can build better AI applications through robust data infrastructure.
+We would love to foster an inclusive, welcoming, and supportive environment. Contributing to CocoIndex should feel collaborative, friendly and enjoyable for everyone. Together, we can build better AI applications through robust data infrastructure.
 
 :::tip Start hacking CocoIndex
 Check out our [Contributing guide](./contributing) to get started!

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/about/contributing.md

@@ -18,10 +18,10 @@ We tag issues with the ["good first issue"](https://github.com/cocoindex-io/coco
 ## How to Contribute
 - If you decide to work on an issue, unless the PR can be sent immediately (e.g. just a few lines of code), we recommend you to leave a comment on the issue like **`I'm working on it`**  or **`Can I work on this issue?`** to avoid duplicating work.
 - For larger features, we recommend you to discuss with us first in our [Discord server](https://discord.com/invite/zpA9S2DR7s) to coordinate the design and work.
-- Our [Discord server](https://discord.com/invite/zpA9S2DR7s) are constantly open. If you are unsure about anything, it is a good place to discuss! We'd love to collaborate and will always be friendly.
+- Our [Discord server](https://discord.com/invite/zpA9S2DR7s) is constantly open. If you are unsure about anything, it is a good place to discuss! We'd love to collaborate and will always be friendly.
 
 ## Start hacking! Setting Up Development Environment
-Following the steps below to get cocoindex build on latest codebase locally - if you are making changes to cocoindex funcionality and want to test it out.
+Follow the steps below to get cocoindex built on the latest codebase locally - if you are making changes to cocoindex functionality and want to test it out.
 
 -   🦀 [Install Rust](https://rust-lang.org/tools/install)
 
@@ -46,12 +46,12 @@ Following the steps below to get cocoindex build on latest codebase locally - if
 
 -   Install required tools:
     ```sh
-    pip install maturin mypy pre-commit
+    pip install maturin
     ```
 
 -   Build the library. Run at the root of cocoindex directory:
     ```sh
-    maturin develop
+    maturin develop -E all,dev
     ```
 
 -   Install and enable pre-commit hooks. This ensures all checks run automatically before each commit:

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/core/data_types.mdx

@@ -28,8 +28,6 @@ When you define a [custom function](/docs/core/custom_function), you need to ann
     we use the type annotation as a guidance to construct the Python value.
     Type annotation is optional for basic types and struct types, and required for table types.
 
-*   When you define a [custom function](/docs/core/custom_function), type annotation is required for arguments and return values.
-
 ### Basic Types
 
 #### Primitive Types

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/core/flow_def.mdx

@@ -57,8 +57,7 @@ After it's called, `demo_flow` becomes an invalid object, and you should not cal
 :::note
 
 This only removes states of the flow from the current process, and it won't affect the persistent states.
-
-If you w
+See [Setup / drop flow](/docs/core/flow_methods#setup--drop-flow) if you want to clean up the persistent states.
 
 :::
 
@@ -469,7 +468,7 @@ Then reference it when building a spec that takes an auth entry:
 Note that CocoIndex backends use the key of an auth entry to identify the backend.
 
 *   Keep the key stable.
-    If the key doesn't change, it's considered to be the same backend (even if the underlying way to connect/authenticate change).
+    If the key doesn't change, it's considered to be the same backend (even if the underlying way to connect/authenticate changes).
 
 *   If a key is no longer referenced in any operation spec, keep it until the next flow setup / drop action,
-    so that cocoindex will be able to clean up the backends.
+    so that CocoIndex will be able to clean up the backends.

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/core/flow_methods.mdx

@@ -44,9 +44,9 @@ For a flow, its persistent backends need to be ready before it can run, includin
 The desired state of the backends for a flow is derived based on the flow definition itself.
 CocoIndex supports two types of actions to manage the persistent backends automatically:
 
-*   *Setup* a flow, which will change the backends owned by the flow to a state to the desired state, e.g. create new tables for new flow, drop an existing table if the corresponding target is gone, add new column to a target table if a new field is collected, etc. It's no-op if the backend states are already in the desired state.
+*   *Setup* a flow, which will change the backends owned by the flow to the desired state, e.g. create new tables for new flow, drop an existing table if the corresponding target is gone, add new column to a target table if a new field is collected, etc. It's no-op if the backend states are already in the desired state.
 
-*   *Drop* a flow, which will drop all backends owned by the flow. It's no-op if there's no existing backends owned by the flow (e.g. never setup or already dropped).
+*   *Drop* a flow, which will drop all backends owned by the flow. It's no-op if there are no existing backends owned by the flow (e.g. never setup or already dropped).
 
 ### CLI
 
@@ -138,7 +138,7 @@ This is to achieve best efficiency.
 
 The `cocoindex update` subcommand creates/updates data in the target.
 
-Once it's done, the target data is fresh up to the moment when the function is called.
+Once it's done, the target data is fresh up to the moment when the command is called.
 
 ```sh
 cocoindex update main.py
@@ -203,7 +203,7 @@ To perform live update, run the `cocoindex update` subcommand with `-L` option:
 cocoindex update main.py -L
 ```
 
-If there's at least one data source with change capture mechanism enabled, it will keep running until the aborted (e.g. by `Ctrl-C`).
+If there's at least one data source with change capture mechanism enabled, it will keep running until aborted (e.g. by `Ctrl-C`).
 Otherwise, it falls back to the same behavior as one time update, and will finish after a one-time update is done.
 
 With a `--setup` option, it will also setup the flow first if needed.
@@ -241,13 +241,27 @@ A `FlowLiveUpdater` object supports the following methods:
 
 *   `start()`: Start the updater.
     CocoIndex will continuously capture changes from the source data and update the target data accordingly in background threads managed by the engine.
+
 *   `abort()`: Abort the updater.
+
 *   `wait()`: Wait for the updater to finish. It only unblocks in one of the following cases:
     *   The updater was aborted.
     *   A one time update is done, and live update is not enabled:
         either `live_mode` is `False`, or all data sources have no change capture mechanisms enabled.
+
+*   `next_status_updates()`: Get the next status updates.
+    It blocks until there's a new status updates, including the processing finishes for a bunch of source updates, and live updater stops (aborted, or no more sources to process).
+    You can continuously call this method in a loop to get the latest status updates and react accordingly.
+
+    It returns a `cocoindex.FlowUpdaterStatusUpdates` object, with the following properties:
+    *   `active_sources`: Names of sources that are still active, i.e. not stopped processing. If it's empty, it means the updater is stopped.
+    *   `updated_sources`: Names of sources with updates since last time.
+        You can check this to see which sources have recent updates and get processed.
+
 *   `update_stats()`: It returns the stats of the updater.
 
+This snippets shows the lifecycle of a live updater:
+
 ```python
 my_updater = cocoindex.FlowLiveUpdater(demo_flow)
 # Start the updater.
@@ -256,14 +270,37 @@ my_updater.start()
 # Perform your own logic (e.g. a query loop).
 ...
 
-# Print the update stats.
-print(my_updater.update_stats())
-# Abort the updater.
-my_updater.abort()
+...
 # Wait for the updater to finish.
 my_updater.wait()
+# Print the update stats.
+print(my_updater.update_stats())
 ```
 
+Somewhere (in the same or other threads) you can also continuously call `next_status_updates()` to get the latest status updates and react accordingly, e.g.
+
+```python
+while True:
+    updates = my_updater.next_status_updates()
+
+    for source in updates.updated_sources:
+        # Perform downstream operations on the target of the source.
+        run_your_downstream_operations_for(source)
+
+    # Break the loop if there's no more active sources.
+    if not updates.active_sources:
+        break
+```
+
+:::info
+
+`next_status_updates()` automatically combines multiple status updates if more than one arrives between two calls,
+e.g. your downstream operations may take more time, or you don't need to process too frequently (in which case you can explicitly sleep for a while).
+
+So you don't need to worry about the status updates piling up.
+
+:::
+
 Python SDK also allows you to use the updater as a context manager.
 It will automatically start the updater during the context entry, and abort and wait for the updater to finish automatically when the context is exited.
 The following code is equivalent to the code above (if no early return happens):
@@ -272,7 +309,6 @@ The following code is equivalent to the code above (if no early return happens):
 with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
     # Perform your own logic (e.g. a query loop).
     ...
-    print(my_updater.update_stats())
 ```
 
 CocoIndex also provides asynchronous versions of APIs for blocking operations, including:
@@ -287,20 +323,27 @@ CocoIndex also provides asynchronous versions of APIs for blocking operations, i
     # Perform your own logic (e.g. a query loop).
     ...
 
-    # Print the update stats.
-    print(my_updater.update_stats())
-    # Abort the updater.
-    my_updater.abort()
     # Wait for the updater to finish.
     await my_updater.wait_async()
+    # Print the update stats.
+    print(my_updater.update_stats())
+    ```
+
+*   `next_status_updates_async()`, e.g.
+
+    ```python
+    while True:
+        updates = await my_updater.next_status_updates_async()
+
+        ...
     ```
+
 *   Async context manager, e.g.
 
     ```python
     async with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
         # Perform your own logic (e.g. a query loop).
         ...
-        print(my_updater.update_stats())
     ```
 
 </TabItem>

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/getting_started/quickstart.md

@@ -7,10 +7,10 @@ import ReactPlayer from 'react-player'
 
 # Build your first CocoIndex project
 
-This guide will help you get up and running with CocoIndex in just a few minutes, that does:
+This guide will help you get up and running with CocoIndex in just a few minutes. We'll build a project that does:
 *   Read files from a directory
 *   Perform basic chunking and embedding
-*   loads the data into a vector store (PG Vector)
+*   Load the data into a vector store (PG Vector)
 
 <ReactPlayer controls url='https://www.youtube.com/watch?v=gv5R8nOXsWU' />
 
@@ -107,11 +107,11 @@ Notes:
 3.  A *data source* extracts data from an external source.
     In this example, the `LocalFile` data source imports local files as a KTable (table with a key field, see [KTable](../core/data_types#ktable) for details), each row has `"filename"` and `"content"` fields.
 
-4. After defining the KTable, we extended a new field `"chunks"` to each row by *transforming* the `"content"` field using `SplitRecursively`. The output of the `SplitRecursively` is also a KTable representing each chunk of the document, with `"location"` and `"text"` fields.
+4. After defining the KTable, we extend a new field `"chunks"` to each row by *transforming* the `"content"` field using `SplitRecursively`. The output of the `SplitRecursively` is also a KTable representing each chunk of the document, with `"location"` and `"text"` fields.
 
-5. After defining the KTable, we extended a new field `"embedding"` to each row by *transforming* the `"text"` field using `SentenceTransformerEmbed`.
+5. After defining the KTable, we extend a new field `"embedding"` to each row by *transforming* the `"text"` field using `SentenceTransformerEmbed`.
 
-6. In CocoIndex, a *collector* collects multiple entries of data together. In this example, the `doc_embeddings` collector collects data from all `chunk`s across all `doc`s, and using the collected data to build a vector index `"doc_embeddings"`, using `Postgres`.
+6. In CocoIndex, a *collector* collects multiple entries of data together. In this example, the `doc_embeddings` collector collects data from all `chunk`s across all `doc`s, and uses the collected data to build a vector index `"doc_embeddings"`, using `Postgres`.
 
 ## Step 3: Run the indexing pipeline and queries
 
@@ -271,7 +271,7 @@ Now we can run the same Python file, which will run the new added main logic:
 python quickstart.py
 ```
 
-It will ask you to enter a query and it will return the top 10 results.
+It will ask you to enter a query and it will return the top 5 results.
 
 ## Next Steps
 

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/ops/functions.md

@@ -31,7 +31,7 @@ The spec takes the following fields:
 
     *   `separators_regex` (`list[str]`): A list of regex patterns to split the text.
         Higher-level boundaries should come first, and lower-level should be listed later. e.g. `[r"\n# ", r"\n## ", r"\n\n", r"\. "]`.
-        See [regex Syntax](https://docs.rs/regex/latest/regex/#syntax) for supported regular expression syntax.
+        See [regex syntax](https://docs.rs/regex/latest/regex/#syntax) for supported regular expression syntax.
 
 Input data:
 
@@ -57,9 +57,12 @@ Input data:
 
     We use the `language` field to determine how to split the input text, following these rules:
 
-    *   We'll match the input `language` field against the `language_name` or `aliases` of each element of `custom_languages`, and use the matched one. If value of `language` is null, it'll be treated as empty string when matching `language_name` or `aliases`.
-    *   If no match is found, we'll match the `language` field against the builtin language configurations.
-        For all supported builtin language names and aliases (extensions), see [the code](https://github.com/search?q=org%3Acocoindex-io+lang%3Arust++%22static+TREE_SITTER_LANGUAGE_BY_LANG%22&type=code).
+    *   We match the input `language` field against the following registries in the following order:
+        *   `custom_languages` in the spec, against the `language_name` or `aliases` field of each entry.
+        *   Builtin languages (see [Supported Languages](#supported-languages) section below), against the language, aliases or file extensions of each entry.
+
+        All matches are in a case-insensitive manner. If the value of `language` is null, it'll be treated as empty string.
+
     *   If no match is found, the input will be treated as plain text.
 
     :::
@@ -73,6 +76,42 @@ Return: [*KTable*](/docs/core/data_types#ktable), each row represents a chunk, w
     *   `line` (*Int64*): The line number of the position. Starting from 1.
     *   `column` (*Int64*): The column number of the position. Starting from 1.
 
+### Supported Languages
+
+Currently, `SplitRecursively` supports the following languages:
+
+| Language | Aliases | File Extensions |
+|----------|---------|-----------------|
+| C | | `.c` |
+| C++ | CPP | `.cpp`, `.cc`, `.cxx`, `.h`, `.hpp` |
+| C# | CSharp, CS | `.cs` |
+| CSS | | `.css`, `.scss` |
+| DTD | | `.dtd` |
+| Fortran | F, F90, F95, F03 | `.f`, `.f90`, `.f95`, `.f03` |
+| Go | Golang | `.go` |
+| HTML | | `.html`, `.htm` |
+| Java | | `.java` |
+| JavaScript | JS | `.js` |
+| JSON | | `.json` |
+| Kotlin | | `.kt`, `.kts` |
+| Markdown | MD | `.md`, `.mdx` |
+| Pascal | PAS, DPR, Delphi | `.pas`, `.dpr` |
+| PHP | | `.php` |
+| Python | | `.py` |
+| R | | `.r` |
+| Ruby | | `.rb` |
+| Rust | RS | `.rs` |
+| Scala | | `.scala` |
+| SQL | | `.sql` |
+| Swift | | `.swift` |
+| TOML | | `.toml` |
+| TSX | | `.tsx` |
+| TypeScript | TS | `.ts` |
+| XML | | `.xml` |
+| YAML | | `.yaml`, `.yml` |
+
+
+
 ## SentenceTransformerEmbed
 
 `SentenceTransformerEmbed` embeds a text into a vector space using the [SentenceTransformer](https://huggingface.co/sentence-transformers) library.

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/ops/sources.md

@@ -111,10 +111,9 @@ This is how to setup:
 
 *   In the [Amazon S3 Console](https://s3.console.aws.amazon.com/s3/home), open your S3 bucket. Under *Properties* tab, click *Create event notification*.
     *   Fill in an arbitrary event name, e.g. `S3ChangeNotifications`.
-    *   If you want your AmazonS3 data source expose a subset of files sharing a prefix, set the same prefix here. Otherwise, leave it empty.
+    *   If you want your AmazonS3 data source to expose a subset of files sharing a prefix, set the same prefix here. Otherwise, leave it empty.
     *   Select the following event types: *All object create events*, *All object removal events*.
     *   Select *SQS queue* as the destination, and specify the SQS queue you created above.
-and enable *Change Event Notifications* for your bucket, and specify the SQS queue as the destination.
 
 AWS's [Guide of Configuring a Bucket for Notifications](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html#step1-create-sqs-queue-for-notification) provides more details.
 
@@ -141,7 +140,7 @@ The spec takes the following fields:
     :::info
 
     We will delete messages from the queue after they're processed.
-    If there're unrelated messages in the queue (e.g. test messages that SQS will send automatically on queue creation, messages for a different bucket, for non-included files, etc.), we will delete the message upon receiving it, to avoid keeping receiving irrelevant messages again and again after they're redelivered.
+    If there are unrelated messages in the queue (e.g. test messages that SQS will send automatically on queue creation, messages for a different bucket, for non-included files, etc.), we will delete the message upon receiving it, to avoid repeatedly receiving irrelevant messages after they're redelivered.
 
     :::
 
@@ -167,7 +166,7 @@ These are actions you need to take:
 *   Create a storage account in the [Azure Portal](https://portal.azure.com/).
 *   Create a container in the storage account.
 *   Upload your files to the container.
-*   Grant the user / identity / service principal (depends on your authentication method, see below) access to the container. At minimum, a **Storage Blob Data Reader** role is needed. See [this doc](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-data-operations-portal) for reference.
+*   Grant the user / identity / service principal (depends on your authentication method, see below) access to the storage account. At minimum, a **Storage Blob Data Reader** role is needed. See [this doc](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-data-operations-portal) for reference.
 
 #### Authentication
 
@@ -253,12 +252,12 @@ The spec takes the following fields:
     it's typically cheaper than a full refresh by setting the [refresh interval](../core/flow_def#refresh-interval) especially when the folder contains a large number of files.
     So you can usually set it with a smaller value compared to the `refresh_interval`.
 
-    On the other hand, this only detects changes for files still exists.
-    If the file is deleted (or the current account no longer has access to), this change will not be detected by this change stream.
+    On the other hand, this only detects changes for files that still exist.
+    If the file is deleted (or the current account no longer has access to it), this change will not be detected by this change stream.
 
-    So when a `GoogleDrive` source enabled `recent_changes_poll_interval`, it's still recommended to set a `refresh_interval`, with a larger value.
+    So when a `GoogleDrive` source has `recent_changes_poll_interval` enabled, it's still recommended to set a `refresh_interval`, with a larger value.
     So that most changes can be covered by polling recent changes (with low latency, like 10 seconds), and remaining changes (files no longer exist or accessible) will still be covered (with a higher latency, like 5 minutes, and should be larger if you have a huge number of files like 1M).
-    In reality, configure them based on your requirement: how freshness do you need to target index to be?
+    In reality, configure them based on your requirement: how fresh do you need the target index to be?
 
     :::
 

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/docs/ops/targets.md

@@ -413,7 +413,7 @@ If you don't have a Neo4j database, you can start a Neo4j database using our doc
 docker compose -f <(curl -L https://raw.githubusercontent.com/cocoindex-io/cocoindex/refs/heads/main/dev/neo4j.yaml) up -d
 ```
 
-If will bring up a Neo4j instance, which can be accessed by username `neo4j` and password `cocoindex`.
+This will bring up a Neo4j instance, which can be accessed by username `neo4j` and password `cocoindex`.
 You can access the Neo4j browser at [http://localhost:7474](http://localhost:7474).
 
 :::warning

{cocoindex-0.1.67 → cocoindex-0.1.69}/docs/src/css/custom.css

@@ -66,6 +66,7 @@
 
   code {
     font-size: var(--ifm-code-font-size);
+    border: none;
   }
 
   a {

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/amazon_s3_embedding/main.py

@@ -101,7 +101,13 @@ def _main() -> None:
     pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
 
     amazon_s3_text_embedding_flow.setup()
-    with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow):
+    with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow) as updater:
+        while True:
+            updates = updater.next_status_updates()
+            print(f"Updates: {updates}")
+            if not updates.active_sources:
+                break
+
         # Run queries in a loop to demonstrate the query capabilities.
         while True:
             query = input("Enter search query (or Enter to quit): ")

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/amazon_s3_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "amazon-s3-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Amazon S3 files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/azure_blob_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "azure-blob-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Azure Blob Storage files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/code_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,7 +3,7 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.63"]
+dependencies = ["cocoindex>=0.1.67"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.67 → cocoindex-0.1.69}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex[embeddings]>=0.1.63", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.67", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []