cocoindex 0.2.17__tar.gz → 0.2.19__tar.gz

{cocoindex-0.2.17 → cocoindex-0.2.19}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.17"
+version = "0.2.19"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -2573,6 +2573,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -3721,6 +3731,7 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b"
 dependencies = [
+ "half",
  "sqlx",
 ]
 
@@ -5845,9 +5856,9 @@ checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8"
 
 [[package]]
 name = "tree-sitter-md"
-version = "0.3.2"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17f968c22a01010b83fc960455ae729db08dbeb6388617d9113897cb9204b030"
+checksum = "2b55ea8733e098490746a07d6f629d1f7820e8953a4aab1341ae39123bcdf93d"
 dependencies = [
  "cc",
  "tree-sitter-language",

{cocoindex-0.2.17 → cocoindex-0.2.19}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.17"
+version = "0.2.19"
 edition = "2024"
 rust-version = "1.89"
 license = "Apache-2.0"
@@ -62,7 +62,7 @@ tower = "0.5.2"
 tower-http = { version = "0.6.6", features = ["cors", "trace"] }
 indexmap = { version = "2.10.0", features = ["serde"] }
 blake2 = "0.10.6"
-pgvector = { version = "0.4.1", features = ["sqlx"] }
+pgvector = { version = "0.4.1", features = ["sqlx", "halfvec"] }
 phf = { version = "0.12.1", features = ["macros"] }
 indenter = "0.3.4"
 itertools = "0.14.0"
@@ -91,7 +91,7 @@ tree-sitter-javascript = "0.23.1"
 tree-sitter-json = "0.24.8"
 # The other more popular crate tree-sitter-kotlin requires tree-sitter < 0.23 for now
 tree-sitter-kotlin-ng = "1.1.0"
-tree-sitter-md = "0.3.2"
+tree-sitter-md = "0.5.1"
 tree-sitter-pascal = "0.10.0"
 tree-sitter-php = "0.23.11"
 tree-sitter-python = "0.23.6"

{cocoindex-0.2.17 → cocoindex-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.17
+Version: 0.2.19
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent

{cocoindex-0.2.17 → cocoindex-0.2.19}/THIRD_PARTY_NOTICES.html

@@ -44,8 +44,8 @@
 
         <h2>Overview of licenses:</h2>
         <ul class="licenses-overview">
-            <li><a href="#Apache-2.0">Apache License 2.0</a> (415)</li>
-            <li><a href="#MIT">MIT License</a> (125)</li>
+            <li><a href="#Apache-2.0">Apache License 2.0</a> (416)</li>
+            <li><a href="#MIT">MIT License</a> (126)</li>
             <li><a href="#Unicode-3.0">Unicode License v3</a> (19)</li>
             <li><a href="#ISC">ISC License</a> (8)</li>
             <li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (6)</li>
@@ -2428,7 +2428,7 @@ Software.
                 <h3 id="Apache-2.0">Apache License 2.0</h3>
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
-                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.17</a></li>
+                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.19</a></li>
                     <li><a href=" https://github.com/awesomized/crc-fast-rust ">crc-fast 1.3.0</a></li>
                     <li><a href=" https://github.com/qdrant/rust-client ">qdrant-client 1.15.0</a></li>
                 </ul>
@@ -9559,6 +9559,7 @@ limitations under the License.
                     <li><a href=" https://github.com/googleapis/google-cloud-rust/tree/main ">google-cloud-rpc 0.4.5</a></li>
                     <li><a href=" https://github.com/googleapis/google-cloud-rust/tree/main ">google-cloud-type 0.4.5</a></li>
                     <li><a href=" https://github.com/googleapis/google-cloud-rust/tree/main ">google-cloud-wkt 0.5.4</a></li>
+                    <li><a href=" https://github.com/VoidStarKat/half-rs ">half 2.6.0</a></li>
                     <li><a href=" https://github.com/TedDriggs/ident_case ">ident_case 1.0.1</a></li>
                     <li><a href=" https://github.com/yaahc/indenter ">indenter 0.3.4</a></li>
                     <li><a href=" https://github.com/dtolnay/indoc ">indoc 2.0.6</a></li>
@@ -11931,7 +11932,7 @@ SOFTWARE.
                     <li><a href=" https://github.com/tree-sitter/tree-sitter-json ">tree-sitter-json 0.24.8</a></li>
                     <li><a href=" https://github.com/tree-sitter-grammars/tree-sitter-kotlin ">tree-sitter-kotlin-ng 1.1.0</a></li>
                     <li><a href=" https://github.com/tree-sitter/tree-sitter ">tree-sitter-language 0.1.5</a></li>
-                    <li><a href=" https://github.com/tree-sitter-grammars/tree-sitter-markdown ">tree-sitter-md 0.3.2</a></li>
+                    <li><a href=" https://github.com/tree-sitter-grammars/tree-sitter-markdown ">tree-sitter-md 0.5.1</a></li>
                     <li><a href=" https://github.com/Isopod/tree-sitter-pascal ">tree-sitter-pascal 0.10.0</a></li>
                     <li><a href=" https://github.com/tree-sitter/tree-sitter-php ">tree-sitter-php 0.23.11</a></li>
                     <li><a href=" https://github.com/tree-sitter/tree-sitter-python ">tree-sitter-python 0.23.6</a></li>
@@ -12174,6 +12175,35 @@ furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+</pre>
+            </li>
+            <li class="license">
+                <h3 id="MIT">MIT License</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    <li><a href=" https://github.com/eira-fransham/crunchy ">crunchy 0.2.4</a></li>
+                </ul>
+                <pre class="license-text">The MIT License (MIT)
+
+Copyright 2017-2023 Eira Fransham.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the &quot;Software&quot;), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
 THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/core/flow_def.mdx

@@ -313,9 +313,9 @@ Types of the fields must be key types. See [Key Types](data_types#key-types) for
 
 *Vector index* is specified by `vector_indexes` (`Sequence[VectorIndexDef]`). `VectorIndexDef` has the following fields:
 
-    *   `field_name`: the field to create vector index.
-    *   `metric`: the similarity metric to use.
-    *   `method` (optional): the index algorithm and optional tuning parameters. Leave unset to use the target default (HNSW for Postgres). Use `cocoindex.HnswVectorIndexMethod()` or `cocoindex.IvfFlatVectorIndexMethod()` to customize the method and its parameters.
+* `field_name`: the field to create vector index.
+* `metric`: the similarity metric to use.
+* `method` (optional): the index algorithm and optional tuning parameters. Leave unset to use the target default (HNSW for Postgres). Use `cocoindex.HnswVectorIndexMethod()` or `cocoindex.IvfFlatVectorIndexMethod()` to customize the method and its parameters.
 
 #### Similarity Metrics
 

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/core/flow_methods.mdx

@@ -210,7 +210,7 @@ A data source may enable one or multiple *change capture mechanisms*:
 *   Configured with a [refresh interval](flow_def#refresh-interval), which is generally applicable to all data sources.
 
 *   Specific data sources also provide their specific change capture mechanisms.
-    For example, [`Postgres` source](../sources/#postgres) listens to PostgreSQL's change notifications, [`AmazonS3` source](../sources/#amazons3) watches S3 bucket's change events, and [`GoogleDrive` source](../sources#googledrive) allows polling recent modified files.
+    For example, [`Postgres` source](../sources/postgres) listens to PostgreSQL's change notifications, [`AmazonS3` source](../sources/amazons3) watches S3 bucket's change events, and [`GoogleDrive` source](../sources/googledrive) allows polling recent modified files.
     See documentations for specific data sources.
 
 Change capture mechanisms enable CocoIndex to continuously capture changes from the source data and update the target data accordingly, under live update mode.

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/core/settings.mdx

@@ -14,9 +14,8 @@ Certain settings need to be provided for CocoIndex to work, e.g. database connec
 
 Note that in general, you have two ways to launch CocoIndex:
 
-*   Call CocoIndex APIs from your own Python application or library.
-*   Use [Cocoindex CLI](cli). It's handy for most routine indexing building and management tasks.
-
+* Call CocoIndex APIs from your own Python application or library.
+* Use [Cocoindex CLI](cli). It's handy for most routine indexing building and management tasks.
 
 CocoIndex exposes process-level settings specified by `cocoindex.Settings` dataclass.
 Settings can be configured in three different ways.
@@ -77,10 +76,10 @@ But be careful that if you call `cocoindex.init()` only under the path of main (
 
 `cocoindex.init()` is optional:
 
--   You can call `cocoindex.init()` with a `cocoindex.Settings` dataclass object as argument, or without any argument.
+* You can call `cocoindex.init()` with a `cocoindex.Settings` dataclass object as argument, or without any argument.
     When without argument, the settings will be loaded from the `@cocoindex.settings` function or environment variables.
 
--   You don't have to explicitly call `cocoindex.init()`.
+* You don't have to explicitly call `cocoindex.init()`.
     CocoIndex will be automatically initialized when needed, e.g. when any method of any flow is called the first time.
     But calling `cocoindex.init()` explicitly (usually at startup time, e.g. in the main function of your application) has the benefit of making sure CocoIndex library is initialized and any potential exceptions are raised earlier before proceeding with the application.
     If you need this clarity, you can call it explicitly even if you don't want to provide settings by the `cocoindex.init()` call.
@@ -91,9 +90,9 @@ But be careful that if you call `cocoindex.init()` only under the path of main (
 
 `cocoindex.Settings` is a dataclass that contains the following fields:
 
-*   `app_namespace` (type: `str`, required): The namespace of the application.
-*   `database` (type: `DatabaseConnectionSpec`, required): The connection to the Postgres database.
-*   `global_execution_options` (type: `GlobalExecutionOptions`, optional): The global execution options shared by all  flows.
+* `app_namespace` (type: `str`, required): The namespace of the application.
+* `database` (type: `DatabaseConnectionSpec`, required): The connection to the Postgres database.
+* `global_execution_options` (type: `GlobalExecutionOptions`, optional): The global execution options shared by all  flows.
 
 ### App Namespace
 
@@ -110,15 +109,15 @@ If not set, all flows are in a default unnamed namespace.
 
 `DatabaseConnectionSpec` configures the connection to a database. Only Postgres is supported for now. It has the following fields:
 
-*   `url` (type: `str`): The URL of the Postgres database to use as the internal storage, e.g. `postgres://cocoindex:cocoindex@localhost/cocoindex`.
+* `url` (type: `str`): The URL of the Postgres database to use as the internal storage, e.g. `postgres://cocoindex:cocoindex@localhost/cocoindex`.
 
     *Environment variable* for `Settings.database.url`: `COCOINDEX_DATABASE_URL`
 
-*   `user` (type: `Optional[str]`, default: `None`): The username for the Postgres database. If not provided, username will come from `url`.
+* `user` (type: `Optional[str]`, default: `None`): The username for the Postgres database. If not provided, username will come from `url`.
 
     *Environment variable* for `Settings.database.user`: `COCOINDEX_DATABASE_USER`
 
-*   `password` (type: `Optional[str]`, default: `None`): The password for the Postgres database. If not provided, password will come from `url`.
+* `password` (type: `Optional[str]`, default: `None`): The password for the Postgres database. If not provided, password will come from `url`.
 
     *Environment variable* for `Settings.database.password`: `COCOINDEX_DATABASE_PASSWORD`
 
@@ -129,21 +128,24 @@ If not set, all flows are in a default unnamed namespace.
 
     :::
 
-*   `max_connections` (type: `int`, default: `25`): The maximum number of connections to keep in the pool.
+* `max_connections` (type: `int`, default: `25`): The maximum number of connections to keep in the pool.
 
     *Environment variable* for `Settings.database.max_connections`: `COCOINDEX_DATABASE_MAX_CONNECTIONS`
 
-*   `min_connections` (type: `int`, default: `5`): The minimum number of connections to keep in the pool.
+* `min_connections` (type: `int`, default: `5`): The minimum number of connections to keep in the pool.
 
     *Environment variable* for `Settings.database.min_connections`: `COCOINDEX_DATABASE_MIN_CONNECTIONS`
 
-
 :::info
 
 If you use the Postgres database hosted by [Supabase](https://supabase.com/), please click **Connect** on your project dashboard and find the following URL:
 
-*    If you're on a IPv6 network, use the URL under **Direct connection**. You can visit [IPv6 test](https://test-ipv6.com/) to see if you have IPv6 Internet connection.
-*    Otherwise, use the URL under **Session pooler**.
+* If you're on a IPv6 network, use the URL under **Direct connection**. You can visit [IPv6 test](https://test-ipv6.com/) to see if you have IPv6 Internet connection.
+* Otherwise, use the URL under **Session pooler**.
+  Note that Supabase has a pool size limit of 15 by default, while CocoIndex's default `max_connections` value is 25.
+  You can adjust either value to make sure Supabase's pool size limit is greater than CocoIndex's `max_connections` value.
+  Supabase's pool size limit can be adjusted under "Database" -> "Settings".
+* CocoIndex doesn't support *Transaction pooler* now.
 
 :::
 
@@ -151,8 +153,8 @@ If you use the Postgres database hosted by [Supabase](https://supabase.com/), pl
 
 `GlobalExecutionOptions` is used to configure the global execution options shared by all flows. It has the following fields:
 
-*   `source_max_inflight_rows` (type: `int | None`, default: `1024`): The maximum number of concurrent inflight rows for all source operations.
-*   `source_max_inflight_bytes` (type: `int | None`, default: `None`): The maximum number of concurrent inflight bytes for all source operations.
+* `source_max_inflight_rows` (type: `int | None`, default: `1024`): The maximum number of concurrent inflight rows for all source operations.
+* `source_max_inflight_bytes` (type: `int | None`, default: `None`): The maximum number of concurrent inflight bytes for all source operations.
 
 See also [flow definition docs](/docs/core/flow_def#control-processing-concurrency) about why it's necessary to control processing concurrency, and how to configure it on per-source basis.
 If both global and per-source limits are specified, both need to be satisfied to admit additional source rows.

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/examples/examples/image_search.md

@@ -66,7 +66,7 @@ def image_object_embedding_flow(flow_builder, data_scope):
 
 The `add_source` function sets up a table with fields like `filename` and `content`. Images are automatically re-scanned every minute.
 
-<DocumentationButton url="https://cocoindex.io/docs/ops/sources#localfile" text="LocalFile" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources/localfile" text="LocalFile" />
 
 
 ## Process Each Image and Collect the Embedding

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/examples/examples/multi_format_index.md

@@ -52,7 +52,7 @@ data_scope["documents"] = flow_builder.add_source(
     cocoindex.sources.LocalFile(path="source_files", binary=True)
 )
 ```
-<DocumentationButton url="https://cocoindex.io/docs/ops/sources#localfile" text="LocalFile" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources/localfile" text="LocalFile" margin="0 0 16px 0" />
 
 
 ## Convert Files to Pages

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/examples/examples/photo_search.md

@@ -65,8 +65,8 @@ def face_recognition_flow(flow_builder, data_scope):
 This creates a table with `filename` and `content` fields. 📂
 
 
-You can connect it to your [S3 Buckets](https://cocoindex.io/docs/ops/sources#amazons3) (with SQS integration, [example](https://cocoindex.io/blogs/s3-incremental-etl))
-or [Azure Blob store](https://cocoindex.io/docs/ops/sources#azureblob).
+You can connect it to your [S3 Buckets](https://cocoindex.io/docs/ops/sources/amazons3) (with SQS integration, [example](https://cocoindex.io/blogs/s3-incremental-etl))
+or [Azure Blob store](https://cocoindex.io/docs/ops/sources/azureblob).
 
 ## Detect and Extract Faces
 

{cocoindex-0.2.17 → cocoindex-0.2.19}/docs/docs/examples/examples/postgres_source.md

@@ -59,7 +59,7 @@ CocoIndex incrementally sync data from Postgres. When new or updated rows are fo
 - `notification` enables change capture based on Postgres LISTEN/NOTIFY. Each change triggers an incremental processing on the specific row immediately.
 - Regardless if `notification` is provided or not, CocoIndex still needs to scan the full table to detect changes in some scenarios (e.g. between two `update` invocation), and the `ordinal_column` provides a field that CocoIndex can use to quickly detect which row has changed without reading value columns.
 
-Check [Postgres source](https://cocoindex.io/docs/ops/sources#postgres) for more details.
+Check [Postgres source](https://cocoindex.io/docs/ops/sources/postgres) for more details.
 
 If you use the Postgres database hosted by Supabase, please click Connect on your project dashboard and find the URL there. Check [DatabaseConnectionSpec](https://cocoindex.io/docs/core/settings#databaseconnectionspec)
 for more details.

cocoindex-0.2.19/docs/docs/ops/functions.md

@@ -0,0 +1,252 @@
+---
+title: Functions
+description: CocoIndex Built-in Functions
+---
+
+# CocoIndex Built-in Functions
+
+## ParseJson
+
+`ParseJson` parses a given text to JSON.
+
+Input data:
+
+* `text` (*Str*): The source text to parse.
+* `language` (*Optional[Str]*, default: `"json"`): The language of the source text.  Only `json` is supported now.
+
+Return: *Json*, the parsed JSON object.
+
+## DetectProgrammingLanguage
+
+`DetectProgrammingLanguage` detects the programming language of a file based on its filename extension.
+
+Input data:
+
+* `filename` (*Str*): The filename (with extension) to detect the language for.
+
+Return: *Str* or *Null*. Returns the programming language name if the file extension is recognized, or *Null* if the extension is not supported.
+
+The returned string values match the language name listed in [`tree-sitter-language-pack`](https://github.com/Goldziher/tree-sitter-language-pack?tab=readme-ov-file#available-languages).
+
+## SplitRecursively
+
+`SplitRecursively` splits a document into chunks of a given size.
+It tries to split at higher-level boundaries. If each chunk is still too large, it tries at the next level of boundaries.
+For example, for a Markdown file, it identifies boundaries in this order: level-1 sections, level-2 sections, level-3 sections, paragraphs, sentences, etc.
+
+The spec takes the following fields:
+
+* `custom_languages` (`list[CustomLanguageSpec]`, optional): This allows you to customize the way to chunking specific languages using regular expressions. Each `CustomLanguageSpec` is a dict with the following fields:
+  * `language_name` (`str`): Name of the language.
+  * `aliases` (`list[str]`, optional): A list of aliases for the language.
+        It's an error if any language name or alias is duplicated.
+
+  * `separators_regex` (`list[str]`): A list of regex patterns to split the text.
+        Higher-level boundaries should come first, and lower-level should be listed later. e.g. `[r"\n# ", r"\n## ", r"\n\n", r"\. "]`.
+        See [regex syntax](https://docs.rs/regex/latest/regex/#syntax) for supported regular expression syntax.
+
+Input data:
+
+* `text` (*Str*): The text to split.
+* `chunk_size` (*Int64*): The maximum size of each chunk, in bytes.
+* `min_chunk_size` (*Int64*, default: `chunk_size / 2`): The minimum size of each chunk, in bytes.
+
+    :::note
+
+    `SplitRecursively` will do its best to make the output chunks sized between `min_chunk_size` and `chunk_size`.
+    However, it's possible that some chunks are smaller than `min_chunk_size` or larger than `chunk_size` in rare cases, e.g. too short input text, or non-splittable large text.
+
+    Please avoid setting `min_chunk_size` to a value too close to `chunk_size`, to leave more rooms for the function to plan the optimal chunking.
+
+    :::
+
+* `chunk_overlap` (*Optional[Int64]*, default: *None*): The maximum overlap size between adjacent chunks, in bytes.
+* `language` (*Str*, default: `""`): The language of the document.
+    Can be a language name (e.g. `python`, `javascript`, `markdown`) or a file extension (e.g. `.py`, `.js`, `.md`).
+
+    :::note
+
+    We use the `language` field to determine how to split the input text, following these rules:
+
+  * We match the input `language` field against the following registries in the following order:
+  * `custom_languages` in the spec, against the `language_name` or `aliases` field of each entry.
+  * Builtin languages (see [Supported Languages](#supported-languages) section below), against the language, aliases or file extensions of each entry.
+
+        All matches are in a case-insensitive manner.
+
+  * If no match is found, the input will be treated as plain text.
+
+    :::
+
+Return: [*KTable*](/docs/core/data_types#ktable), each row represents a chunk, with the following sub fields:
+
+* `location` (*Range*): The location of the chunk.
+* `text` (*Str*): The text of the chunk.
+* `start` / `end` (*Struct*): Details about the start position (inclusive) and end position (exclusive) of the chunk. They have the following sub fields:
+  * `offset` (*Int64*): The byte offset of the position.
+  * `line` (*Int64*): The line number of the position. Starting from 1.
+  * `column` (*Int64*): The column number of the position. Starting from 1.
+
+### Supported Languages
+
+Currently, `SplitRecursively` supports the following languages:
+
+| Language | Aliases | File Extensions |
+|----------|---------|-----------------|
+| c | | `.c` |
+| cpp | c++ | `.cpp`, `.cc`, `.cxx`, `.h`, `.hpp` |
+| csharp | csharp, cs | `.cs` |
+| css | | `.css`, `.scss` |
+| dtd | | `.dtd` |
+| fortran | f, f90, f95, f03 | `.f`, `.f90`, `.f95`, `.f03` |
+| go | golang | `.go` |
+| html | | `.html`, `.htm` |
+| java | | `.java` |
+| javascript | js | `.js` |
+| json | | `.json` |
+| kotlin | | `.kt`, `.kts` |
+| markdown | md | `.md`, `.mdx` |
+| pascal | pas, dpr, delphi | `.pas`, `.dpr` |
+| php | | `.php` |
+| python | | `.py` |
+| r | | `.r` |
+| ruby | | `.rb` |
+| rust | rs | `.rs` |
+| scala | | `.scala` |
+| solidity | | `.sol` |
+| sql | | `.sql` |
+| swift | | `.swift` |
+| toml | | `.toml` |
+| tsx | | `.tsx` |
+| typescript | ts | `.ts` |
+| xml | | `.xml` |
+| yaml | | `.yaml`, `.yml` |
+
+## SentenceTransformerEmbed
+
+`SentenceTransformerEmbed` embeds a text into a vector space using the [SentenceTransformer](https://huggingface.co/sentence-transformers) library.
+
+:::note Optional Dependency Required
+
+This function requires the 'sentence-transformers' library, which is an optional dependency. Install CocoIndex with:
+
+```bash
+pip install 'cocoindex[embeddings]'
+```
+
+:::
+
+The spec takes the following fields:
+
+* `model` (`str`): The name of the SentenceTransformer model to use.
+* `args` (`dict[str, Any]`, optional): Additional arguments to pass to the SentenceTransformer constructor. e.g. `{"trust_remote_code": True}`
+
+Input data:
+
+* `text` (*Str*): The text to embed.
+
+Return: *Vector[Float32, N]*, where *N* is determined by the model
+
+## ExtractByLlm
+
+`ExtractByLlm` extracts structured information from a text using specified LLM. The spec takes the following fields:
+
+* `llm_spec` (`cocoindex.LlmSpec`): The specification of the LLM to use. See [LLM Spec](/docs/ai/llm#llm-spec) for more details.
+* `output_type` (`type`): The type of the output. e.g. a dataclass type name. See [Data Types](/docs/core/data_types) for all supported data types. The LLM will output values that match the schema of the type.
+* `instruction` (`str`, optional): Additional instruction for the LLM.
+
+:::tip Clear type definitions
+
+Definitions of the `output_type` is fed into LLM as guidance to generate the output.
+To improve the quality of the extracted information, giving clear definitions for your dataclasses is especially important, e.g.
+
+* Provide readable field names for your dataclasses.
+* Provide reasonable docstrings for your dataclasses.
+* For any optional fields, clearly annotate that they are optional, by `SomeType | None` or `typing.Optional[SomeType]`.
+
+:::
+
+Input data:
+
+* `text` (*Str*): The text to extract information from.
+
+Return: As specified by the `output_type` field in the spec. The extracted information from the input text.
+
+## EmbedText
+
+`EmbedText` embeds a text into a vector space using various LLM APIs that support text embedding.
+
+The spec takes the following fields:
+
+* `api_type` ([`cocoindex.LlmApiType`](/docs/ai/llm#llm-api-types)): The type of LLM API to use for embedding.
+* `model` (`str`): The name of the embedding model to use.
+* `address` (`str`, optional): The address of the LLM API. If not specified, uses the default address for the API type.
+* `output_dimension` (`int`, optional): The expected dimension of the output embedding vector. If not specified, use the default dimension of the model.
+
+    For most API types, the function internally keeps a registry for the default output dimension of known model.
+    You need to explicitly specify the `output_dimension` if you want to use a new model that is not in the registry yet.
+
+* `task_type` (`str`, optional): The task type for embedding, used by some embedding models to optimize the embedding for specific use cases.
+
+:::note Supported APIs for Text Embedding
+
+Not all LLM APIs support text embedding. See the [LLM API Types table](/docs/ai/llm#llm-api-types) for which APIs support text embedding functionality.
+
+:::
+
+Input data:
+
+* `text` (*Str*): The text to embed.
+
+Return: *Vector[Float32, N]*, where *N* is the dimension of the embedding vector determined by the model.
+
+## ColPali Functions
+
+ColPali functions enable multimodal document retrieval using ColVision models. These functions support ALL models available in the [colpali-engine library](https://github.com/illuin-tech/colpali), including:
+
+* **ColPali models** (colpali-*): PaliGemma-based, best for general document retrieval
+* **ColQwen2 models** (colqwen-*): Qwen2-VL-based, excellent for multilingual text (29+ languages) and general vision
+* **ColSmol models** (colsmol-*): Lightweight, good for resource-constrained environments
+* Any future ColVision models supported by colpali-engine
+
+These models use late interaction between image patch embeddings and text token embeddings for retrieval.
+
+:::note Optional Dependency Required
+
+These functions require the `colpali-engine` library, which is an optional dependency. Install CocoIndex with:
+
+```bash
+pip install 'cocoindex[colpali]'
+```
+
+:::
+
+### ColPaliEmbedImage
+
+`ColPaliEmbedImage` embeds images using ColVision multimodal models.
+
+The spec takes the following fields:
+
+* `model` (`str`): Any ColVision model name supported by colpali-engine (e.g., "vidore/colpali-v1.2", "vidore/colqwen2.5-v0.2", "vidore/colsmol-v1.0"). See the [complete list of supported models](https://github.com/illuin-tech/colpali#list-of-colvision-models).
+
+Input data:
+
+* `img_bytes` (*Bytes*): The image data in bytes format.
+
+Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable patches and fixed hidden dimension.
+
+### ColPaliEmbedQuery
+
+`ColPaliEmbedQuery` embeds text queries using ColVision multimodal models.
+
+This produces query embeddings compatible with ColVision image embeddings for late interaction scoring (MaxSim).
+
+The spec takes the following fields:
+
+* `model` (`str`): Any ColVision model name supported by colpali-engine (e.g., "vidore/colpali-v1.2", "vidore/colqwen2.5-v0.2", "vidore/colsmol-v1.0"). See the [complete list of supported models](https://github.com/illuin-tech/colpali#list-of-colvision-models).
+
+Input data:
+
+* `query` (*Str*): The text query to embed.
+
+Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable tokens and fixed hidden dimension.