cocoindex 0.2.12__tar.gz → 0.2.14__tar.gz

cocoindex-0.2.14/.github/SECURITY.md

@@ -0,0 +1,26 @@
+# Security Policy for CocoIndex
+
+## Reporting a Vulnerability
+If you discover a security vulnerability in CocoIndex, please report it responsibly to our security team:
+
+**Email:** [support@cocoindex.io](mailto:support@cocoindex.io)
+
+⚠️ Please do not file GitHub issues for security vulnerabilities as they are public! ⚠️
+
+Please provide:
+- A detailed description of the vulnerability
+- Steps to reproduce the issue
+- Any relevant logs, screenshots, or proof-of-concept code
+
+We will acknowledge your report promptly and work with you to resolve the issue.
+
+## Scope
+This policy covers security issues related to CocoIndex open-source software.
+
+## Response & Disclosure
+- We aim to respond as soon as we can.
+- Security fixes will be released as soon as practical after verification.
+
+---
+
+Thank you for helping us keep CocoIndex secure!

{cocoindex-0.2.12 → cocoindex-0.2.14}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.12"
+version = "0.2.14"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.2.12 → cocoindex-0.2.14}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.12"
+version = "0.2.14"
 edition = "2024"
 rust-version = "1.89"
 license = "Apache-2.0"

{cocoindex-0.2.12 → cocoindex-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.12
+Version: 0.2.14
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
@@ -16,6 +16,7 @@ Classifier: Topic :: Text Processing :: Indexing
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
 Classifier: Typing :: Typed
+Requires-Dist: typing-extensions>=4.12 ; python_full_version < '3.13'
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -230,6 +231,7 @@ It defines an index flow like this:
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
+| [Embeddings to LanceDB](examples/text_embedding_lancedb) | Index documents in a LanceDB collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
 | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|

{cocoindex-0.2.12 → cocoindex-0.2.14}/README.md

@@ -181,6 +181,7 @@ It defines an index flow like this:
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
+| [Embeddings to LanceDB](examples/text_embedding_lancedb) | Index documents in a LanceDB collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
 | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|

{cocoindex-0.2.12 → cocoindex-0.2.14}/THIRD_PARTY_NOTICES.html

@@ -2428,7 +2428,7 @@ Software.
                 <h3 id="Apache-2.0">Apache License 2.0</h3>
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
-                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.12</a></li>
+                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.14</a></li>
                     <li><a href=" https://github.com/awesomized/crc-fast-rust ">crc-fast 1.3.0</a></li>
                     <li><a href=" https://github.com/qdrant/rust-client ">qdrant-client 1.15.0</a></li>
                 </ul>
@@ -10673,38 +10673,6 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.</pre>
-            </li>
-            <li class="license">
-                <h3 id="MIT">MIT License</h3>
-                <h4>Used by:</h4>
-                <ul class="license-used-by">
-                    <li><a href=" https://github.com/tree-sitter/tree-sitter-scala ">tree-sitter-scala 0.24.0</a></li>
-                </ul>
-                <pre class="license-text">(The MIT License)
-
-Copyright (c) 2014 Nathan Rajlich &lt;nathan@tootallnate.net&gt;
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the &quot;Software&quot;), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-</pre>
             </li>
             <li class="license">
                 <h3 id="MIT">MIT License</h3>
@@ -12299,6 +12267,32 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+</pre>
+            </li>
+            <li class="license">
+                <h3 id="MIT">MIT License</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    <li><a href=" https://github.com/tree-sitter/tree-sitter-scala ">tree-sitter-scala 0.24.0</a></li>
+                </ul>
+                <pre class="license-text">This software is released under the MIT license:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the &quot;Software&quot;), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 </pre>
             </li>
             <li class="license">

{cocoindex-0.2.12 → cocoindex-0.2.14}/docs/docs/ai/llm.mdx

@@ -107,6 +107,8 @@ For text embedding, a spec for OpenAI looks like this:
 cocoindex.functions.EmbedText(
     api_type=cocoindex.LlmApiType.OPENAI,
     model="text-embedding-3-small",
+    # Optional, use the default output dimension if not specified
+    output_dimension=1536,
 )
 ```
 
@@ -199,7 +201,10 @@ For text embedding, a spec looks like this:
 cocoindex.functions.EmbedText(
     api_type=cocoindex.LlmApiType.GEMINI,
     model="text-embedding-004",
+    # Optional, use the default task type if not specified
     task_type="SEMANTICS_SIMILARITY",
+    # Optional, use the default output dimension if not specified
+    output_dimension=1536,
 )
 ```
 
@@ -260,7 +265,10 @@ For text embedding, a spec for Vertex AI looks like this:
 cocoindex.functions.EmbedText(
     api_type=cocoindex.LlmApiType.VERTEX_AI,
     model="text-embedding-005",
+    # Optional, use the default task type if not specified
     task_type="SEMANTICS_SIMILARITY",
+    # Optional, use the default output dimension if not specified
+    output_dimension=1536,
     api_config=cocoindex.llm.VertexAiConfig(project="your-project-id"),
 )
 ```

{cocoindex-0.2.12 → cocoindex-0.2.14}/docs/docs/core/flow_def.mdx

@@ -313,6 +313,7 @@ Types of the fields must be key types. See [Key Types](data_types#key-types) for
 
     *   `field_name`: the field to create vector index.
     *   `metric`: the similarity metric to use.
+    *   `method` (optional): the index algorithm and optional tuning parameters. Leave unset to use the target default (HNSW for Postgres). Use `cocoindex.HnswVectorIndexMethod()` or `cocoindex.IvfFlatVectorIndexMethod()` to customize the method and its parameters.
 
 #### Similarity Metrics
 

{cocoindex-0.2.12 → cocoindex-0.2.14}/docs/docs/examples/examples/simple_vector_index.md

@@ -105,6 +105,16 @@ doc_embeddings.export(
 CocoIndex supports other vector databases as well, with 1-line switch.
 <DocumentationButton url="https://cocoindex.io/docs/ops/targets" text="Targets" />
 
+Need IVFFlat or custom HNSW parameters? Pass a method, for example:
+
+```python
+cocoindex.VectorIndexDef(
+    field_name="embedding",
+    metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+    method=cocoindex.IvfFlatVectorIndexMethod(lists=200),
+)
+```
+
 ## Query the index
 
 ### Define a shared flow for both indexing and querying

cocoindex-0.2.14/examples/text_embedding_lancedb/.gitignore

@@ -0,0 +1 @@
+/lancedb_data

cocoindex-0.2.14/examples/text_embedding_lancedb/README.md

@@ -0,0 +1,58 @@
+# Build text embedding and semantic search 🔍 with LanceDB
+
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+CocoIndex supports LanceDB natively. In this example, we will build index flow from text embedding from local markdown files, and query the index. We will use **LanceDB** as the vector database.
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+
+## Steps
+### Indexing Flow
+
+1.  We will ingest a list of local files.
+2.  For each file, perform chunking (recursively split) and then embedding.
+3.  We will save the embeddings and the metadata in LanceDB.
+
+### Query
+
+1.  We have `search()` as a [query handler](https://cocoindex.io/docs/query#query-handler), to query the LanceDB table with LanceDB client.
+2.  We share the embedding operation `text_to_embedding()` between indexing and querying,
+  by wrapping it as a [transform flow](https://cocoindex.io/docs/query#transform-flow).
+
+## Pre-requisites
+
+1.  [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one. Although the target store is LanceDB, CocoIndex uses Postgres to track the data lineage for incremental processing.
+
+2.  Install dependencies:
+
+    ```sh
+    pip install -e .
+    ```
+
+LanceDB will automatically create a local database directory when you run the example (no additional setup required).
+
+## Run
+
+Update index, which will also setup LanceDB tables at the first time:
+
+```bash
+cocoindex update --setup main
+```
+
+You can also run the command with `-L`, which will watch for file changes and update the index automatically.
+
+```bash
+cocoindex update --setup -L main
+```
+
+## CocoInsight
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline.
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
+
+```bash
+cocoindex server -ci -L main
+```
+
+Open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
+You can run queries in the CocoInsight UI.

cocoindex-0.2.14/examples/text_embedding_lancedb/main.py

@@ -0,0 +1,109 @@
+from dotenv import load_dotenv
+import datetime
+import cocoindex
+import math
+import cocoindex.targets.lancedb as coco_lancedb
+
+# Define LanceDB connection constants
+LANCEDB_URI = "./lancedb_data"  # Local directory for LanceDB
+LANCEDB_TABLE = "TextEmbedding"
+
+
+@cocoindex.transform_flow()
+def text_to_embedding(
+    text: cocoindex.DataSlice[str],
+) -> cocoindex.DataSlice[list[float]]:
+    """
+    Embed the text using a SentenceTransformer model.
+    This is a shared logic between indexing and querying, so extract it as a function.
+    """
+    return text.transform(
+        cocoindex.functions.SentenceTransformerEmbed(
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    )
+
+
+@cocoindex.flow_def(name="TextEmbeddingWithLanceDB")
+def text_embedding_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+) -> None:
+    """
+    Define an example flow that embeds text into a vector database.
+    """
+    data_scope["documents"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="markdown_files"),
+        refresh_interval=datetime.timedelta(seconds=5),
+    )
+
+    doc_embeddings = data_scope.add_collector()
+
+    with data_scope["documents"].row() as doc:
+        doc["chunks"] = doc["content"].transform(
+            cocoindex.functions.SplitRecursively(),
+            language="markdown",
+            chunk_size=500,
+            chunk_overlap=100,
+        )
+
+        with doc["chunks"].row() as chunk:
+            chunk["embedding"] = text_to_embedding(chunk["text"])
+            doc_embeddings.collect(
+                id=cocoindex.GeneratedField.UUID,
+                filename=doc["filename"],
+                location=chunk["location"],
+                text=chunk["text"],
+                # 'text_embedding' is the name of the vector we've created the LanceDB table with.
+                text_embedding=chunk["embedding"],
+            )
+
+    doc_embeddings.export(
+        "doc_embeddings",
+        coco_lancedb.LanceDB(db_uri=LANCEDB_URI, table_name=LANCEDB_TABLE),
+        primary_key_fields=["id"],
+        # We cannot enable it when the table has no data yet, as LanceDB requires data to train the index.
+        # See: https://github.com/lancedb/lance/issues/4034
+        #
+        #   vector_indexes=[
+        #       cocoindex.VectorIndexDef(
+        #           "text_embedding", cocoindex.VectorSimilarityMetric.L2_DISTANCE
+        #       ),
+        #   ],
+    )
+
+
+@text_embedding_flow.query_handler(
+    result_fields=cocoindex.QueryHandlerResultFields(
+        embedding=["embedding"],
+        score="score",
+    ),
+)
+async def search(query: str) -> cocoindex.QueryOutput:
+    print("Searching...", query)
+    db = await coco_lancedb.connect_async(LANCEDB_URI)
+    table = await db.open_table(LANCEDB_TABLE)
+
+    # Get the embedding for the query
+    query_embedding = await text_to_embedding.eval_async(query)
+
+    search = await table.search(query_embedding, vector_column_name="text_embedding")
+    search_results = await search.limit(5).to_list()
+
+    print(search_results)
+
+    return cocoindex.QueryOutput(
+        results=[
+            {
+                "filename": result["filename"],
+                "text": result["text"],
+                "embedding": result["text_embedding"],
+                # Qdrant's L2 "distance" is squared, so we take the square root to align with normal L2 distance
+                "score": math.sqrt(result["_distance"]),
+            }
+            for result in search_results
+        ],
+        query_info=cocoindex.QueryInfo(
+            embedding=query_embedding,
+            similarity_metric=cocoindex.VectorSimilarityMetric.L2_DISTANCE,
+        ),
+    )