cocoindex 0.1.41__tar.gz → 0.1.42__tar.gz

{cocoindex-0.1.41 → cocoindex-0.1.42}/Cargo.lock

@@ -993,7 +993,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.41"
+version = "0.1.42"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.41 → cocoindex-0.1.42}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.41"
+version = "0.1.42"
 edition = "2024"
 
 [profile.release]

{cocoindex-0.1.41 → cocoindex-0.1.42}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.41
+Version: 0.1.42
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
+Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: pytest ; extra == 'test'
 Provides-Extra: test
 License-File: LICENSE

{cocoindex-0.1.41 → cocoindex-0.1.42}/docs/docs/core/data_types.mdx

@@ -35,6 +35,7 @@ This is the list of all basic types supported by CocoIndex:
 | Time | | `datetime.time` | `datetime.time` |
 | LocalDatetime | Date and time without timezone | `cocoindex.LocalDateTime` | `datetime.datetime` |
 | OffsetDatetime | Date and time with a timezone offset | `cocoindex.OffsetDateTime` | `datetime.datetime` |
+| TimeDelta | A duration of time | `datetime.timedelta` | `datetime.timedelta` |
 | Vector[*T*, *Dim*?] | *T* must be basic type. *Dim* is a positive integer and optional. |`cocoindex.Vector[T]` or `cocoindex.Vector[T, Dim]` | `list[T]` | 
 | Json | | `cocoindex.Json` | Any data convertible to JSON by `json` package | 
 

cocoindex-0.1.42/examples/code_embedding/README.md

@@ -0,0 +1,71 @@
+# Build real-time index for codebase 
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+CocoIndex provides built-in support for code base chunking, using Tree-sitter to keep syntax boundary. In this example, we will build real-time index for codebase using CocoIndex. 
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+![Build embedding index for codebase](https://github.com/user-attachments/assets/6dc5ce89-c949-41d4-852f-ad95af163dbd)
+
+[Tree-sitter](https://en.wikipedia.org/wiki/Tree-sitter_%28parser_generator%29) is a parser generator tool and an incremental parsing library. It is available in Rust 🦀 - [GitHub](https://github.com/tree-sitter/tree-sitter). CocoIndex has built-in Rust integration with Tree-sitter to efficiently parse code and extract syntax trees for various programming languages. Check out the list of supported languages [here](https://cocoindex.io/docs/ops/functions#splitrecursively) - in the `language` section.
+
+
+## Tutorials
+- Step by step tutorial - Check out the [blog](https://cocoindex.io/blogs/index-code-base-for-rag).
+- Video tutorial - [Youtube](https://youtu.be/G3WstvhHO24?si=Bnxu67Ax5Lv8b-J2).
+
+## Steps
+
+### Indexing Flow
+<p align='center'>
+  <img width="434" alt="Screenshot 2025-05-19 at 10 14 36 PM" src="https://github.com/user-attachments/assets/3a506034-698f-480a-b653-22184dae4e14" />
+</p>
+
+1. We will ingest CocoIndex codebase.
+2. For each file, perform chunking (Tree-sitter) and then embedding. 
+3. We will save the embeddings and the metadata in Postgres with PGVector.
+   
+### Query:
+We will match against user-provided text by a SQL query, reusing the embedding operation in the indexing flow.
+
+
+## Prerequisite
+[Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+## Run
+
+- Install dependencies:
+  ```bash
+  pip install -e .
+  ```
+
+- Setup:
+
+  ```bash
+  python main.py cocoindex setup
+  ```
+
+- Update index:
+  
+  ```bash
+  python main.py cocoindex update
+  ```
+
+- Run:
+
+  ```bash
+  python main.py
+  ```
+
+## CocoInsight
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline. 
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run the following command to start CocoInsight:
+
+```
+python main.py cocoindex server -ci
+```
+
+Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
+
+<img width="1305" alt="Chunking Visualization" src="https://github.com/user-attachments/assets/8e83b9a4-2bed-456b-83e5-b5381b28b84a" />
+

{cocoindex-0.1.41 → cocoindex-0.1.42}/examples/code_embedding/main.py

@@ -1,5 +1,5 @@
 from dotenv import load_dotenv
-
+from psycopg_pool import ConnectionPool
 import cocoindex
 import os
 
@@ -8,7 +8,8 @@ def extract_extension(filename: str) -> str:
     """Extract the extension of a filename."""
     return os.path.splitext(filename)[1]
 
-def code_to_embedding(text: cocoindex.DataSlice) -> cocoindex.DataSlice:
+@cocoindex.transform_flow()
+def code_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
     """
     Embed the text using a SentenceTransformer model.
     """
@@ -24,7 +25,7 @@ def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
     data_scope["files"] = flow_builder.add_source(
         cocoindex.sources.LocalFile(path="../..",
                                     included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx"],
-                                    excluded_patterns=[".*", "target", "**/node_modules"]))
+                                    excluded_patterns=["**/.*", "target", "**/node_modules"]))
     code_embeddings = data_scope.add_collector()
 
     with data_scope["files"].row() as file:
@@ -47,26 +48,40 @@ def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
                 metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
 
 
-query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
-    name="SemanticsSearch",
-    flow=code_embedding_flow,
-    target_name="code_embeddings",
-    query_transform_flow=code_to_embedding,
-    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)
+
+def search(pool: ConnectionPool, query: str, top_k: int = 5):
+    # Get the table name, for the export target in the code_embedding_flow above.
+    table_name = cocoindex.utils.get_target_storage_default_name(code_embedding_flow, "code_embeddings")
+    # Evaluate the transform flow defined above with the input query, to get the embedding.
+    query_vector = code_to_embedding.eval(query)
+    # Run the query and get the results.
+    with pool.connection() as conn:
+        with conn.cursor() as cur:
+            cur.execute(f"""
+                SELECT filename, code, embedding <=> %s::vector AS distance
+                FROM {table_name} ORDER BY distance LIMIT %s
+            """, (query_vector, top_k))
+            return [
+                {"filename": row[0], "code": row[1], "score": 1.0 - row[2]}
+                for row in cur.fetchall()
+            ]
 
 @cocoindex.main_fn()
 def _run():
+    # Initialize the database connection pool.
+    pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
     # Run queries in a loop to demonstrate the query capabilities.
     while True:
         try:
             query = input("Enter search query (or Enter to quit): ")
             if query == '':
                 break
-            results, _ = query_handler.search(query, 10)
+            # Run the query function with the database connection pool and the query.
+            results = search(pool, query)
             print("\nSearch results:")
             for result in results:
-                print(f"[{result.score:.3f}] {result.data['filename']}")
-                print(f"    {result.data['code']}")
+                print(f"[{result['score']:.3f}] {result['filename']}")
+                print(f"    {result['code']}")
                 print("---")
             print()
         except KeyboardInterrupt:

cocoindex-0.1.42/examples/text_embedding/README.md

@@ -0,0 +1,63 @@
+# Build text embedding and semantic search 🔍
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/cocoindex-io/cocoindex/blob/main/examples/text_embedding/Text_Embedding.ipynb)
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+In this example, we will build index flow from text embedding from local markdown files, and query the index.
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+## Steps
+🌱 A detailed step by step tutorial can be found here: [Get Started Documentation](https://cocoindex.io/docs/getting_started/quickstart)
+
+### Indexing Flow
+<img width="461" alt="Screenshot 2025-05-19 at 5 48 28 PM" src="https://github.com/user-attachments/assets/b6825302-a0c7-4b86-9a2d-52da8286b4bd" />
+
+1. We will ingest a list of local files.
+2. For each file, perform chunking (recursively split) and then embedding. 
+3. We will save the embeddings and the metadata in Postgres with PGVector.
+   
+### Query
+We will match against user-provided text by a SQL query, and reuse the embedding operation in the indexing flow.
+
+
+## Prerequisite
+
+[Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+## Run
+
+Install dependencies:
+
+```bash
+pip install -e .
+```
+
+Setup:
+
+```bash
+python main.py cocoindex setup
+```
+
+Update index:
+
+```bash
+python main.py cocoindex update
+```
+
+Run:
+
+```bash
+python main.py
+```
+
+## CocoInsight
+
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline. 
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
+
+```
+python main.py cocoindex server -ci
+```
+
+Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
+

{cocoindex-0.1.41 → cocoindex-0.1.42}/examples/text_embedding/main.py

@@ -1,8 +1,7 @@
-import os
 from dotenv import load_dotenv
 from psycopg_pool import ConnectionPool
-
 import cocoindex
+import os
 
 @cocoindex.transform_flow()
 def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
@@ -20,7 +19,7 @@ def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
     Define an example flow that embeds text into a vector database.
     """
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="markdown_files", included_patterns=["*.md"]))
+        cocoindex.sources.LocalFile(path="markdown_files"))
 
     doc_embeddings = data_scope.add_collector()
 
@@ -43,33 +42,27 @@ def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
                 field_name="embedding",
                 metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
 
-# Keep for now to allow CocoInsight to query.
-# Will be removed later after we expose `search()` below as a query function (https://github.com/cocoindex-io/cocoindex/issues/502).
-cocoindex.query.SimpleSemanticsQueryHandler(
-    name="SemanticsSearch",
-    flow=text_embedding_flow,
-    target_name="doc_embeddings",
-    query_transform_flow=text_to_embedding,
-    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)
 
 def search(pool: ConnectionPool, query: str, top_k: int = 5):
+    # Get the table name, for the export target in the text_embedding_flow above.
     table_name = cocoindex.utils.get_target_storage_default_name(text_embedding_flow, "doc_embeddings")
+    # Evaluate the transform flow defined above with the input query, to get the embedding.
     query_vector = text_to_embedding.eval(query)
+    # Run the query and get the results.
     with pool.connection() as conn:
         with conn.cursor() as cur:
             cur.execute(f"""
-                SELECT filename, location, text, embedding <=> %s::vector AS distance
-                FROM {table_name}
-                ORDER BY distance
-                LIMIT %s
+                SELECT filename, text, embedding <=> %s::vector AS distance
+                FROM {table_name} ORDER BY distance LIMIT %s
             """, (query_vector, top_k))
             return [
-                {"filename": row[0], "location": row[1], "text": row[2], "score": 1.0 - row[3]}
+                {"filename": row[0], "text": row[1], "score": 1.0 - row[2]}
                 for row in cur.fetchall()
             ]
 
 @cocoindex.main_fn()
 def _run():
+    # Initialize the database connection pool.
     pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
     # Run queries in a loop to demonstrate the query capabilities.
     while True:
@@ -77,10 +70,11 @@ def _run():
             query = input("Enter search query (or Enter to quit): ")
             if query == '':
                 break
+            # Run the query function with the database connection pool and the query.
             results = search(pool, query)
             print("\nSearch results:")
             for result in results:
-                print(f"[{result['score']:.3f}] {result['filename']} location:{result['location']}")
+                print(f"[{result['score']:.3f}] {result['filename']}")
                 print(f"    {result['text']}")
                 print("---")
             print()

{cocoindex-0.1.41 → cocoindex-0.1.42}/examples/text_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.39",
+    "cocoindex>=0.1.40",
     "python-dotenv>=1.0.1",
     "psycopg[binary,pool]",
 ]

cocoindex-0.1.42/examples/text_embedding_qdrant/README.md

@@ -0,0 +1,87 @@
+# Build text embedding and semantic search 🔍 with Qdrant
+
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+CocoIndex supports Qdrant natively - [documentation](https://cocoindex.io/docs/ops/storages#qdrant). In this example, we will build index flow from text embedding from local markdown files, and query the index. We will use **Qdrant** as the vector database.
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+<img width="860" alt="CocoIndex supports Qdrant" src="https://github.com/user-attachments/assets/a9deecfa-dd94-4b97-a1b1-90488d8178df" />
+
+## Steps
+### Indexing Flow
+<img width="480" alt="Index flow for text embedding" src="https://github.com/user-attachments/assets/44d47b5e-b49b-4f05-9a00-dcb8027602a1" />
+
+1. We will ingest a list of local files.
+2. For each file, perform chunking (recursively split) and then embedding. 
+3. We will save the embeddings and the metadata in Postgres with PGVector.
+   
+### Query
+We use Qdrant client to query the index, and reuse the embedding operation in the indexing flow.
+
+
+## Pre-requisites
+
+- [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one. Although the target store is Qdrant, CocoIndex uses Postgress to track the data lineage for incremental processing.
+
+- Run Qdrant.
+
+   ```bash
+   docker run -d -p 6334:6334 -p 6333:6333 qdrant/qdrant
+   ```
+
+- [Create a collection](https://qdrant.tech/documentation/concepts/vectors/#named-vectors) to export the embeddings to.
+
+   ```bash
+   curl  -X PUT \
+     'http://localhost:6333/collections/cocoindex' \
+     --header 'Content-Type: application/json' \
+     --data-raw '{
+     "vectors": {
+       "text_embedding": {
+         "size": 384,
+         "distance": "Cosine"
+       }
+     }
+   }'
+   ```
+
+   You can view the collections and data with the Qdrant dashboard at <http://localhost:6333/dashboard>.
+
+## Run
+
+- Install dependencies:
+
+   ```bash
+   pip install -e .
+   ```
+
+- Setup:
+
+   ```bash
+   python main.py cocoindex setup
+   ```
+
+- Update index:
+
+   ```bash
+   python main.py cocoindex update
+   ```
+
+- Run:
+
+   ```bash
+   python main.py
+   ```
+
+## CocoInsight
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline. 
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
+
+```bash
+python main.py cocoindex server -ci
+```
+
+Open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
+
+

{cocoindex-0.1.41 → cocoindex-0.1.42}/examples/text_embedding_qdrant/main.py

@@ -1,21 +1,26 @@
 from dotenv import load_dotenv
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Filter, FieldCondition, MatchValue
 
 import cocoindex
 
+# Define Qdrant connection constants
+QDRANT_GRPC_URL = "http://localhost:6334"
+QDRANT_COLLECTION = "cocoindex"
 
-def text_to_embedding(text: cocoindex.DataSlice) -> cocoindex.DataSlice:
+
+@cocoindex.transform_flow()
+def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
     """
     Embed the text using a SentenceTransformer model.
     This is a shared logic between indexing and querying, so extract it as a function.
     """
     return text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
-            model="sentence-transformers/all-MiniLM-L6-v2"
-        )
-    )
+            model="sentence-transformers/all-MiniLM-L6-v2"))
 
 
-@cocoindex.flow_def(name="TextEmbedding")
+@cocoindex.flow_def(name="TextEmbeddingWithQdrant")
 def text_embedding_flow(
     flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
 ):
@@ -50,35 +55,39 @@ def text_embedding_flow(
     doc_embeddings.export(
         "doc_embeddings",
         cocoindex.storages.Qdrant(
-            collection_name="cocoindex", grpc_url="http://localhost:6334/"
+            collection_name=QDRANT_COLLECTION, grpc_url=QDRANT_GRPC_URL
         ),
         primary_key_fields=["id"],
         setup_by_user=True,
     )
 
 
-query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
-    name="SemanticsSearch",
-    flow=text_embedding_flow,
-    target_name="doc_embeddings",
-    query_transform_flow=text_to_embedding,
-    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
-)
-
-
 @cocoindex.main_fn()
 def _run():
+    # Initialize Qdrant client
+    client = QdrantClient(url=QDRANT_GRPC_URL, prefer_grpc=True)
+    
     # Run queries in a loop to demonstrate the query capabilities.
     while True:
         try:
             query = input("Enter search query (or Enter to quit): ")
             if query == "":
                 break
-            results, _ = query_handler.search(query, 10, "text_embedding")
+            
+            # Get the embedding for the query
+            query_embedding = text_to_embedding.eval(query)
+            
+            search_results = client.search(
+                collection_name=QDRANT_COLLECTION,
+                query_vector=("text_embedding", query_embedding),
+                limit=10
+            )
             print("\nSearch results:")
-            for result in results:
-                print(f"[{result.score:.3f}] {result.data['filename']}")
-                print(f"    {result.data['text']}")
+            for result in search_results:
+                score = result.score
+                payload = result.payload
+                print(f"[{score:.3f}] {payload['filename']}")
+                print(f"    {payload['text']}")
                 print("---")
             print()
         except KeyboardInterrupt:

{cocoindex-0.1.41 → cocoindex-0.1.42}/examples/text_embedding_qdrant/pyproject.toml

@@ -3,7 +3,7 @@ name = "text-embedding-qdrant"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.39", "python-dotenv>=1.0.1", "qdrant-client>=1.6.0"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.41 → cocoindex-0.1.42}/pyproject.toml

@@ -9,10 +9,13 @@ description = "With CocoIndex, users declare the transformation, CocoIndex creat
 authors = [{ name = "CocoIndex", email = "cocoindex.io@gmail.com" }]
 readme = "README.md"
 requires-python = ">=3.11"
-dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8", "rich>=14.0.0"]
+dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8", "rich>=14.0.0", "python-dotenv>=1.1.0"]
 license = "Apache-2.0"
 urls = { Homepage = "https://cocoindex.io/" }
 
+[project.scripts]
+cocoindex = "cocoindex.cli:cli"
+
 [tool.maturin]
 bindings = "pyo3"
 python-source = "python"