cocoindex 0.1.49__tar.gz → 0.1.50__tar.gz

{cocoindex-0.1.49 → cocoindex-0.1.50}/Cargo.lock

@@ -676,16 +676,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
  "async-trait",
- "axum-core",
+ "axum-core 0.4.5",
  "bytes",
  "futures-util",
  "http 1.3.1",
  "http-body 1.0.1",
  "http-body-util",
+ "itoa",
+ "matchit 0.7.3",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper",
+ "tower 0.5.2",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
+dependencies = [
+ "axum-core 0.5.2",
+ "bytes",
+ "form_urlencoded",
+ "futures-util",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "http-body-util",
  "hyper 1.6.0",
  "hyper-util",
  "itoa",
- "matchit",
+ "matchit 0.8.4",
  "memchr",
  "mime",
  "percent-encoding",
@@ -721,28 +748,48 @@ dependencies = [
  "sync_wrapper",
  "tower-layer",
  "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
  "tracing",
 ]
 
 [[package]]
 name = "axum-extra"
-version = "0.9.6"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04"
+checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d"
 dependencies = [
- "axum",
- "axum-core",
+ "axum 0.8.4",
+ "axum-core 0.5.2",
  "bytes",
- "fastrand",
+ "form_urlencoded",
  "futures-util",
  "http 1.3.1",
  "http-body 1.0.1",
  "http-body-util",
  "mime",
- "multer",
  "pin-project-lite",
+ "rustversion",
  "serde",
  "serde_html_form",
+ "serde_path_to_error",
  "tower 0.5.2",
  "tower-layer",
  "tower-service",
@@ -993,7 +1040,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.49"
+version = "0.1.50"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -1002,7 +1049,7 @@ dependencies = [
  "aws-config",
  "aws-sdk-s3",
  "aws-sdk-sqs",
- "axum",
+ "axum 0.8.4",
  "axum-extra",
  "base64 0.22.1",
  "blake2",
@@ -1028,6 +1075,7 @@ dependencies = [
  "neo4rs",
  "owo-colors",
  "pgvector",
+ "phf",
  "pyo3",
  "pyo3-async-runtimes",
  "pythonize",
@@ -2610,6 +2658,12 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
+[[package]]
+name = "matchit"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
+
 [[package]]
 name = "md-5"
 version = "0.10.6"
@@ -2677,23 +2731,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "multer"
-version = "3.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
-dependencies = [
- "bytes",
- "encoding_rs",
- "futures-util",
- "http 1.3.1",
- "httparse",
- "memchr",
- "mime",
- "spin",
- "version_check",
-]
-
 [[package]]
 name = "neo4rs"
 version = "0.8.0"
@@ -2993,6 +3030,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
 dependencies = [
+ "phf_macros",
  "phf_shared",
 ]
 
@@ -3016,6 +3054,19 @@ dependencies = [
  "rand 0.8.5",
 ]
 
+[[package]]
+name = "phf_macros"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.11.3"
@@ -4653,7 +4704,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
 dependencies = [
  "async-stream",
  "async-trait",
- "axum",
+ "axum 0.7.9",
  "base64 0.22.1",
  "bytes",
  "flate2",

{cocoindex-0.1.49 → cocoindex-0.1.50}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.49"
+version = "0.1.50"
 edition = "2024"
 
 [profile.release]
@@ -20,8 +20,8 @@ pyo3-async-runtimes = { version = "0.24.0", features = ["tokio-runtime"] }
 
 anyhow = { version = "1.0.97", features = ["std"] }
 async-trait = "0.1.88"
-axum = "0.7.9"
-axum-extra = { version = "0.9.6", features = ["query"] }
+axum = "0.8.4"
+axum-extra = { version = "0.10.1", features = ["query"] }
 base64 = "0.22.1"
 chrono = "0.4.40"
 config = "0.14.1"
@@ -50,6 +50,7 @@ tower-http = { version = "0.6.2", features = ["cors", "trace"] }
 indexmap = { version = "2.8.0", features = ["serde"] }
 blake2 = "0.10.6"
 pgvector = { version = "0.4.0", features = ["sqlx"] }
+phf = { version = "0.11.3", features = ["macros"] }
 indenter = "0.3.3"
 itertools = "0.14.0"
 derivative = "2.2.0"

{cocoindex-0.1.49 → cocoindex-0.1.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.49
+Version: 0.1.50
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0

{cocoindex-0.1.49 → cocoindex-0.1.50}/docs/docs/core/data_types.mdx

@@ -36,16 +36,17 @@ This is the list of all basic types supported by CocoIndex:
 | LocalDatetime | Date and time without timezone | `cocoindex.LocalDateTime` | `datetime.datetime` |
 | OffsetDatetime | Date and time with a timezone offset | `cocoindex.OffsetDateTime` | `datetime.datetime` |
 | TimeDelta | A duration of time | `datetime.timedelta` | `datetime.timedelta` |
-| Vector[*T*, *Dim*?] | *T* must be basic type. *Dim* is a positive integer and optional. |`cocoindex.Vector[T]` or `cocoindex.Vector[T, Dim]` | `list[T]` | 
 | Json | | `cocoindex.Json` | Any data convertible to JSON by `json` package | 
+| Vector[*T*, *Dim*?] | *T* can be a basic type or a numeric type. *Dim* is a positive integer and optional. | `cocoindex.Vector[T]` or `cocoindex.Vector[T, Dim]` | `numpy.typing.NDArray[T]` or `list[T]` |
 
 Values of all data types can be represented by values in Python's native types (as described under the Native Python Type column).
 However, the underlying execution engine and some storage system (like Postgres) has finer distinctions for some types, specifically:
 
 *   *Float32* and *Float64* for `float`, with different precision.
 *   *LocalDateTime* and *OffsetDateTime* for `datetime.datetime`, with different timezone awareness.
-*   *Vector* has optional dimension information.
 *   *Range* and *Json* provide a clear tag for the type, to clearly distinguish the type in CocoIndex.
+*   *Vector* holds elements of type *T*. If *T* is numeric (e.g., `np.float32` or `np.float64`), it's represented as `NDArray[T]`; otherwise, as `list[T]`.
+*   *Vector* also has optional dimension information.
 
 The native Python type is always more permissive and can represent a superset of possible values.
 *   Only when you annotate the return type of a custom function, you should use the specific type,

{cocoindex-0.1.49 → cocoindex-0.1.50}/docs/docs/getting_started/overview.md

@@ -9,6 +9,8 @@ CocoIndex is an ultra-performant real-time data transformation framework for AI,
 
 As a data framework, CocoIndex takes it to the next level on data freshness. **Incremental processing** is one of the core values provided by CocoIndex.
 
+![Incremental Processing](/img/incremental-etl.gif)
+
 ## Programming Model
 CocoIndex follows the idea of [Dataflow programming](https://en.wikipedia.org/wiki/Dataflow_programming) model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
 

{cocoindex-0.1.49 → cocoindex-0.1.50}/docs/docs/getting_started/quickstart.md

@@ -154,11 +154,11 @@ The goal of transforming your data is usually to query against it.
 Once you already have your index built, you can directly access the transformed data in the target database.
 CocoIndex also provides utilities for you to do this more seamlessly.
 
-In this example, we'll use the [`psycopg` library](https://www.psycopg.org/) to connect to the database and run queries.
-Please make sure it's installed:
+In this example, we'll use the [`psycopg` library](https://www.psycopg.org/) along with pgvector to connect to the database and run queries on vector data.
+Please make sure the required packages are installed:
 
 ```bash
-pip install psycopg[binary,pool]
+pip install numpy psycopg[binary,pool] pgvector
 ```
 
 ### Step 4.1: Extract common transformations
@@ -169,8 +169,11 @@ i.e. they should use exactly the same embedding model and parameters.
 Let's extract that into a function:
 
 ```python title="quickstart.py"
+from numpy.typing import NDArray
+import numpy as np
+
 @cocoindex.transform_flow()
-def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
+def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[NDArray[np.float32]]:
     return text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
             model="sentence-transformers/all-MiniLM-L6-v2"))
@@ -207,6 +210,7 @@ Now we can create a function to query the index upon a given input query:
 
 ```python title="quickstart.py"
 from psycopg_pool import ConnectionPool
+from pgvector.psycopg import register_vector
 
 def search(pool: ConnectionPool, query: str, top_k: int = 5):
     # Get the table name, for the export target in the text_embedding_flow above.
@@ -215,9 +219,10 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5):
     query_vector = text_to_embedding.eval(query)
     # Run the query and get the results.
     with pool.connection() as conn:
+        register_vector(conn)
         with conn.cursor() as cur:
             cur.execute(f"""
-                SELECT filename, text, embedding <=> %s::vector AS distance
+                SELECT filename, text, embedding <=> %s AS distance
                 FROM {table_name} ORDER BY distance LIMIT %s
             """, (query_vector, top_k))
             return [
@@ -236,7 +241,7 @@ There're two CocoIndex-specific logic:
 
 2.  Evaluate the transform flow defined above with the input query, to get the embedding.
     It's done by the `eval()` method of the transform flow `text_to_embedding`.
-    The return type of this method is `list[float]` as declared in the `text_to_embedding()` function (`cocoindex.DataSlice[list[float]]`).
+    The return type of this method is `NDArray[np.float32]` as declared in the `text_to_embedding()` function (`cocoindex.DataSlice[NDArray[np.float32]]`).
 
 ### Step 4.3: Add the main script logic
 

{cocoindex-0.1.49 → cocoindex-0.1.50}/docs/docs/query.mdx

@@ -41,7 +41,7 @@ The [quickstart](getting_started/quickstart#step-41-extract-common-transformatio
 
 ```python
 @cocoindex.transform_flow()
-def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
+def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[NDArray[np.float32]]:
     return text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
             model="sentence-transformers/all-MiniLM-L6-v2"))
@@ -61,7 +61,7 @@ with doc["chunks"].row() as chunk:
     chunk["embedding"] = chunk["text"].call(text_to_embedding)
 ```
 
-Any time, you can call the `eval()` method with specific string, which will return a `list[float]`:
+Any time, you can call the `eval()` method with specific string, which will return a `NDArray[np.float32]`:
 
 ```python
 print(text_to_embedding.eval("Hello, world!"))
@@ -93,7 +93,7 @@ For example:
 
 ```python
 table_name = cocoindex.utils.get_target_storage_default_name(text_embedding_flow, "doc_embeddings")
-query = f"SELECT filename, text FROM {table_name} ORDER BY embedding <=> %s::vector DESC LIMIT 5"
+query = f"SELECT filename, text FROM {table_name} ORDER BY embedding <=> %s DESC LIMIT 5"
 ...
 ```
 

{cocoindex-0.1.49 → cocoindex-0.1.50}/examples/fastapi_server_docker/main.py

@@ -2,7 +2,9 @@ import cocoindex
 import uvicorn
 from dotenv import load_dotenv
 from fastapi import FastAPI, Query
+from fastapi import Request
 from psycopg_pool import ConnectionPool
+from contextlib import asynccontextmanager
 import os
 
 
@@ -86,27 +88,31 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5):
             ]
 
 
-fastapi_app = FastAPI()
-
-
-@fastapi_app.on_event("startup")
-def startup_event():
+@asynccontextmanager
+def lifespan(app: FastAPI):
     load_dotenv()
     cocoindex.init()
-    # Initialize database connection pool
-    fastapi_app.state.pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
+    pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
+    app.state.pool = pool
+    try:
+        yield
+    finally:
+        pool.close()
+
+
+fastapi_app = FastAPI(lifespan=lifespan)
 
 
 @fastapi_app.get("/search")
 def search_endpoint(
+    request: Request,
     q: str = Query(..., description="Search query"),
     limit: int = Query(5, description="Number of results"),
 ):
-    results = search(fastapi_app.state.pool, q, limit)
+    pool = request.app.state.pool
+    results = search(pool, q, limit)
     return {"results": results}
 
 
 if __name__ == "__main__":
-    load_dotenv()
-    cocoindex.init()
     uvicorn.run(fastapi_app, host="0.0.0.0", port=8080)

{cocoindex-0.1.49 → cocoindex-0.1.50}/examples/text_embedding/Text_Embedding.ipynb

@@ -45,7 +45,7 @@
    },
    "outputs": [],
    "source": [
-    "%pip install cocoindex python-dotenv psycopg[binary,pool]"
+    "%pip install cocoindex numpy python-dotenv psycopg[binary,pool] pgvector"
    ]
   },
   {
@@ -164,7 +164,10 @@
     "from dotenv import load_dotenv\n",
     "import os\n",
     "from psycopg_pool import ConnectionPool\n",
-    "import cocoindex\n"
+    "from pgvector.psycopg import register_vector\n",
+    "import cocoindex\n",
+    "from numpy.typing import NDArray\n",
+    "import numpy as np\n"
    ]
   },
   {
@@ -187,7 +190,7 @@
     "%%writefile -a main.py\n",
     "\n",
     "@cocoindex.transform_flow()\n",
-    "def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:\n",
+    "def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[NDArray[np.float32]]:\n",
     "    \"\"\"\n",
     "    Embed the text using a SentenceTransformer model.\n",
     "    This is shared logic between indexing and querying.\n",
@@ -274,9 +277,10 @@
     "    query_vector = text_to_embedding.eval(query)\n",
     "    # Run the query and get the results.\n",
     "    with pool.connection() as conn:\n",
+    "        register_vector(conn)\n",
     "        with conn.cursor() as cur:\n",
     "            cur.execute(f\"\"\"\n",
-    "                SELECT filename, text, embedding <=> %s::vector AS distance\n",
+    "                SELECT filename, text, embedding <=> %s AS distance\n",
     "                FROM {table_name} ORDER BY distance LIMIT %s\n",
     "            \"\"\", (query_vector, top_k))\n",
     "            return [\n",

{cocoindex-0.1.49 → cocoindex-0.1.50}/examples/text_embedding/main.py

@@ -1,13 +1,16 @@
 from dotenv import load_dotenv
 from psycopg_pool import ConnectionPool
+from pgvector.psycopg import register_vector
 import cocoindex
 import os
+from numpy.typing import NDArray
+import numpy as np
 
 
 @cocoindex.transform_flow()
 def text_to_embedding(
     text: cocoindex.DataSlice[str],
-) -> cocoindex.DataSlice[list[float]]:
+) -> cocoindex.DataSlice[NDArray[np.float32]]:
     """
     Embed the text using a SentenceTransformer model.
     This is a shared logic between indexing and querying, so extract it as a function.
@@ -71,10 +74,11 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5):
     query_vector = text_to_embedding.eval(query)
     # Run the query and get the results.
     with pool.connection() as conn:
+        register_vector(conn)
         with conn.cursor() as cur:
             cur.execute(
                 f"""
-                SELECT filename, text, embedding <=> %s::vector AS distance
+                SELECT filename, text, embedding <=> %s AS distance
                 FROM {table_name} ORDER BY distance LIMIT %s
             """,
                 (query_vector, top_k),

{cocoindex-0.1.49 → cocoindex-0.1.50}/examples/text_embedding/pyproject.toml

@@ -6,7 +6,9 @@ requires-python = ">=3.10"
 dependencies = [
     "cocoindex>=0.1.42",
     "python-dotenv>=1.0.1",
+    "pgvector>=0.4.1",
     "psycopg[binary,pool]",
+    "numpy",
 ]
 
 [tool.setuptools]

{cocoindex-0.1.49 → cocoindex-0.1.50}/python/cocoindex/__init__.py

@@ -2,7 +2,7 @@
 Cocoindex is a framework for building and running indexing pipelines.
 """
 
-from . import functions, query, sources, storages, cli, utils
+from . import functions, sources, storages, cli, utils
 
 from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow

{cocoindex-0.1.49 → cocoindex-0.1.50}/python/cocoindex/convert.py

@@ -6,6 +6,7 @@ import dataclasses
 import datetime
 import inspect
 import uuid
+import numpy as np
 
 from enum import Enum
 from typing import Any, Callable, get_origin, Mapping
@@ -15,6 +16,7 @@ from .typing import (
     is_namedtuple_type,
     TABLE_TYPES,
     KEY_FIELD_NAME,
+    DtypeRegistry,
 )
 
 
@@ -27,6 +29,8 @@ def encode_engine_value(value: Any) -> Any:
         ]
     if is_namedtuple_type(type(value)):
         return [encode_engine_value(getattr(value, name)) for name in value._fields]
+    if isinstance(value, np.ndarray):
+        return value
     if isinstance(value, (list, tuple)):
         return [encode_engine_value(v) for v in value]
     if isinstance(value, dict):
@@ -122,6 +126,38 @@ def make_engine_value_decoder(
     if src_type_kind == "Uuid":
         return lambda value: uuid.UUID(bytes=value)
 
+    if src_type_kind == "Vector":
+        elem_coco_type_info = analyze_type_info(dst_type_info.elem_type)
+        dtype_info = DtypeRegistry.get_by_kind(elem_coco_type_info.kind)
+
+        def decode_vector(value: Any) -> Any | None:
+            if value is None:
+                if dst_type_info.nullable:
+                    return None
+                raise ValueError(
+                    f"Received null for non-nullable vector `{''.join(field_path)}`"
+                )
+
+            if not isinstance(value, (np.ndarray, list)):
+                raise TypeError(
+                    f"Expected NDArray or list for vector `{''.join(field_path)}`, got {type(value)}"
+                )
+            expected_dim = (
+                dst_type_info.vector_info.dim if dst_type_info.vector_info else None
+            )
+            if expected_dim is not None and len(value) != expected_dim:
+                raise ValueError(
+                    f"Vector dimension mismatch for `{''.join(field_path)}`: "
+                    f"expected {expected_dim}, got {len(value)}"
+                )
+
+            # Use NDArray for supported numeric dtypes, else return list
+            if dtype_info is not None:
+                return np.array(value, dtype=dtype_info.numpy_dtype)
+            return value
+
+        return decode_vector
+
     return lambda value: value
 
 

{cocoindex-0.1.49 → cocoindex-0.1.50}/python/cocoindex/functions.py

@@ -1,6 +1,8 @@
 """All builtin functions."""
 
-from typing import Annotated, Any, TYPE_CHECKING
+from typing import Annotated, Any, TYPE_CHECKING, Literal
+import numpy as np
+from numpy.typing import NDArray
 import dataclasses
 
 from .typing import Float32, Vector, TypeAttr
@@ -66,11 +68,11 @@ class SentenceTransformerEmbedExecutor:
         self._model = sentence_transformers.SentenceTransformer(self.spec.model, **args)
         dim = self._model.get_sentence_embedding_dimension()
         result: type = Annotated[
-            Vector[Float32, dim],  # type: ignore
+            Vector[np.float32, Literal[dim]],  # type: ignore
             TypeAttr("cocoindex.io/vector_origin_text", text.analyzed_value),
         ]
         return result
 
-    def __call__(self, text: str) -> list[Float32]:
-        result: list[Float32] = self._model.encode(text).tolist()
+    def __call__(self, text: str) -> NDArray[np.float32]:
+        result: NDArray[np.float32] = self._model.encode(text, convert_to_numpy=True)
         return result

{cocoindex-0.1.49 → cocoindex-0.1.50}/python/cocoindex/lib.py

@@ -6,7 +6,7 @@ import warnings
 from typing import Callable, Any
 
 from . import _engine  # type: ignore
-from . import flow, query, setting
+from . import flow, setting
 from .convert import dump_engine_object
 
 
@@ -24,7 +24,6 @@ def init(settings: setting.Settings | None = None) -> None:
 def start_server(settings: setting.ServerSettings) -> None:
     """Start the cocoindex server."""
     flow.ensure_all_flows_built()
-    query.ensure_all_handlers_built()
     _engine.start_server(settings.__dict__)