cocoindex 0.1.49__cp311-cp311-win_amd64.whl → 0.1.51__cp311-cp311-win_amd64.whl

cocoindex/__init__.py

@@ -2,7 +2,7 @@
 Cocoindex is a framework for building and running indexing pipelines.
 """
 
-from . import functions, query, sources, storages, cli, utils
+from . import functions, sources, storages, cli, utils
 
 from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
@@ -15,3 +15,54 @@ from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
 from .setting import DatabaseConnectionSpec, Settings, ServerSettings
 from .setting import get_app_namespace
 from .typing import Float32, Float64, LocalDateTime, OffsetDateTime, Range, Vector, Json
+
+__all__ = [
+    # Submodules
+    "_engine",
+    "functions",
+    "sources",
+    "storages",
+    "cli",
+    "utils",
+    # Auth registry
+    "AuthEntryReference",
+    "add_auth_entry",
+    "ref_auth_entry",
+    # Flow
+    "FlowBuilder",
+    "DataScope",
+    "DataSlice",
+    "Flow",
+    "transform_flow",
+    "flow_def",
+    "EvaluateAndDumpOptions",
+    "GeneratedField",
+    "update_all_flows_async",
+    "FlowLiveUpdater",
+    "FlowLiveUpdaterOptions",
+    # Lib
+    "init",
+    "start_server",
+    "stop",
+    "main_fn",
+    # LLM
+    "LlmSpec",
+    "LlmApiType",
+    # Index
+    "VectorSimilarityMetric",
+    "VectorIndexDef",
+    "IndexOptions",
+    # Settings
+    "DatabaseConnectionSpec",
+    "Settings",
+    "ServerSettings",
+    "get_app_namespace",
+    # Typing
+    "Float32",
+    "Float64",
+    "LocalDateTime",
+    "OffsetDateTime",
+    "Range",
+    "Vector",
+    "Json",
+]

cocoindex/cli.py

@@ -237,7 +237,15 @@ def show(app_flow_specifier: str, color: bool, verbose: bool) -> None:
 
 @cli.command()
 @click.argument("app_target", type=str)
-def setup(app_target: str) -> None:
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Force setup without confirmation prompts.",
+)
+def setup(app_target: str, force: bool) -> None:
     """
     Check and apply backend setup changes for flows, including the internal and target storage
     (to export).
@@ -252,7 +260,7 @@ def setup(app_target: str) -> None:
     if setup_status.is_up_to_date():
         click.echo("No changes need to be pushed.")
         return
-    if not click.confirm(
+    if not force and not click.confirm(
         "Changes need to be pushed. Continue? [yes/N]",
         default=False,
         show_default=False,
@@ -275,7 +283,17 @@ def setup(app_target: str) -> None:
     "even if not defined in the current process."
     "If used, APP_TARGET and any listed flow names are ignored.",
 )
-def drop(app_target: str | None, flow_name: tuple[str, ...], drop_all: bool) -> None:
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Force drop without confirmation prompts.",
+)
+def drop(
+    app_target: str | None, flow_name: tuple[str, ...], drop_all: bool, force: bool
+) -> None:
     """
     Drop the backend setup for flows.
 
@@ -328,7 +346,7 @@ def drop(app_target: str | None, flow_name: tuple[str, ...], drop_all: bool) ->
     if setup_status.is_up_to_date():
         click.echo("No flows need to be dropped.")
         return
-    if not click.confirm(
+    if not force and not click.confirm(
         f"\nThis will apply changes to drop setup for: {', '.join(flow_names)}. Continue? [yes/N]",
         default=False,
         show_default=False,

cocoindex/convert.py

@@ -6,6 +6,7 @@ import dataclasses
 import datetime
 import inspect
 import uuid
+import numpy as np
 
 from enum import Enum
 from typing import Any, Callable, get_origin, Mapping
@@ -15,6 +16,7 @@ from .typing import (
     is_namedtuple_type,
     TABLE_TYPES,
     KEY_FIELD_NAME,
+    DtypeRegistry,
 )
 
 
@@ -27,6 +29,8 @@ def encode_engine_value(value: Any) -> Any:
         ]
     if is_namedtuple_type(type(value)):
         return [encode_engine_value(getattr(value, name)) for name in value._fields]
+    if isinstance(value, np.ndarray):
+        return value
     if isinstance(value, (list, tuple)):
         return [encode_engine_value(v) for v in value]
     if isinstance(value, dict):
@@ -122,6 +126,37 @@ def make_engine_value_decoder(
     if src_type_kind == "Uuid":
         return lambda value: uuid.UUID(bytes=value)
 
+    if src_type_kind == "Vector":
+        dtype_info = DtypeRegistry.get_by_dtype(dst_type_info.np_number_type)
+
+        def decode_vector(value: Any) -> Any | None:
+            if value is None:
+                if dst_type_info.nullable:
+                    return None
+                raise ValueError(
+                    f"Received null for non-nullable vector `{''.join(field_path)}`"
+                )
+
+            if not isinstance(value, (np.ndarray, list)):
+                raise TypeError(
+                    f"Expected NDArray or list for vector `{''.join(field_path)}`, got {type(value)}"
+                )
+            expected_dim = (
+                dst_type_info.vector_info.dim if dst_type_info.vector_info else None
+            )
+            if expected_dim is not None and len(value) != expected_dim:
+                raise ValueError(
+                    f"Vector dimension mismatch for `{''.join(field_path)}`: "
+                    f"expected {expected_dim}, got {len(value)}"
+                )
+
+            # Use NDArray for supported numeric dtypes, else return list
+            if dtype_info is not None:
+                return np.array(value, dtype=dtype_info.numpy_dtype)
+            return value
+
+        return decode_vector
+
     return lambda value: value
 
 
@@ -201,7 +236,12 @@ def dump_engine_object(v: Any) -> Any:
         nanos = int((total_secs - secs) * 1e9)
         return {"secs": secs, "nanos": nanos}
     elif hasattr(v, "__dict__"):
-        s = {k: dump_engine_object(v) for k, v in v.__dict__.items()}
+        s = {}
+        for k, val in v.__dict__.items():
+            if val is None:
+                # Skip None values
+                continue
+            s[k] = dump_engine_object(val)
         if hasattr(v, "kind") and "kind" not in s:
             s["kind"] = v.kind
         return s

cocoindex/functions.py

@@ -1,6 +1,8 @@
 """All builtin functions."""
 
-from typing import Annotated, Any, TYPE_CHECKING
+from typing import Annotated, Any, TYPE_CHECKING, Literal
+import numpy as np
+from numpy.typing import NDArray
 import dataclasses
 
 from .typing import Float32, Vector, TypeAttr
@@ -66,11 +68,11 @@ class SentenceTransformerEmbedExecutor:
         self._model = sentence_transformers.SentenceTransformer(self.spec.model, **args)
         dim = self._model.get_sentence_embedding_dimension()
         result: type = Annotated[
-            Vector[Float32, dim],  # type: ignore
+            Vector[np.float32, Literal[dim]],  # type: ignore
             TypeAttr("cocoindex.io/vector_origin_text", text.analyzed_value),
         ]
         return result
 
-    def __call__(self, text: str) -> list[Float32]:
-        result: list[Float32] = self._model.encode(text).tolist()
+    def __call__(self, text: str) -> NDArray[np.float32]:
+        result: NDArray[np.float32] = self._model.encode(text, convert_to_numpy=True)
         return result

cocoindex/lib.py

@@ -6,7 +6,7 @@ import warnings
 from typing import Callable, Any
 
 from . import _engine  # type: ignore
-from . import flow, query, setting
+from . import flow, setting
 from .convert import dump_engine_object
 
 
@@ -24,7 +24,6 @@ def init(settings: setting.Settings | None = None) -> None:
 def start_server(settings: setting.ServerSettings) -> None:
     """Start the cocoindex server."""
     flow.ensure_all_flows_built()
-    query.ensure_all_handlers_built()
     _engine.start_server(settings.__dict__)
 
 

cocoindex/setting.py

@@ -62,18 +62,22 @@ def _load_field(
 class Settings:
     """Settings for the cocoindex library."""
 
-    database: DatabaseConnectionSpec
+    database: DatabaseConnectionSpec | None = None
     app_namespace: str = ""
 
     @classmethod
     def from_env(cls) -> Self:
         """Load settings from environment variables."""
 
-        db_kwargs: dict[str, str] = dict()
-        _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
-        _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
-        _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
-        database = DatabaseConnectionSpec(**db_kwargs)
+        database_url = os.getenv("COCOINDEX_DATABASE_URL")
+        if database_url is not None:
+            db_kwargs: dict[str, str] = dict()
+            _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
+            _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
+            _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
+            database = DatabaseConnectionSpec(**db_kwargs)
+        else:
+            database = None
 
         app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", "")