cocoindex 0.1.50__tar.gz → 0.1.51__tar.gz

cocoindex-0.1.51/.github/workflows/_doc_release.yml

@@ -0,0 +1,31 @@
+name: Release Docs
+
+on:
+  workflow_call:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: docs-release
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: yarn
+          cache-dependency-path: docs/yarn.lock
+      - uses: webfactory/ssh-agent@v0.5.0
+        with:
+          ssh-private-key: ${{ secrets.GH_PAGES_DEPLOY }}
+      - name: Deploy to GitHub Pages
+        env:
+          USE_SSH: true
+        run: |
+          export COCOINDEX_DOCS_POSTHOG_API_KEY=${{ vars.COCOINDEX_DOCS_POSTHOG_API_KEY }}
+          export COCOINDEX_DOCS_MIXPANEL_API_KEY=${{ vars.COCOINDEX_DOCS_MIXPANEL_API_KEY }}
+          export COCOINDEX_DOCS_ALGOLIA_APP_ID=${{ vars.COCOINDEX_DOCS_ALGOLIA_APP_ID }}
+          export COCOINDEX_DOCS_ALGOLIA_API_KEY=${{ vars.COCOINDEX_DOCS_ALGOLIA_API_KEY }}
+          git config --global user.email "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_EMAIL }}"
+          git config --global user.name "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_NAME }}"
+          yarn --cwd docs install --frozen-lockfile
+          yarn --cwd docs deploy

{cocoindex-0.1.50 → cocoindex-0.1.51}/.github/workflows/_test.yml

@@ -27,15 +27,16 @@ jobs:
           run: cargo test --verbose
 
         - uses: actions/setup-python@v5
+          id: setup_python
           with:
             python-version: ${{ matrix.python-version }}
             cache: 'pip'
         - uses: actions/cache@v4
           with:
             path: .venv
-            key: ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
+            key: ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
             restore-keys: |
-              ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-
+              ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-
         - name: Setup venv
           run: |
             python -m venv .venv

cocoindex-0.1.51/.github/workflows/docs.yml

@@ -0,0 +1,34 @@
+name: docs
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - docs/**
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  test-deploy:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: yarn
+          cache-dependency-path: docs/yarn.lock
+      - name: Install dependencies
+        run: yarn --cwd docs install --frozen-lockfile
+      - name: Test build website
+        run: yarn --cwd docs build
+
+  deploy:
+    name: Release Docs
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    uses: ./.github/workflows/_doc_release.yml
+    secrets: inherit

{cocoindex-0.1.50 → cocoindex-0.1.51}/.github/workflows/release.yml

@@ -107,3 +107,9 @@ jobs:
         with:
           command: upload
           args: --non-interactive --skip-existing wheels-*/*
+    
+  release-docs:
+    name: Release Docs
+    needs: [release]
+    uses: ./.github/workflows/_doc_release.yml
+    secrets: inherit

{cocoindex-0.1.50 → cocoindex-0.1.51}/.gitignore

@@ -9,7 +9,7 @@ __pycache__/
 *.so
 
 # Distribution / packaging
-.venv/
+.venv*/
 dist/
 
 .DS_Store

{cocoindex-0.1.50 → cocoindex-0.1.51}/Cargo.lock

@@ -1040,7 +1040,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.50"
+version = "0.1.51"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -1073,6 +1073,7 @@ dependencies = [
  "json5",
  "log",
  "neo4rs",
+ "numpy",
  "owo-colors",
  "pgvector",
  "phf",
@@ -2664,6 +2665,16 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
 
+[[package]]
+name = "matrixmultiply"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
 [[package]]
 name = "md-5"
 version = "0.10.6"
@@ -2731,6 +2742,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "ndarray"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
 [[package]]
 name = "neo4rs"
 version = "0.8.0"
@@ -2796,6 +2822,15 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.1.0"
@@ -2851,6 +2886,22 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "numpy"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29f1dee9aa8d3f6f8e8b9af3803006101bb3653866ef056d530d53ae68587191"
+dependencies = [
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash 2.1.1",
+]
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -3228,11 +3279,10 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.24.2"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
+checksum = "f239d656363bcee73afef85277f1b281e8ac6212a1d42aa90e55b90ed43c47a4"
 dependencies = [
- "cfg-if",
  "chrono",
  "indoc",
  "libc",
@@ -3247,9 +3297,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-async-runtimes"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0b83dc42f9d41f50d38180dad65f0c99763b65a3ff2a81bf351dd35a1df8bf"
+checksum = "d73cc6b1b7d8b3cef02101d37390dbdfe7e450dfea14921cae80a9534ba59ef2"
 dependencies = [
  "futures",
  "once_cell",
@@ -3260,9 +3310,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.24.2"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
+checksum = "755ea671a1c34044fa165247aaf6f419ca39caa6003aee791a0df2713d8f1b6d"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -3270,9 +3320,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.24.2"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
+checksum = "fc95a2e67091e44791d4ea300ff744be5293f394f1bafd9f78c080814d35956e"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -3280,9 +3330,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.24.2"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
+checksum = "a179641d1b93920829a62f15e87c0ed791b6c8db2271ba0fd7c2686090510214"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -3292,9 +3342,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.24.2"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
+checksum = "9dff85ebcaab8c441b0e3f7ae40a6963ecea8a9f5e74f647e33fcf5ec9a1e89e"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -3305,9 +3355,9 @@ dependencies = [
 
 [[package]]
 name = "pythonize"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5bcac0d0b71821f0d69e42654f1e15e5c94b85196446c4de9588951a2117e7b"
+checksum = "597907139a488b22573158793aa7539df36ae863eba300c75f3a0d65fc475e27"
 dependencies = [
  "pyo3",
  "serde",
@@ -3462,6 +3512,12 @@ dependencies = [
  "getrandom 0.3.2",
 ]
 
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.11"

{cocoindex-0.1.50 → cocoindex-0.1.51}/Cargo.toml

@@ -2,8 +2,9 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.50"
+version = "0.1.51"
 edition = "2024"
+rust-version = "1.86"
 
 [profile.release]
 codegen-units = 1
@@ -14,9 +15,9 @@ name = "cocoindex_engine"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.24.1", features = ["chrono"] }
-pythonize = "0.24.0"
-pyo3-async-runtimes = { version = "0.24.0", features = ["tokio-runtime"] }
+pyo3 = { version = "0.25.0", features = ["chrono", "auto-initialize"] }
+pythonize = "0.25.0"
+pyo3-async-runtimes = { version = "0.25.0", features = ["tokio-runtime"] }
 
 anyhow = { version = "1.0.97", features = ["std"] }
 async-trait = "0.1.88"
@@ -113,3 +114,4 @@ json5 = "0.4.1"
 aws-config = "1.6.2"
 aws-sdk-s3 = "1.85.0"
 aws-sdk-sqs = "1.67.0"
+numpy = "0.25.0"

{cocoindex-0.1.50 → cocoindex-0.1.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.50
+Version: 0.1.51
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0

{cocoindex-0.1.50 → cocoindex-0.1.51}/docs/docs/getting_started/quickstart.md

@@ -158,7 +158,7 @@ In this example, we'll use the [`psycopg` library](https://www.psycopg.org/) alo
 Please make sure the required packages are installed:
 
 ```bash
-pip install numpy psycopg[binary,pool] pgvector
+pip install numpy "psycopg[binary,pool]" pgvector
 ```
 
 ### Step 4.1: Extract common transformations

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/code_embedding/pyproject.toml

@@ -2,7 +2,7 @@
 name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = ["cocoindex>=0.1.42", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -2,7 +2,7 @@
 name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = ["cocoindex>=0.1.42"]
 
 [tool.setuptools]

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/manuals_llm_extraction/pyproject.toml

@@ -2,7 +2,7 @@
 name = "manuals-llm-extraction"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = ["cocoindex>=0.1.42", "marker-pdf>=1.5.2"]
 
 [tool.setuptools]

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/pdf_embedding/pyproject.toml

@@ -2,7 +2,7 @@
 name = "pdf-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "cocoindex>=0.1.42",
     "python-dotenv>=1.0.1",

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/product_recommendation/pyproject.toml

@@ -2,7 +2,7 @@
 name = "cocoindex-ecommerce-taxonomy"
 version = "0.1.0"
 description = "Simple example for CocoIndex: extract taxonomy from e-commerce products and build knowledge graph."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = ["cocoindex>=0.1.42", "jinja2>=3.1.6"]
 
 [tool.setuptools]

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/text_embedding/pyproject.toml

@@ -2,7 +2,7 @@
 name = "text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "cocoindex>=0.1.42",
     "python-dotenv>=1.0.1",

{cocoindex-0.1.50 → cocoindex-0.1.51}/examples/text_embedding_qdrant/pyproject.toml

@@ -2,7 +2,7 @@
 name = "text-embedding-qdrant"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "cocoindex>=0.1.42",
     "python-dotenv>=1.0.1",

{cocoindex-0.1.50 → cocoindex-0.1.51}/pyproject.toml

@@ -35,4 +35,3 @@ dev = ["ruff"]
 python_version = "3.11"
 strict = true
 files = "python/cocoindex"
-exclude = "python/cocoindex/tests"

cocoindex-0.1.51/python/cocoindex/__init__.py

@@ -0,0 +1,68 @@
+"""
+Cocoindex is a framework for building and running indexing pipelines.
+"""
+
+from . import functions, sources, storages, cli, utils
+
+from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
+from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
+from .flow import flow_def
+from .flow import EvaluateAndDumpOptions, GeneratedField
+from .flow import update_all_flows_async, FlowLiveUpdater, FlowLiveUpdaterOptions
+from .lib import init, start_server, stop, main_fn
+from .llm import LlmSpec, LlmApiType
+from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
+from .setting import DatabaseConnectionSpec, Settings, ServerSettings
+from .setting import get_app_namespace
+from .typing import Float32, Float64, LocalDateTime, OffsetDateTime, Range, Vector, Json
+
+__all__ = [
+    # Submodules
+    "_engine",
+    "functions",
+    "sources",
+    "storages",
+    "cli",
+    "utils",
+    # Auth registry
+    "AuthEntryReference",
+    "add_auth_entry",
+    "ref_auth_entry",
+    # Flow
+    "FlowBuilder",
+    "DataScope",
+    "DataSlice",
+    "Flow",
+    "transform_flow",
+    "flow_def",
+    "EvaluateAndDumpOptions",
+    "GeneratedField",
+    "update_all_flows_async",
+    "FlowLiveUpdater",
+    "FlowLiveUpdaterOptions",
+    # Lib
+    "init",
+    "start_server",
+    "stop",
+    "main_fn",
+    # LLM
+    "LlmSpec",
+    "LlmApiType",
+    # Index
+    "VectorSimilarityMetric",
+    "VectorIndexDef",
+    "IndexOptions",
+    # Settings
+    "DatabaseConnectionSpec",
+    "Settings",
+    "ServerSettings",
+    "get_app_namespace",
+    # Typing
+    "Float32",
+    "Float64",
+    "LocalDateTime",
+    "OffsetDateTime",
+    "Range",
+    "Vector",
+    "Json",
+]

{cocoindex-0.1.50 → cocoindex-0.1.51}/python/cocoindex/cli.py

@@ -237,7 +237,15 @@ def show(app_flow_specifier: str, color: bool, verbose: bool) -> None:
 
 @cli.command()
 @click.argument("app_target", type=str)
-def setup(app_target: str) -> None:
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Force setup without confirmation prompts.",
+)
+def setup(app_target: str, force: bool) -> None:
     """
     Check and apply backend setup changes for flows, including the internal and target storage
     (to export).
@@ -252,7 +260,7 @@ def setup(app_target: str) -> None:
     if setup_status.is_up_to_date():
         click.echo("No changes need to be pushed.")
         return
-    if not click.confirm(
+    if not force and not click.confirm(
         "Changes need to be pushed. Continue? [yes/N]",
         default=False,
         show_default=False,
@@ -275,7 +283,17 @@ def setup(app_target: str) -> None:
     "even if not defined in the current process."
     "If used, APP_TARGET and any listed flow names are ignored.",
 )
-def drop(app_target: str | None, flow_name: tuple[str, ...], drop_all: bool) -> None:
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Force drop without confirmation prompts.",
+)
+def drop(
+    app_target: str | None, flow_name: tuple[str, ...], drop_all: bool, force: bool
+) -> None:
     """
     Drop the backend setup for flows.
 
@@ -328,7 +346,7 @@ def drop(app_target: str | None, flow_name: tuple[str, ...], drop_all: bool) ->
     if setup_status.is_up_to_date():
         click.echo("No flows need to be dropped.")
         return
-    if not click.confirm(
+    if not force and not click.confirm(
         f"\nThis will apply changes to drop setup for: {', '.join(flow_names)}. Continue? [yes/N]",
         default=False,
         show_default=False,

{cocoindex-0.1.50 → cocoindex-0.1.51}/python/cocoindex/convert.py

@@ -127,8 +127,7 @@ def make_engine_value_decoder(
         return lambda value: uuid.UUID(bytes=value)
 
     if src_type_kind == "Vector":
-        elem_coco_type_info = analyze_type_info(dst_type_info.elem_type)
-        dtype_info = DtypeRegistry.get_by_kind(elem_coco_type_info.kind)
+        dtype_info = DtypeRegistry.get_by_dtype(dst_type_info.np_number_type)
 
         def decode_vector(value: Any) -> Any | None:
             if value is None:
@@ -237,7 +236,12 @@ def dump_engine_object(v: Any) -> Any:
         nanos = int((total_secs - secs) * 1e9)
         return {"secs": secs, "nanos": nanos}
     elif hasattr(v, "__dict__"):
-        s = {k: dump_engine_object(v) for k, v in v.__dict__.items()}
+        s = {}
+        for k, val in v.__dict__.items():
+            if val is None:
+                # Skip None values
+                continue
+            s[k] = dump_engine_object(val)
         if hasattr(v, "kind") and "kind" not in s:
             s["kind"] = v.kind
         return s

{cocoindex-0.1.50 → cocoindex-0.1.51}/python/cocoindex/setting.py

@@ -62,18 +62,22 @@ def _load_field(
 class Settings:
     """Settings for the cocoindex library."""
 
-    database: DatabaseConnectionSpec
+    database: DatabaseConnectionSpec | None = None
     app_namespace: str = ""
 
     @classmethod
     def from_env(cls) -> Self:
         """Load settings from environment variables."""
 
-        db_kwargs: dict[str, str] = dict()
-        _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
-        _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
-        _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
-        database = DatabaseConnectionSpec(**db_kwargs)
+        database_url = os.getenv("COCOINDEX_DATABASE_URL")
+        if database_url is not None:
+            db_kwargs: dict[str, str] = dict()
+            _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
+            _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
+            _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
+            database = DatabaseConnectionSpec(**db_kwargs)
+        else:
+            database = None
 
         app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", "")