cocoindex 0.2.2__tar.gz → 0.2.3__tar.gz

cocoindex-0.2.3/.github/workflows/_test.yml

@@ -0,0 +1,65 @@
+name: Run Tests
+
+on:
+    workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+    build-test:
+        strategy:
+            matrix:
+                python-version: [3.11]
+                platform:
+                  - { runner: ubuntu-latest, python_exec: ".venv/bin/python" }
+                  - { runner: ubuntu-24.04-arm, python_exec: ".venv/bin/python" }
+                  - { runner: macos-latest, python_exec: ".venv/bin/python" }
+                  - { runner: macos-13, python_exec: ".venv/bin/python" }
+                  - { runner: windows-latest, python_exec: ".venv\\Scripts\\python" }
+        runs-on: ${{ matrix.platform.runner }}
+        steps:
+        - uses: actions/checkout@v4
+
+        - uses: actions/setup-python@v5
+          id: setup_python
+          with:
+            python-version: ${{ matrix.python-version }}
+            cache: 'pip'
+
+        - run: rustup toolchain install stable --profile minimal
+        - name: Rust Cache
+          uses: Swatinem/rust-cache@v2
+          with:
+            key: rust-${{ matrix.platform.runner }}-${{ matrix.python-version }}
+        - name: Rust tests
+          run: cargo test --verbose
+
+        - uses: actions/cache@v4
+          with:
+            path: .venv
+            key: pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
+            restore-keys: |
+              pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-
+
+        - name: Setup venv
+          run: |
+            python -m venv .venv
+        - name: Install Python toolchains
+          run: |
+            ${{ matrix.platform.python_exec }} -m pip install maturin mypy pytest pytest-asyncio
+        - name: Python build
+          run: |
+            ${{ matrix.platform.python_exec }} -m maturin develop -E all
+        - name: Python type check (mypy)
+          run: |
+            ${{ matrix.platform.python_exec }} -m mypy python
+        - name: Python tests
+          if: ${{ !startsWith(matrix.platform.runner, 'windows') }}
+          run: |
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests
+        - name: Python tests (Windows cmd)
+          if: ${{ startsWith(matrix.platform.runner, 'windows') }}
+          shell: cmd  # Use `cmd` to run test for Windows, as PowerShell doesn't detect exit code by `os._exit(0)` correctly.
+          run: |
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests

{cocoindex-0.2.2 → cocoindex-0.2.3}/.github/workflows/release.yml

@@ -1,8 +1,7 @@
-# This file is autogenerated by maturin v1.8.1
-# To update, run
-#
-#    maturin generate-ci github
+# This workflow can be triggered on tags push (automatic release) or manually on any branch.
 #
+# - When triggered on tags push, it will build and publishes a new version including docs.
+# - When triggered manually, it's a dry-run: only build, without publishing anything.
 name: release
 
 on:
@@ -31,11 +30,11 @@ jobs:
     strategy:
       matrix:
         platform:
-          - { os: linux, runner: ubuntu-24.04, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
-          - { os: linux, runner: ubuntu-24.04, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
-          - { os: windows, runner: windows-latest, target: x64 }
+          - { os: linux, runner: ubuntu-latest, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
+          - { os: linux, runner: ubuntu-24.04-arm, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
+          - { os: macos, runner: macos-latest, target: aarch64 }
           - { os: macos, runner: macos-13, target: x86_64 }
-          - { os: macos, runner: macos-14, target: aarch64 }
+          - { os: windows, runner: windows-latest, target: x64 }
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
@@ -43,12 +42,12 @@ jobs:
           name: Cargo.toml
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.x
+          python-version: 3.13
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter
+          args: --release --out dist
           sccache: 'true'
           manylinux: auto
           container: ${{ matrix.platform.container }}
@@ -58,6 +57,24 @@ jobs:
           name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}
           path: dist
 
+  test-abi3:
+    runs-on: ubuntu-24.04
+    needs: build
+    strategy:
+      matrix:
+        py: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheels-linux-x86_64
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
+      - run: python -V
+      - run: pip install --find-links=./ cocoindex
+      - run: python -c "import cocoindex, sys; print('import ok on', sys.version)"
+
+
   sdist:
     runs-on: ubuntu-latest
     needs: [create-versioned-toml]
@@ -80,7 +97,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: [create-versioned-toml, build, sdist]
+    needs: [create-versioned-toml, build, test-abi3, sdist]
     permissions:
       # Use to sign the release artifacts
       id-token: write
@@ -111,5 +128,6 @@ jobs:
   release-docs:
     name: Release Docs
     needs: [release]
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
     uses: ./.github/workflows/_doc_release.yml
     secrets: inherit

{cocoindex-0.2.2 → cocoindex-0.2.3}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.2.2 → cocoindex-0.2.3}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.2"
+version = "0.2.3"
 edition = "2024"
 rust-version = "1.88"
 readme = "README.md"
@@ -16,7 +16,12 @@ name = "cocoindex_engine"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.25.1", features = ["chrono", "auto-initialize", "uuid"] }
+pyo3 = { version = "0.25.1", features = [
+    "abi3-py311",
+    "auto-initialize",
+    "chrono",
+    "uuid",
+] }
 pythonize = "0.25.0"
 pyo3-async-runtimes = { version = "0.25.0", features = ["tokio-runtime"] }
 

{cocoindex-0.2.2 → cocoindex-0.2.3}/PKG-INFO

@@ -1,6 +1,21 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.2
+Version: 0.2.3
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Typing :: Typed
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -21,6 +36,7 @@ Provides-Extra: colpali
 Provides-Extra: all
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
+Keywords: indexing,real-time,incremental,pipeline,search,ai,etl,rag,dataflow,context-engineering
 Author-email: CocoIndex <cocoindex.io@gmail.com>
 License-Expression: Apache-2.0
 Requires-Python: >=3.11

{cocoindex-0.2.2 → cocoindex-0.2.3}/docs/docs/examples/examples/simple_vector_index.md

@@ -71,34 +71,21 @@ with data_scope["documents"].row() as doc:
 ### Embed each chunk 
 
 ```python
-@cocoindex.transform_flow()
-def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
-    """
-    Embed the text using a SentenceTransformer model.
-    This is a shared logic between indexing and querying, so extract it as a function.
-    """
-    return text.transform(
+with doc["chunks"].row() as chunk:
+    chunk["embedding"] = chunk["text"].transform(
         cocoindex.functions.SentenceTransformerEmbed(
-            model="sentence-transformers/all-MiniLM-L6-v2"))
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    ) 
+    doc_embeddings.collect(filename=doc["filename"], location=chunk["location"],
+                            text=chunk["text"], embedding=chunk["embedding"])
 ```
-![Embedding](/img/examples/simple_vector_index/embed.png)
-
-This code defines a transformation function that converts text into vector embeddings using the SentenceTransformer model.
-`@cocoindex.transform_flow()` is needed to share the transformation across indexing and query.
-This decorator marks this as a reusable transformation flow that can be called on specific input data from user code using `eval()`, as shown in the search function below. 
 
 The `MiniLM-L6-v2` model is a good balance of speed and quality for text embeddings, though you can swap in other SentenceTransformer models as needed.
 
 <DocumentationButton url="https://cocoindex.io/docs/ops/functions#sentencetransformerembed" text="SentenceTransformerEmbed" margin="0 0 16px 0" />
- 
-Plug in the `text_to_embedding` function and collect the embeddings.
 
-```python
-with doc["chunks"].row() as chunk:
-    chunk["embedding"] = text_to_embedding(chunk["text"])
-    doc_embeddings.collect(filename=doc["filename"], location=chunk["location"],
-                            text=chunk["text"], embedding=chunk["embedding"])
-```
+![Embedding](/img/examples/simple_vector_index/embed.png)
 
 ## Export the embeddings
 
@@ -119,10 +106,32 @@ CocoIndex supports other vector databases as well, with 1-line switch.
 
 ## Query the index
 
+### Define a shared flow for both indexing and querying
+
+```python
+@cocoindex.transform_flow()
+def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
+    """
+    Embed the text using a SentenceTransformer model.
+    This is a shared logic between indexing and querying, so extract it as a function.
+    """
+    return text.transform(
+        cocoindex.functions.SentenceTransformerEmbed(
+            model="sentence-transformers/all-MiniLM-L6-v2"))
+```
+
+This code defines a transformation function that converts text into vector embeddings using the SentenceTransformer model.
+`@cocoindex.transform_flow()` is needed to share the transformation across indexing and query.
+
+This decorator marks this as a reusable transformation flow that can be called on specific input data from user code using `eval()`, as shown in the search function below. 
+
+### Write query
+
 CocoIndex doesn't provide additional query interface at the moment. We can write SQL or rely on the query engine by the target storage, if any.
 
 <DocumentationButton url="https://cocoindex.io/docs/ops/targets#postgres" text="Postgres" margin="0 0 16px 0" />
 
+
 ```python
 def search(pool: ConnectionPool, query: str, top_k: int = 5):
     table_name = cocoindex.utils.get_target_storage_default_name(text_embedding_flow, "doc_embeddings")
@@ -166,6 +175,19 @@ if __name__ == "__main__":
     _main()
 ```
 
+In the function above, most parts are standard query logic - you can use any libraries you like.
+There're two CocoIndex-specific logic:
+
+1.  Get the table name from the export target in the `text_embedding_flow` above.
+    Since the table name for the `Postgres` target is not explicitly specified in the `export()` call,
+    CocoIndex uses a default name.
+    `cocoindex.utils.get_target_default_name()` is a utility function to get the default table name for this case.
+
+2.  Evaluate the transform flow defined above with the input query, to get the embedding.
+    It's done by the `eval()` method of the transform flow `text_to_embedding`.
+    The return type of this method is `NDArray[np.float32]` as declared in the `text_to_embedding()` function (`cocoindex.DataSlice[NDArray[np.float32]]`).
+
+
 
 ## Time to have fun!
 - Run the following command to setup and update the index.

cocoindex-0.2.3/docs/docs/getting_started/quickstart.md

@@ -0,0 +1,177 @@
+---
+title: Quickstart
+description: Get started with CocoIndex in 10 minutes
+---
+
+import { GitHubButton, YouTubeButton, DocumentationButton } from '../../src/components/GitHubButton';
+
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex-quickstart" margin="0 0 24px 0"/>
+<YouTubeButton url="https://www.youtube.com/watch?v=gv5R8nOXsWU" margin="0 0 24px 0"/>
+
+In this tutorial, we’ll build an index with text embeddings, keeping it minimal and focused on the core indexing flow.
+
+
+## Flow Overview
+![Flow](/img/examples/simple_vector_index/flow.png)
+
+1. Read text files from the local filesystem
+2. Chunk each document
+3. For each chunk, embed it with a text embedding model
+4. Store the embeddings in a vector database for retrieval
+
+
+## Setup
+1.  Install CocoIndex:
+
+    ```bash
+    pip install -U 'cocoindex[embeddings]'
+    ```
+
+2.  [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres).
+
+3.  Create a new directory for your project:
+
+    ```bash
+    mkdir cocoindex-quickstart
+    cd cocoindex-quickstart
+    ```
+
+4.  Place input files in a directory `markdown_files`. You may download from [markdown_files.zip](markdown_files.zip).
+
+
+## Define a flow
+
+Create a new file `main.py` and define a flow.
+
+```python title="main.py"
+import cocoindex
+
+@cocoindex.flow_def(name="TextEmbedding")
+def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    # ... See subsections below for function body
+```
+
+###  Add Source and Collector
+
+```python title="main.py"
+# add source
+data_scope["documents"] = flow_builder.add_source(
+    cocoindex.sources.LocalFile(path="markdown_files"))
+
+# add data collector
+doc_embeddings = data_scope.add_collector()
+```
+
+`flow_builder.add_source` will create a table with sub fields (`filename`, `content`)
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Source" />
+
+<DocumentationButton url="https://cocoindex.io/docs/core/flow_def#data-collector" text="Data Collector" />
+
+### Process each document 
+
+With CocoIndex, it is easy to process nested data structures.
+
+```python title="main.py"
+with data_scope["documents"].row() as doc:
+    # ... See subsections below for function body
+```
+
+
+#### Chunk each document
+
+```python title="main.py"
+doc["chunks"] = doc["content"].transform(
+    cocoindex.functions.SplitRecursively(),
+    language="markdown", chunk_size=2000, chunk_overlap=500)
+```
+
+We extend a new field `chunks` to each row by *transforming* the `content` field using `SplitRecursively`. The output of the `SplitRecursively` is a KTable representing each chunk of the document.
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/functions#splitrecursively" text="SplitRecursively" margin="0 0 16px 0" />
+
+![Chunking](/img/examples/simple_vector_index/chunk.png)
+
+
+
+#### Embed each chunk and collect the embeddings
+
+```python title="main.py"
+with doc["chunks"].row() as chunk:
+    # embed 
+    chunk["embedding"] = chunk["text"].transform(
+        cocoindex.functions.SentenceTransformerEmbed(
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    )
+
+    # collect
+    doc_embeddings.collect(
+        filename=doc["filename"],
+        location=chunk["location"],
+        text=chunk["text"],
+        embedding=chunk["embedding"],
+    )
+```
+
+This code embeds each chunk using the SentenceTransformer library and collects the results.
+
+![Embedding](/img/examples/simple_vector_index/embed.png)
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/functions#sentencetransformerembed" text="SentenceTransformerEmbed" margin="0 0 16px 0" />
+ 
+### Export the embeddings to Postgres
+
+```python title="main.py"
+doc_embeddings.export(
+    "doc_embeddings",
+    cocoindex.storages.Postgres(),
+    primary_key_fields=["filename", "location"],
+    vector_indexes=[
+        cocoindex.VectorIndexDef(
+            field_name="embedding",
+            metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+        )
+    ],
+)
+```
+
+CocoIndex supports other vector databases as well, with 1-line switch.
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/targets" text="Targets" />
+
+
+## Run the indexing pipeline
+
+- Specify the database URL by environment variable:
+
+    ```bash
+    export COCOINDEX_DATABASE_URL="postgresql://cocoindex:cocoindex@localhost:5432/cocoindex"
+    ```
+
+- Build the index:
+
+    ```bash
+    cocoindex update --setup main.py
+    ```
+
+CocoIndex will run for a few seconds and populate the target table with data as declared by the flow. It will output the following statistics:
+
+```
+documents: 3 added, 0 removed, 0 updated
+```
+
+That's it for the main indexing flow.
+
+
+## End to end: Query the index (Optional)
+
+If you want to build a end to end query flow that also searches the index, you can follow the [simple_vector_index](https://cocoindex.io/docs/examples/simple_vector_index#query-the-index) example.
+
+
+## Next Steps
+
+Next, you may want to:
+
+*   Learn about [CocoIndex Basics](../core/basics.md).
+*   Explore more of what you can build with CocoIndex in the [examples](https://cocoindex.io/docs/examples) directory. 

{cocoindex-0.2.2 → cocoindex-0.2.3}/docs/docs/ops/sources.md

@@ -313,6 +313,27 @@ The spec takes the following fields:
 *   `included_columns` (`list[str]`, optional): non-primary-key columns to include. If not specified, all non-PK columns are included.
 *   `ordinal_column` (`str`, optional): to specify a non-primary-key column used for change tracking and ordering, e.g. can be a modified timestamp or a monotonic version number. Supported types are integer-like (`bigint`/`integer`) and timestamps (`timestamp`, `timestamptz`).
     `ordinal_column` must not be a primary key column.
+*   `notification` (`cocoindex.sources.PostgresNotification`, optional): when present, enable change capture based on Postgres LISTEN/NOTIFY. It has the following fields:
+    *   `channel_name` (`str`, optional): the Postgres notification channel to listen on. CocoIndex will automatically create the channel with the given name. If omitted, CocoIndex uses `{flow_name}__{source_name}__cocoindex`.
+
+    :::info
+
+    If `notification` is provided, CocoIndex listens for row changes using Postgres LISTEN/NOTIFY and creates the required database objects on demand when the flow starts listening:
+
+    - Function to create notification message: `{channel_name}_n`.
+    - Trigger to react to table changes: `{channel_name}_t` on the specified `table_name`.
+
+    Creation is automatic when listening begins.
+
+    Currently CocoIndex doesn't automatically clean up these objects when the flow is dropped (unlike targets)
+    It's usually OK to leave them as they are, but if you want to clean them up, you can run the following SQL statements to manually drop them:
+
+    ```sql
+    DROP TRIGGER IF EXISTS {channel_name}_t ON "{table_name}";
+    DROP FUNCTION IF EXISTS {channel_name}_n();
+    ```
+
+    :::
 
 ### Schema
 

{cocoindex-0.2.2 → cocoindex-0.2.3}/docs/docusaurus.config.ts

@@ -115,7 +115,7 @@ const config: Config = {
         {
           label: 'Documentation',
           type: 'doc',
-          docId: 'getting_started/overview',
+          docId: 'getting_started/quickstart',
           position: 'left',
         },
         {

{cocoindex-0.2.2 → cocoindex-0.2.3}/docs/src/components/GitHubButton/index.tsx

@@ -35,7 +35,7 @@ type GitHubButtonProps = {
     margin?: string;
 };
 
-function GitHubButton({ url, margin }: GitHubButtonProps): ReactNode {
+function GitHubButton({ url, margin = '0' }: GitHubButtonProps): ReactNode {
     return (
         <Button href={url} margin={margin}>
             <FaGithub style={{ marginRight: '8px', verticalAlign: 'middle', fontSize: '1rem' }} />
@@ -49,7 +49,7 @@ type YouTubeButtonProps = {
     margin?: string;
 };
 
-function YouTubeButton({ url, margin }: YouTubeButtonProps): ReactNode {
+function YouTubeButton({ url, margin = '0' }: YouTubeButtonProps): ReactNode {
     return (
         <Button href={url} margin={margin}>
             <FaYoutube style={{ marginRight: '8px', verticalAlign: 'middle', fontSize: '1rem' }} />