cocoindex 0.1.56__tar.gz → 0.1.58__tar.gz

{cocoindex-0.1.56 → cocoindex-0.1.58}/Cargo.lock

@@ -1040,7 +1040,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.56"
+version = "0.1.58"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.56 → cocoindex-0.1.58}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.56"
+version = "0.1.58"
 edition = "2024"
 rust-version = "1.86"
 

{cocoindex-0.1.56 → cocoindex-0.1.58}/PKG-INFO

@@ -1,16 +1,18 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.56
-Requires-Dist: sentence-transformers>=3.3.1
+Version: 0.1.58
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: watchfiles>=1.1.0
+Requires-Dist: numpy>=1.23.2
 Requires-Dist: pytest ; extra == 'test'
 Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
 Provides-Extra: test
 Provides-Extra: dev
+Provides-Extra: embeddings
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Author-email: CocoIndex <cocoindex.io@gmail.com>

{cocoindex-0.1.56 → cocoindex-0.1.58}/docs/docs/core/basics.md

@@ -101,4 +101,4 @@ As an indexing flow is long-lived, it needs to store intermediate data to keep t
 CocoIndex uses internal storage for this purpose.
 
 Currently, CocoIndex uses Postgres database as the internal storage.
-See [Settings](settings#databaseconnectionspec) for configuring its location, and `cocoindex setup` CLI command (see [CocoIndex CLI](cli)) creates tables for the internal storage.
+See [Settings](settings#databaseconnectionspec) for configuring its location. The internal storage is managed by CocoIndex, see [Setup / drop flow](/docs/core/flow_methods#setup--drop-flow) for more details.

{cocoindex-0.1.56 → cocoindex-0.1.58}/docs/docs/core/flow_def.mdx

@@ -256,7 +256,7 @@ Export must happen at the top level of a flow, i.e. not within any child scopes
 *   `target_spec`: the target spec as the export target.
 *   `setup_by_user` (optional):
      whether the export target is setup by user.
-     By default, CocoIndex is managing the target setup (surfaced by the `cocoindex setup` CLI subcommand), e.g. create related tables/collections/etc. with compatible schema, and update them upon change.
+     By default, CocoIndex is managing the target setup (see [Setup / drop flow](/docs/core/flow_methods#setup--drop-flow)), e.g. create related tables/collections/etc. with compatible schema, and update them upon change.
      If `True`, the export target will be managed by users, and users are responsible for creating the target and updating it upon change.
 *   Fields to configure [storage indexes](#storage-indexes). `primary_key_fields` is required, and all others are optional.
 
@@ -278,7 +278,7 @@ def demo_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataSco
 </TabItem>
 </Tabs>
 
-The target is managed by CocoIndex, i.e. it'll be created by [CocoIndex CLI](/docs/core/cli) when you run `cocoindex setup`, and the data will be automatically updated (including stale data removal) when updating the index.
+The target is managed by CocoIndex, i.e. it'll be created or dropped when [setup / drop flow](/docs/core/flow_methods#setup--drop-flow), and the data will be automatically updated (including stale data removal) when updating the index.
 The `name` for the same target should remain stable across different runs.
 If it changes, CocoIndex will treat it as an old target removed and a new one created, and perform setup changes and reindexing accordingly.
 
@@ -370,11 +370,11 @@ flow_builder.declare(
 
 CocoIndex manages an auth registry. It's an in-memory key-value store, mainly to store authentication information for a backend.
 
-Operation spec is the default way to configure a backend. But it has the following limitations:
+Operation spec is the default way to configure a persistent backend. But it has the following limitations:
 
 *   The spec isn't supposed to contain secret information, and it's frequently shown in various places, e.g. `cocoindex show`.
 *   Once an operation is removed after flow definition code change, the spec is also gone.
-    But we still need to be able to drop the backend (e.g. a table) by `cocoindex setup` or `cocoindex drop`.
+    But we still need to be able to drop the backend (e.g. a table) when [setup / drop flow](/docs/core/flow_methods#setup--drop-flow).
 
 Auth registry is introduced to solve the problems above. It works as follows:
 
@@ -423,5 +423,5 @@ Note that CocoIndex backends use the key of an auth entry to identify the backen
 *   Keep the key stable.
     If the key doesn't change, it's considered to be the same backend (even if the underlying way to connect/authenticate change).
 
-*   If a key is no longer referenced in any operation spec, keep it until the next `cocoindex setup` or `cocoindex drop`,
-    so that when cocoindex will be able to perform cleanups.
+*   If a key is no longer referenced in any operation spec, keep it until the next flow setup / drop action,
+    so that cocoindex will be able to clean up the backends.

{cocoindex-0.1.56 → cocoindex-0.1.58}/docs/docs/core/flow_methods.mdx

@@ -34,6 +34,71 @@ It creates a `demo_flow` object in `cocoindex.Flow` type.
 </TabItem>
 </Tabs>
 
+## Setup / drop flow
+
+For a flow, its persistent backends need to be ready before it can run, including:
+
+*   [Internal storage](/docs/core/basics#internal-storage) for CocoIndex.
+*   Backend entities for targets exported by the flow, e.g. a table (in relational databases), a collection (in some vector databases), etc.
+
+The desired state of the backends for a flow is derived based on the flow definition itself.
+CocoIndex supports two types of actions to manage the persistent backends automatically:
+
+*   *Setup* a flow, which will change the backends owned by the flow to a state to the desired state, e.g. create new tables for new flow, drop an existing table if the corresponding target is gone, add new column to a target table if a new field is collected, etc. It's no-op if the backend states are already in the desired state.
+
+*   *Drop* a flow, which will drop all backends owned by the flow. It's no-op if there's no existing backends owned by the flow (e.g. never setup or already dropped).
+
+### CLI
+
+`cocoindex setup` subcommand will setup all flows.
+`cocoindex update` and `cocoindex server` also provide a `--setup` option to setup the flow if needed before performing the main action of updating or starting the server.
+
+`cocoindex drop` subcommand will drop all flows.
+
+### Library API
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+`Flow` provides the following APIs to setup / drop individual flows:
+
+*   `setup(report_to_stdout: bool = False)`: Setup the flow.
+*   `drop(report_to_stdout: bool = False)`: Drop the flow.
+
+For example:
+
+```python
+demo_flow.setup(report_to_stdout=True)
+demo_flow.drop(report_to_stdout=True)
+```
+
+We also provide the following asynchronous versions of the APIs:
+
+*   `setup_async(report_to_stdout: bool = False)`: Setup the flow asynchronously.
+*   `drop_async(report_to_stdout: bool = False)`: Drop the flow asynchronously.
+
+For example:
+
+```python
+await demo_flow.setup_async(report_to_stdout=True)
+await demo_flow.drop_async(report_to_stdout=True)
+```
+
+
+Besides, CocoIndex also provides APIs to setup / drop all flows at once:
+
+*   `setup_all_flows(report_to_stdout: bool = False)`: Setup all flows.
+*   `drop_all_flows(report_to_stdout: bool = False)`: Drop all flows.
+
+For example:
+
+```python
+cocoindex.setup_all_flows(report_to_stdout=True)
+cocoindex.drop_all_flows(report_to_stdout=True)
+```
+
+</TabItem>
+</Tabs>
 
 ## Build / update target data
 
@@ -71,12 +136,18 @@ Once it's done, the target data is fresh up to the moment when the function is c
 cocoindex update main.py
 ```
 
+With a `--setup` option, it will also setup the flow first if needed.
+
+```sh
+cocoindex update --setup main.py
+```
+
 #### Library API
 
 <Tabs>
 <TabItem value="python" label="Python">
 
-The `update()` async method creates/updates data in the target.
+The `Flow.update()` method creates/updates data in the target.
 
 Once the function returns, the target data is fresh up to the moment when the function is called.
 
@@ -85,6 +156,13 @@ stats = demo_flow.update()
 print(stats)
 ```
 
+`update_async()` is the asynchronous version of `update()`.
+
+```python
+stats = await demo_flow.update_async()
+print(stats)
+```
+
 </TabItem>
 </Tabs>
 
@@ -111,6 +189,12 @@ cocoindex update main.py -L
 If there's at least one data source with change capture mechanism enabled, it will keep running until the aborted (e.g. by `Ctrl-C`).
 Otherwise, it falls back to the same behavior as one time update, and will finish after a one-time update is done.
 
+With a `--setup` option, it will also setup the flow first if needed.
+
+```sh
+cocoindex update main.py -L --setup
+```
+
 #### Library API
 
 <Tabs>

{cocoindex-0.1.56 → cocoindex-0.1.58}/docs/docs/getting_started/quickstart.md

@@ -21,7 +21,7 @@ We'll need to install a bunch of dependencies for this project.
 1.  Install CocoIndex:
 
     ```bash
-    pip install -U cocoindex
+    pip install -U 'cocoindex[embeddings]'
     ```
 
 2.  You can skip this step if you already have a Postgres database with pgvector extension installed.
@@ -121,27 +121,16 @@ Specify the database URL by environment variable:
 export COCOINDEX_DATABASE_URL="postgresql://cocoindex:cocoindex@localhost:5432/cocoindex"
 ```
 
-### Step 3.1: Setup the index pipeline
-
-We need to setup the index:
-
-```bash
-cocoindex setup quickstart.py
-```
-
-Enter `yes` and it will automatically create a few tables in the database.
-
-Now we have tables needed by this CocoIndex flow.
-
-### Step 3.2: Build the index
-
 Now we're ready to build the index:
 
 ```bash
-cocoindex update quickstart.py
+cocoindex update --setup quickstart.py
 ```
 
-It will run for a few seconds and output the following statistics:
+If you run it the first time for this flow, CocoIndex will automatically create its persistent backends (tables in the database).
+CocoIndex will ask you to confirm the action, enter `yes` to proceed.
+
+CocoIndex will run for a few seconds and populate the target table with data as declared by the flow. It will output the following statistics:
 
 ```
 documents: 3 added, 0 removed, 0 updated

{cocoindex-0.1.56 → cocoindex-0.1.58}/docs/docs/ops/functions.md

@@ -77,6 +77,15 @@ Return: [*KTable*](/docs/core/data_types#ktable), each row represents a chunk, w
 
 `SentenceTransformerEmbed` embeds a text into a vector space using the [SentenceTransformer](https://huggingface.co/sentence-transformers) library.
 
+:::note Optional Dependency Required
+
+This function requires the 'sentence-transformers' library, which is an optional dependency. Install CocoIndex with:
+
+```bash
+pip install 'cocoindex[embeddings]'
+```
+:::
+
 The spec takes the following fields:
 
 *   `model` (`str`): The name of the SentenceTransformer model to use.

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/amazon_s3_embedding/README.md

@@ -37,12 +37,6 @@ Install dependencies:
 pip install -e .
 ```
 
-Setup:
-
-```sh
-cocoindex setup main.py
-```
-
 Run:
 
 ```sh

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/amazon_s3_embedding/main.py

@@ -2,6 +2,7 @@ from dotenv import load_dotenv
 from psycopg_pool import ConnectionPool
 import cocoindex
 import os
+from typing import Any
 
 
 @cocoindex.transform_flow()
@@ -22,7 +23,7 @@ def text_to_embedding(
 @cocoindex.flow_def(name="AmazonS3TextEmbedding")
 def amazon_s3_text_embedding_flow(
     flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
-):
+) -> None:
     """
     Define an example flow that embeds text from Amazon S3 into a vector database.
     """
@@ -72,7 +73,7 @@ def amazon_s3_text_embedding_flow(
     )
 
 
-def search(pool: ConnectionPool, query: str, top_k: int = 5):
+def search(pool: ConnectionPool, query: str, top_k: int = 5) -> list[dict[str, Any]]:
     # Get the table name, for the export target in the amazon_s3_text_embedding_flow above.
     table_name = cocoindex.utils.get_target_default_name(
         amazon_s3_text_embedding_flow, "doc_embeddings"
@@ -95,9 +96,11 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5):
             ]
 
 
-def _main():
+def _main() -> None:
     # Initialize the database connection pool.
     pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
+
+    amazon_s3_text_embedding_flow.setup()
     with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow):
         # Run queries in a loop to demonstrate the query capabilities.
         while True:

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/amazon_s3_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "amazon-s3-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Amazon S3 files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.52", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.57", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/code_embedding/main.py

@@ -68,6 +68,7 @@ def code_embedding_flow(
                 code=chunk["text"],
                 embedding=chunk["embedding"],
                 start=chunk["start"],
+                end=chunk["end"],
             )
 
     code_embeddings.export(
@@ -96,13 +97,19 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5) -> list[dict[str, A
         with conn.cursor() as cur:
             cur.execute(
                 f"""
-                SELECT filename, code, embedding <=> %s AS distance
+                SELECT filename, code, embedding <=> %s AS distance, start, "end"
                 FROM {table_name} ORDER BY distance LIMIT %s
             """,
                 (query_vector, top_k),
             )
             return [
-                {"filename": row[0], "code": row[1], "score": 1.0 - row[2]}
+                {
+                    "filename": row[0],
+                    "code": row[1],
+                    "score": 1.0 - row[2],
+                    "start": row[3],
+                    "end": row[4],
+                }
                 for row in cur.fetchall()
             ]
 
@@ -123,7 +130,9 @@ def _main() -> None:
         results = search(pool, query)
         print("\nSearch results:")
         for result in results:
-            print(f"[{result['score']:.3f}] {result['filename']}")
+            print(
+                f"[{result['score']:.3f}] {result['filename']} (L{result['start']['line']}-L{result['end']['line']})"
+            )
             print(f"    {result['code']}")
             print("---")
         print()

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/code_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.52", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.56", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/fastapi_server_docker/requirements.txt

@@ -1,4 +1,4 @@
-cocoindex>=0.1.52
+cocoindex[embeddings]>=0.1.52
 python-dotenv>=1.0.1
 fastapi==0.115.12
 fastapi-cli==0.0.7

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,7 +3,7 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.52", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex[embeddings]>=0.1.52", "python-dotenv>=1.0.1"]
 
 [tool.setuptools]
 packages = []

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/image_search/README.md

@@ -29,7 +29,6 @@ We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/c
 
 - Run Backend
   ```
-  cocoindex setup main.py
   uvicorn main:app --reload --host 0.0.0.0 --port 8000
   ```
 

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/image_search/main.py

@@ -84,9 +84,11 @@ def image_object_embedding_flow(
 
 
 @asynccontextmanager
-async def lifespan(app: FastAPI):
+async def lifespan(app: FastAPI) -> None:
     load_dotenv()
     cocoindex.init()
+    image_object_embedding_flow.setup(report_to_stdout=True)
+
     app.state.qdrant_client = QdrantClient(url=QDRANT_URL, prefer_grpc=True)
 
     # Start updater

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/image_search/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on images."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.52",
+    "cocoindex>=0.1.57",
     "python-dotenv>=1.0.1",
     "fastapi>=0.100.0",
     "torch>=2.0.0",

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.52",
+    "cocoindex[embeddings]>=0.1.52",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/text_embedding/main.py

@@ -6,6 +6,7 @@ import cocoindex
 import os
 from numpy.typing import NDArray
 import numpy as np
+from datetime import timedelta
 
 
 @cocoindex.transform_flow()
@@ -38,7 +39,8 @@ def text_embedding_flow(
     Define an example flow that embeds text into a vector database.
     """
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="markdown_files")
+        cocoindex.sources.LocalFile(path="markdown_files"),
+        refresh_interval=timedelta(seconds=5),
     )
 
     doc_embeddings = data_scope.add_collector()

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/text_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.52",
+    "cocoindex[embeddings]>=0.1.52",
     "python-dotenv>=1.0.1",
     "pgvector>=0.4.1",
     "psycopg[binary,pool]",

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/text_embedding_qdrant/README.md

@@ -37,6 +37,7 @@ We use Qdrant client to query the index, and reuse the embedding operation in th
    pip install -e .
    ```
 
+
 - Setup:
 
    ```bash

{cocoindex-0.1.56 → cocoindex-0.1.58}/examples/text_embedding_qdrant/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.52",
+    "cocoindex[embeddings]>=0.1.52",
     "python-dotenv>=1.0.1",
     "qdrant-client>=1.6.0",
 ]

{cocoindex-0.1.56 → cocoindex-0.1.58}/pyproject.toml

@@ -10,11 +10,11 @@ authors = [{ name = "CocoIndex", email = "cocoindex.io@gmail.com" }]
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "sentence-transformers>=3.3.1",
     "click>=8.1.8",
     "rich>=14.0.0",
     "python-dotenv>=1.1.0",
     "watchfiles>=1.1.0",
+    "numpy>=1.23.2",
 ]
 license = "Apache-2.0"
 urls = { Homepage = "https://cocoindex.io/" }
@@ -31,9 +31,11 @@ features = ["pyo3/extension-module"]
 [project.optional-dependencies]
 test = ["pytest"]
 dev = ["ruff", "pre-commit"]
+embeddings = ["sentence-transformers>=3.3.1"]
 
 [tool.mypy]
 python_version = "3.11"
 strict = true
 files = "python/cocoindex"
 exclude = "(\\.venv|site-packages)"
+disable_error_code = ["unused-ignore"]

{cocoindex-0.1.56 → cocoindex-0.1.58}/python/cocoindex/__init__.py

@@ -10,7 +10,8 @@ from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
-from .flow import update_all_flows_async, FlowLiveUpdater, FlowLiveUpdaterOptions
+from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions
+from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
 from .lib import init, start_server, stop, main_fn
 from .llm import LlmSpec, LlmApiType
 from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
@@ -40,9 +41,11 @@ __all__ = [
     "flow_def",
     "EvaluateAndDumpOptions",
     "GeneratedField",
-    "update_all_flows_async",
     "FlowLiveUpdater",
     "FlowLiveUpdaterOptions",
+    "update_all_flows_async",
+    "setup_all_flows",
+    "drop_all_flows",
     # Lib
     "init",
     "start_server",