cocoindex 0.1.44__tar.gz → 0.1.46__tar.gz

cocoindex-0.1.46/.github/workflows/CI.yml

@@ -0,0 +1,46 @@
+# This file is autogenerated by maturin v1.8.1
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - src/**
+      - python/**
+      - "*.toml"
+      - ".github/workflows/*.yml"
+  push:
+    branches: [main]
+    paths:
+      - src/**
+      - python/**
+      - "*.toml"
+      - ".github/workflows/*.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  format-check:
+    name: Check Python formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install Ruff
+        run: |
+          pip install ruff
+      - name: Check Python formatting
+        run: |
+          ruff format --check .
+
+  test:
+    name: Run test
+    uses: ./.github/workflows/_test.yml

{cocoindex-0.1.44 → cocoindex-0.1.46}/.github/workflows/_test.yml

@@ -36,10 +36,22 @@ jobs:
             key: ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
             restore-keys: |
               ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-
-        - name: Python build & test
+        - name: Setup venv
           run: |
             python -m venv .venv
+        - name: Install Python toolchains
+          run: |
+            source .venv/bin/activate
+            pip install maturin pytest mypy
+        - name: Python build
+          run: |
             source .venv/bin/activate
-            pip install maturin pytest
             maturin develop
+        - name: Python type check (mypy)
+          run: |
+            source .venv/bin/activate
+            mypy python
+        - name: Python tests
+          run: |
+            source .venv/bin/activate
             pytest python/cocoindex/tests

{cocoindex-0.1.44 → cocoindex-0.1.46}/.github/workflows/docs.yml

@@ -22,22 +22,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          sparse-checkout: docs
-          path: src-staging
-      - name: Move docs
-        run: |
-          shopt -s dotglob
-          mv src-staging/docs/* .
       - uses: actions/setup-node@v4
         with:
           node-version: 18
           cache: yarn
+          cache-dependency-path: docs/yarn.lock
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn --cwd docs install --frozen-lockfile
       - name: Test build website
-        run: yarn build
+        run: yarn --cwd docs build
 
   deploy-precheck:
     if: ${{ github.event_name != 'pull_request' }}
@@ -59,18 +52,11 @@ jobs:
     environment: docs-release
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          sparse-checkout: docs
-          path: src-staging
-      - name: Move docs
-        run: |
-          shopt -s dotglob
-          mv src-staging/docs/* .
       - uses: actions/setup-node@v4
         with:
           node-version: 18
           cache: yarn
+          cache-dependency-path: docs/yarn.lock
       - uses: webfactory/ssh-agent@v0.5.0
         with:
           ssh-private-key: ${{ secrets.GH_PAGES_DEPLOY }}
@@ -84,5 +70,5 @@ jobs:
           export COCOINDEX_DOCS_ALGOLIA_API_KEY=${{ vars.COCOINDEX_DOCS_ALGOLIA_API_KEY }}
           git config --global user.email "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_EMAIL }}"
           git config --global user.name "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_NAME }}"
-          yarn install --frozen-lockfile
-          yarn deploy
+          yarn --cwd docs install --frozen-lockfile
+          yarn --cwd docs deploy

{cocoindex-0.1.44 → cocoindex-0.1.46}/.vscode/settings.json

@@ -3,5 +3,7 @@
         "cocoindex",
         "reindexing",
         "timedelta"
-    ]
+    ],
+    "editor.formatOnSave": true,
+    "python.formatting.provider": "ruff"
 }

{cocoindex-0.1.44 → cocoindex-0.1.46}/Cargo.lock

@@ -993,7 +993,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.44"
+version = "0.1.46"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.44 → cocoindex-0.1.46}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.44"
+version = "0.1.46"
 edition = "2024"
 
 [profile.release]

{cocoindex-0.1.44 → cocoindex-0.1.46}/PKG-INFO

@@ -1,12 +1,14 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.44
+Version: 0.1.46
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: ruff ; extra == 'dev'
 Provides-Extra: test
+Provides-Extra: dev
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Author-email: CocoIndex <cocoindex.io@gmail.com>

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docs/core/flow_def.mdx

@@ -348,7 +348,7 @@ It will use `Staging__doc_embeddings` as the collection name if the current app
 Most time a target storage is created by calling `export()` method on a collector, and this `export()` call comes with configurations needed for the target storage, e.g. options for storage indexes.
 Occasionally, you may need to specify some configurations for target storage out of the context of any specific data collector.
 
-For example, for graph database targets like `Neo4j`, you may have a data collector to export data to Neo4j relationships, which will create nodes referenced by various relationships in turn.
+For example, for graph database targets like `Neo4j` and `Kuzu`, you may have a data collector to export data to relationships, which will create nodes referenced by various relationships in turn.
 These nodes don't directly come from any specific data collector (consider relationships from different data collectors may share the same nodes).
 To specify configurations for these nodes, you can *declare* spec for related node labels.
 

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docs/core/settings.mdx

@@ -6,7 +6,7 @@ description: Provide settings for CocoIndex, e.g. database connection, app names
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# CocoIndex Settings
+# CocoIndex Setting
 
 Certain settings need to be provided for CocoIndex to work, e.g. database connections, app namespace, etc.
 

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docs/ops/functions.md

@@ -26,6 +26,17 @@ Input data:
 
 *   `text` (type: `str`, required): The text to split.
 *   `chunk_size` (type: `int`, required): The maximum size of each chunk, in bytes.
+*   `min_chunk_size` (type: `int`, optional): The minimum size of each chunk, in bytes. If not provided, default to `chunk_size / 2`.
+
+    :::note
+
+    `SplitRecursively` will do its best to make the output chunks sized between `min_chunk_size` and `chunk_size`.
+    However, it's possible that some chunks are smaller than `min_chunk_size` or larger than `chunk_size` in rare cases, e.g. too short input text, or non-splittable large text.
+
+    Please avoid setting `min_chunk_size` to a value too close to `chunk_size`, to leave more rooms for the function to plan the optimal chunking.
+
+    :::
+
 *   `chunk_overlap` (type: `int`, optional): The maximum overlap size between adjacent chunks, in bytes.
 *   `language` (type: `str`, optional): The language of the document.
     Can be a langauge name (e.g. `Python`, `Javascript`, `Markdown`) or a file extension (e.g. `.py`, `.js`, `.md`).

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docs/ops/sources.md

@@ -95,6 +95,7 @@ This is how to setup:
                 "Service": "s3.amazonaws.com"
               },
               "Resource": "${SQS_QUEUE_ARN}",
+              "Action": "SQS:SendMessage",
               "Condition": {
                 "ArnLike": {
                   "aws:SourceArn": "${S3_BUCKET_ARN}"

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docs/ops/storages.md

@@ -391,6 +391,12 @@ graph TD
   classDef node font-size:8pt,text-align:left,stroke-width:2;
 ```
 
+#### Examples
+
+You can find end-to-end examples fitting into any of supported property graphs in the following directories:
+*   [examples/docs_to_knowledge_graph](https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph)
+*   [examples/product_recommendation](https://github.com/cocoindex-io/cocoindex/tree/main/examples/product_recommendation)
+
 ### Neo4j
 
 If you don't have a Neo4j database, you can start a Neo4j database using our docker compose config:
@@ -407,14 +413,14 @@ Please read and agree the license before starting the instance.
 
 :::
 
-The `Neo4j` storage exports each row as a relationship to Neo4j Knowledge Graph. The spec takes the following fields:
+The `Neo4j` target spec takes the following fields:
 
 *   `connection` (type: [auth reference](../core/flow_def#auth-registry) to `Neo4jConnectionSpec`): The connection to the Neo4j database. `Neo4jConnectionSpec` has the following fields:
     *   `url` (type: `str`): The URI of the Neo4j database to use as the internal storage, e.g. `bolt://localhost:7687`.
     *   `user` (type: `str`): Username for the Neo4j database.
     *   `password` (type: `str`): Password for the Neo4j database.
     *   `db` (type: `str`, optional): The name of the Neo4j database to use as the internal storage, e.g. `neo4j`.
-*   `mapping` (type: `NodeMapping | RelationshipMapping`): The mapping from collected row to nodes or relationships of the graph. 2 variations are supported:
+*   `mapping` (type: `Nodes | Relationships`): The mapping from collected row to nodes or relationships of the graph. For either [nodes to export](#nodes-to-export) or [relationships to export](#relationships-to-export).
 
 Neo4j also provides a declaration spec `Neo4jDeclaration`, to configure indexing options for nodes only referenced by relationships. It has the following fields:
 
@@ -424,4 +430,26 @@ Neo4j also provides a declaration spec `Neo4jDeclaration`, to configure indexing
     *   `primary_key_fields` (required)
     *   `vector_indexes` (optional)
 
-You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph).
+### Kuzu
+
+CocoIndex supports talking to Kuzu through its [API server](https://github.com/kuzudb/api-server).
+You can bring up a Kuzu API server locally by running:
+
+```bash
+KUZU_DB_DIR=$HOME/.kuzudb
+KUZU_PORT=8123
+docker run -d --name kuzu -p ${KUZU_PORT}:8000 -v ${KUZU_DB_DIR}:/database kuzudb/api-server:latest
+```
+
+The `Kuzu` target spec takes the following fields:
+
+*   `connection` (type: [auth reference](../core/flow_def#auth-registry) to `KuzuConnectionSpec`): The connection to the Kuzu database. `KuzuConnectionSpec` has the following fields:
+    *   `api_server_url` (type: `str`): The URL of the Kuzu API server, e.g. `http://localhost:8123`.
+*   `mapping` (type: `Nodes | Relationships`): The mapping from collected row to nodes or relationships of the graph. For either [nodes to export](#nodes-to-export) or [relationships to export](#relationships-to-export).
+
+Kuzu also provides a declaration spec `KuzuDeclaration`, to configure indexing options for nodes only referenced by relationships. It has the following fields:
+
+*   `connection` (type: auth reference to `KuzuConnectionSpec`)
+*   Fields for [nodes to declare](#declare-extra-node-labels), including
+    *   `nodes_label` (required)
+    *   `primary_key_fields` (required)

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/docusaurus.config.ts

@@ -86,6 +86,11 @@ const config: Config = {
     // Replace with your project's social card
     image: 'img/social-card.jpg',
     metadata: [{ name: 'description', content: 'Official documentation for CocoIndex - Learn how to use CocoIndex to build robust data indexing pipelines for AI applications. Comprehensive guides, API references, and best practices for implementing efficient data processing workflows.' }],
+    colorMode: {
+      defaultMode: 'light',
+      disableSwitch: false,
+      respectPrefersColorScheme: true,
+    },
     navbar: {
       title: 'CocoIndex',
       logo: {

{cocoindex-0.1.44 → cocoindex-0.1.46}/docs/package.json

@@ -15,10 +15,10 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "3.7.0",
-    "@docusaurus/plugin-client-redirects": "^3.7.0",
-    "@docusaurus/preset-classic": "3.7.0",
-    "@docusaurus/theme-mermaid": "^3.7.0",
+    "@docusaurus/core": "^3.8.0",
+    "@docusaurus/plugin-client-redirects": "^3.8.0",
+    "@docusaurus/preset-classic": "^3.8.0",
+    "@docusaurus/theme-mermaid": "^3.8.0",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.0.0",
     "mixpanel-browser": "^2.59.0",