cocoindex 0.2.1__tar.gz → 0.2.3__tar.gz

cocoindex-0.2.3/.github/workflows/_test.yml

@@ -0,0 +1,65 @@
+name: Run Tests
+
+on:
+    workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+    build-test:
+        strategy:
+            matrix:
+                python-version: [3.11]
+                platform:
+                  - { runner: ubuntu-latest, python_exec: ".venv/bin/python" }
+                  - { runner: ubuntu-24.04-arm, python_exec: ".venv/bin/python" }
+                  - { runner: macos-latest, python_exec: ".venv/bin/python" }
+                  - { runner: macos-13, python_exec: ".venv/bin/python" }
+                  - { runner: windows-latest, python_exec: ".venv\\Scripts\\python" }
+        runs-on: ${{ matrix.platform.runner }}
+        steps:
+        - uses: actions/checkout@v4
+
+        - uses: actions/setup-python@v5
+          id: setup_python
+          with:
+            python-version: ${{ matrix.python-version }}
+            cache: 'pip'
+
+        - run: rustup toolchain install stable --profile minimal
+        - name: Rust Cache
+          uses: Swatinem/rust-cache@v2
+          with:
+            key: rust-${{ matrix.platform.runner }}-${{ matrix.python-version }}
+        - name: Rust tests
+          run: cargo test --verbose
+
+        - uses: actions/cache@v4
+          with:
+            path: .venv
+            key: pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
+            restore-keys: |
+              pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-
+
+        - name: Setup venv
+          run: |
+            python -m venv .venv
+        - name: Install Python toolchains
+          run: |
+            ${{ matrix.platform.python_exec }} -m pip install maturin mypy pytest pytest-asyncio
+        - name: Python build
+          run: |
+            ${{ matrix.platform.python_exec }} -m maturin develop -E all
+        - name: Python type check (mypy)
+          run: |
+            ${{ matrix.platform.python_exec }} -m mypy python
+        - name: Python tests
+          if: ${{ !startsWith(matrix.platform.runner, 'windows') }}
+          run: |
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests
+        - name: Python tests (Windows cmd)
+          if: ${{ startsWith(matrix.platform.runner, 'windows') }}
+          shell: cmd  # Use `cmd` to run test for Windows, as PowerShell doesn't detect exit code by `os._exit(0)` correctly.
+          run: |
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests

{cocoindex-0.2.1 → cocoindex-0.2.3}/.github/workflows/release.yml

@@ -1,8 +1,7 @@
-# This file is autogenerated by maturin v1.8.1
-# To update, run
-#
-#    maturin generate-ci github
+# This workflow can be triggered on tags push (automatic release) or manually on any branch.
 #
+# - When triggered on tags push, it will build and publishes a new version including docs.
+# - When triggered manually, it's a dry-run: only build, without publishing anything.
 name: release
 
 on:
@@ -31,11 +30,11 @@ jobs:
     strategy:
       matrix:
         platform:
-          - { os: linux, runner: ubuntu-24.04, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
-          - { os: linux, runner: ubuntu-24.04, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
-          - { os: windows, runner: windows-latest, target: x64 }
+          - { os: linux, runner: ubuntu-latest, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
+          - { os: linux, runner: ubuntu-24.04-arm, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
+          - { os: macos, runner: macos-latest, target: aarch64 }
           - { os: macos, runner: macos-13, target: x86_64 }
-          - { os: macos, runner: macos-14, target: aarch64 }
+          - { os: windows, runner: windows-latest, target: x64 }
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
@@ -43,12 +42,12 @@ jobs:
           name: Cargo.toml
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.x
+          python-version: 3.13
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter
+          args: --release --out dist
           sccache: 'true'
           manylinux: auto
           container: ${{ matrix.platform.container }}
@@ -58,6 +57,24 @@ jobs:
           name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}
           path: dist
 
+  test-abi3:
+    runs-on: ubuntu-24.04
+    needs: build
+    strategy:
+      matrix:
+        py: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheels-linux-x86_64
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
+      - run: python -V
+      - run: pip install --find-links=./ cocoindex
+      - run: python -c "import cocoindex, sys; print('import ok on', sys.version)"
+
+
   sdist:
     runs-on: ubuntu-latest
     needs: [create-versioned-toml]
@@ -80,7 +97,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: [create-versioned-toml, build, sdist]
+    needs: [create-versioned-toml, build, test-abi3, sdist]
     permissions:
       # Use to sign the release artifacts
       id-token: write
@@ -111,5 +128,6 @@ jobs:
   release-docs:
     name: Release Docs
     needs: [release]
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
     uses: ./.github/workflows/_doc_release.yml
     secrets: inherit

{cocoindex-0.2.1 → cocoindex-0.2.3}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.1"
+version = "0.2.3"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.2.1 → cocoindex-0.2.3}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.1"
+version = "0.2.3"
 edition = "2024"
 rust-version = "1.88"
 readme = "README.md"
@@ -16,7 +16,12 @@ name = "cocoindex_engine"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.25.1", features = ["chrono", "auto-initialize", "uuid"] }
+pyo3 = { version = "0.25.1", features = [
+    "abi3-py311",
+    "auto-initialize",
+    "chrono",
+    "uuid",
+] }
 pythonize = "0.25.0"
 pyo3-async-runtimes = { version = "0.25.0", features = ["tokio-runtime"] }
 

{cocoindex-0.2.1 → cocoindex-0.2.3}/PKG-INFO

@@ -1,6 +1,21 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.1
+Version: 0.2.3
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Typing :: Typed
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -21,6 +36,7 @@ Provides-Extra: colpali
 Provides-Extra: all
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
+Keywords: indexing,real-time,incremental,pipeline,search,ai,etl,rag,dataflow,context-engineering
 Author-email: CocoIndex <cocoindex.io@gmail.com>
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -84,7 +100,7 @@ CocoIndex makes it effortless to transform data with AI, and keep source data an
 </br>
 
 <p align="center">
-<img width="4187" height="1883" alt="CocoIndex Features" src="https://github.com/user-attachments/assets/6147673f-0daf-4313-a0c7-7c4205ba2e31" />
+<img alt="CocoIndex Features" src="https://cocoindex.io/images/venn2.svg" />
 </p>
 
 </br>

{cocoindex-0.2.1 → cocoindex-0.2.3}/README.md

@@ -55,7 +55,7 @@ CocoIndex makes it effortless to transform data with AI, and keep source data an
 </br>
 
 <p align="center">
-<img width="4187" height="1883" alt="CocoIndex Features" src="https://github.com/user-attachments/assets/6147673f-0daf-4313-a0c7-7c4205ba2e31" />
+<img alt="CocoIndex Features" src="https://cocoindex.io/images/venn2.svg" />
 </p>
 
 </br>

{cocoindex-0.2.1 → cocoindex-0.2.3}/docs/docs/examples/examples/academic_papers_index.md

@@ -12,7 +12,9 @@ tags: [vector-index, metadata]
 
 import { GitHubButton, YouTubeButton, DocumentationButton } from '../../../src/components/GitHubButton';
 
-<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/paper_metadata"/>
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/paper_metadata" margin="0 0 24px 0" />
+
+![Academic Papers Index](/img/examples/academic_papers_index/cover.png)
 
 ## What we will achieve
 
@@ -45,7 +47,7 @@ to answer questions like "Give me all the papers by Jeff Dean."
   CocoIndex uses PostgreSQL internally for incremental processing.
 - [Configure your OpenAI API key](https://cocoindex.io/docs/ai/llm#openai). Alternatively, we have native support for Gemini, Ollama, LiteLLM. You can choose your favorite LLM provider and work completely on-premises.
 
-  <DocumentationButton href="https://cocoindex.io/docs/ai/llm" text="LLM" margin="0 0 16px 0" />
+  <DocumentationButton url="https://cocoindex.io/docs/ai/llm" text="LLM" margin="0 0 16px 0" />
 
 ## Import the Papers
 
@@ -61,7 +63,7 @@ def paper_metadata_flow(
 ```
 
 `flow_builder.add_source` will create a table with sub fields (`filename`, `content`).
-<DocumentationButton href="https://cocoindex.io/docs/ops/sources" text="Sources" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Sources" margin="0 0 16px 0" />
 
 ## Extract and collect metadata
 
@@ -321,7 +323,7 @@ metadata_embeddings.export(
 
 In this example we use PGVector as embedding store. With CocoIndex, you can do one line switch on other supported Vector databases.
 
-<DocumentationButton href="https://cocoindex.io/docs/ops/targets#entry-oriented-targets" text="Entry Oriented Targets" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/targets#entry-oriented-targets" text="Entry Oriented Targets" margin="0 0 16px 0" />
 
 ## Query the index
 

{cocoindex-0.2.1 → cocoindex-0.2.3}/docs/docs/examples/examples/codebase_index.md

@@ -12,23 +12,23 @@ tags: [vector-index, codebase]
 
 import { GitHubButton, YouTubeButton, DocumentationButton } from '../../../src/components/GitHubButton';
 
-<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/code_embedding"/>
-<YouTubeButton url="https://youtu.be/G3WstvhHO24?si=ndYfM0XRs03_hVPR" />
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/code_embedding" margin="0 0 24px 0" />
+<YouTubeButton url="https://youtu.be/G3WstvhHO24?si=ndYfM0XRs03_hVPR" margin="0 0 24px 0" />
+
+![Codebase Index](/img/examples/codebase_index/cover.png)
 
 ## Overview
 In this tutorial, we will build codebase index. [CocoIndex](https://github.com/cocoindex-io/cocoindex) provides built-in support for codebase chunking, with native Tree-sitter support. It works with large codebases, and can be updated in near real-time with incremental processing - only reprocess what's changed. 
 
 ## Use Cases
-A wide range of applications can be built with an effective codebase index that is always up-to-date. Some examples include:
-
-![Use case illustration](/img/examples/codebase_index/usecase.png)
+A wide range of applications can be built with an effective codebase index that is always up-to-date.
 
 - Semantic code context for AI coding agents like Claude, Codex, Gemini CLI.
 - MCP for code editors such as Cursor, Windsurf, and VSCode.
 - Context-aware code search applications—semantic code search, natural language code retrieval.
 - Context for code review agents—AI code review, automated code analysis, code quality checks, pull request summarization.
 - Automated code refactoring, large-scale code migration.
-- Enhance SRE workflows: enable rapid root cause analysis, incident response, and change impact assessment by indexing infrastructure-as-code, deployment scripts, and config files for semantic search and lineage tracking.
+- SRE workflows: enable rapid root cause analysis, incident response, and change impact assessment by indexing infrastructure-as-code, deployment scripts, and config files for semantic search and lineage tracking.
 - Automatically generate design documentation from code—keep design docs up-to-date.
 
 ## Flow Overview
@@ -67,7 +67,7 @@ def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
 - Exclude files and directories starting `.`,  `target` in the root and `node_modules` under any directory.
 
 `flow_builder.add_source` will create a table with sub fields (`filename`, `content`). 
-<DocumentationButton href="https://cocoindex.io/docs/ops/sources" text="Sources" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Sources" />
 
 
 ## Process each file and collect the information
@@ -84,7 +84,7 @@ def extract_extension(filename: str) -> str:
     return os.path.splitext(filename)[1]
 ```
 
-<DocumentationButton href="https://cocoindex.io/docs/custom_ops/custom_functions" text="Custom Function" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_functions" text="Custom Function" margin="0 0 16px 0" />
 
 ### Split the file into chunks
 We use the `SplitRecursively` function to split the file into chunks.  `SplitRecursively` is CocoIndex building block, with native integration with Tree-sitter. You need to pass in the language to the `language` parameter if you are processing code.
@@ -97,7 +97,7 @@ with data_scope["files"].row() as file:
           cocoindex.functions.SplitRecursively(),
           language=file["extension"], chunk_size=1000, chunk_overlap=300) 
 ```
-<DocumentationButton href="https://cocoindex.io/docs/ops/functions#splitrecursively" text="SplitRecursively" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/functions#splitrecursively" text="SplitRecursively" margin="0 0 16px 0" />
 
 ![SplitRecursively](/img/examples/codebase_index/chunk.png)
 
@@ -112,13 +112,13 @@ def code_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[lis
             model="sentence-transformers/all-MiniLM-L6-v2"))
 ```
 
-<DocumentationButton href="https://cocoindex.io/docs/ops/functions#sentencetransformerembed" text="SentenceTransformerEmbed" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/functions#sentencetransformerembed" text="SentenceTransformerEmbed" margin="0 0 16px 0" />
 
 :::tip
 `@cocoindex.transform_flow()` is needed to share the transformation across indexing and query. When building a vector index and querying against it, the embedding computation must remain consistent between indexing and querying.
 :::
 
-<DocumentationButton href="https://cocoindex.io/docs/query#transform-flow" text="Transform Flow" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/query#transform-flow" text="Transform Flow" margin="0 0 16px 0" />
 
 Then for each chunk, we will embed it using the `code_to_embedding` function, and collect the embeddings to the `code_embeddings` collector.
 
@@ -222,4 +222,11 @@ To spin up, it is super easy.
 ```
 cocoindex server main.py -ci
 ```
-Follow the url from the terminal - `https://cocoindex.io/cocoinsight` to access the CocoInsight.
+Follow the url from the terminal - `https://cocoindex.io/cocoinsight` to access the CocoInsight.
+
+
+## Supported Languages
+
+SplitRecursively has native support for all major programming languages.
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/functions#supported-languages" text="Supported Languages" margin="0 0 16px 0" />

{cocoindex-0.2.1 → cocoindex-0.2.3}/docs/docs/examples/examples/custom_targets.md

@@ -11,7 +11,9 @@ tags: [custom-building-blocks]
 ---
 import { GitHubButton, YouTubeButton, DocumentationButton } from '../../../src/components/GitHubButton';
 
-<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/custom_output_files"/>
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/custom_output_files" margin="0 0 24px 0" />
+
+![Custom Targets](/img/examples/custom_targets/cover.png)
 
 ## Overview
 
@@ -33,7 +35,7 @@ flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
 	)
 ```
 This ingestion creates a table with `filename` and `content` fields. 
-<DocumentationButton href="https://cocoindex.io/docs/ops/sources" text="Sources" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Sources" />
 
 ## Process each file and collect
 
@@ -45,7 +47,7 @@ def markdown_to_html(text: str) -> str:
     return _markdown_it.render(text)
 ```
 
-<DocumentationButton href="https://cocoindex.io/docs/custom_ops/custom_functions" text="Custom Function" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_functions" text="Custom Function" margin="0 0 16px 0" />
 
 Define data collector and transform each document to html.
 
@@ -62,7 +64,7 @@ with data_scope["documents"].row() as doc:
 
 ### Define the target spec
 
-<DocumentationButton href="https://cocoindex.io/docs/custom_ops/custom_targets#target-spec" text="Target Spec" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_targets#target-spec" text="Target Spec" margin="0 0 16px 0" />
 
 The target spec contains a directory for output files:
 
@@ -74,7 +76,7 @@ class LocalFileTarget(cocoindex.op.TargetSpec):
 
 ### Implement the connector
 
-<DocumentationButton href="https://cocoindex.io/docs/custom_ops/custom_targets#target-connector" text="Target Connector" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_targets#target-connector" text="Target Connector" margin="0 0 16px 0" />
 
 `get_persistent_key()` defines the persistent key,
 which uniquely identifies the target for change tracking and incremental updates. Here, we simply use the target directory as the key (e.g., `./data/output`).

{cocoindex-0.2.1 → cocoindex-0.2.3}/docs/docs/examples/examples/docs_to_knowledge_graph.md

@@ -12,8 +12,10 @@ tags: [knowledge-graph, structured-data-extraction]
 
 import { GitHubButton, YouTubeButton, DocumentationButton } from '../../../src/components/GitHubButton';
 
-<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph"/>
-<YouTubeButton url="https://youtu.be/2KVkpUGRtnk?si=MRalDweWrid-IFje" />
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph" margin="0 0 24px 0" />
+<YouTubeButton url="https://youtu.be/2KVkpUGRtnk?si=MRalDweWrid-IFje" margin="0 0 24px 0" />
+
+![Knowledge Graph for Docs](/img/examples/docs_to_knowledge_graph/cover.png)
 
 ## Overview
 [CocoIndex](https://github.com/cocoindex-io/cocoindex) makes it easy to build and maintain knowledge graphs with continuous source updates. In this tutorial, we will use LLM to extract relationships between the concepts in each document, and generate two kinds of relationships:
@@ -37,11 +39,11 @@ and then build a knowledge graph.
 *   [Install Neo4j](https://cocoindex.io/docs/ops/targets#neo4j-dev-instance), a graph database.
 *   [Configure your OpenAI API key](https://cocoindex.io/docs/ai/llm#openai).  Alternatively, we have native support for Gemini, Ollama, LiteLLM. You can choose your favorite LLM provider and work completely on-premises.
 
-    <DocumentationButton href="https://cocoindex.io/docs/ai/llm" text="LLM" margin="0 0 16px 0" />
+    <DocumentationButton url="https://cocoindex.io/docs/ai/llm" text="LLM" margin="0 0 16px 0" />
 
 
 ## Documentation
-<DocumentationButton href="https://cocoindex.io/docs/ops/targets#property-graph-targets" text="Property Graph Targets" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/targets#property-graph-targets" text="Property Graph Targets" margin="0 0 16px 0" />
 
 
 ## Data flow to build knowledge graph
@@ -61,7 +63,7 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
 Here `flow_builder.add_source` creates a [KTable](https://cocoindex.io/docs/core/data_types#KTable).
 `filename` is the key of the KTable.
 
-<DocumentationButton href="https://cocoindex.io/docs/ops/sources" text="Sources" margin="0 0 16px 0" />
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Sources" margin="0 0 16px 0" />
 
 
 ### Add data collectors
@@ -366,7 +368,7 @@ MATCH p=()-->() RETURN p
 ## Kuzu
 Cocoindex natively supports Kuzu - a high performant, embedded open source graph database.
 
-<DocumentationButton href="https://cocoindex.io/docs/ops/targets#kuzu" text="Kuzu" margin="0 0 16px 0" /> 
+<DocumentationButton url="https://cocoindex.io/docs/ops/targets#kuzu" text="Kuzu" margin="0 0 16px 0" /> 
 
 The GraphDB interface in CocoIndex is standardized, you just need to **switch the configuration** without any additional code changes. CocoIndex supports exporting to Kuzu through its API server. You can bring up a Kuzu API server locally by running: