cocoindex 0.1.44__tar.gz → 0.1.45__tar.gz

cocoindex-0.1.45/.github/workflows/CI.yml

@@ -0,0 +1,46 @@
+# This file is autogenerated by maturin v1.8.1
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - src/**
+      - python/**
+      - "*.toml"
+      - ".github/workflows/*.yml"
+  push:
+    branches: [main]
+    paths:
+      - src/**
+      - python/**
+      - "*.toml"
+      - ".github/workflows/*.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  format-check:
+    name: Check Python formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install Ruff
+        run: |
+          pip install ruff
+      - name: Check Python formatting
+        run: |
+          ruff format --check .
+
+  test:
+    name: Run test
+    uses: ./.github/workflows/_test.yml

{cocoindex-0.1.44 → cocoindex-0.1.45}/.github/workflows/_test.yml

@@ -36,10 +36,22 @@ jobs:
             key: ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
             restore-keys: |
               ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-
-        - name: Python build & test
+        - name: Setup venv
           run: |
             python -m venv .venv
+        - name: Install Python toolchains
+          run: |
+            source .venv/bin/activate
+            pip install maturin pytest mypy
+        - name: Python build
+          run: |
             source .venv/bin/activate
-            pip install maturin pytest
             maturin develop
+        - name: Python type check (mypy)
+          run: |
+            source .venv/bin/activate
+            mypy python
+        - name: Python tests
+          run: |
+            source .venv/bin/activate
             pytest python/cocoindex/tests

{cocoindex-0.1.44 → cocoindex-0.1.45}/.vscode/settings.json

@@ -3,5 +3,7 @@
         "cocoindex",
         "reindexing",
         "timedelta"
-    ]
+    ],
+    "editor.formatOnSave": true,
+    "python.formatting.provider": "ruff"
 }

{cocoindex-0.1.44 → cocoindex-0.1.45}/Cargo.lock

@@ -993,7 +993,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.44"
+version = "0.1.45"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.44 → cocoindex-0.1.45}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.44"
+version = "0.1.45"
 edition = "2024"
 
 [profile.release]

{cocoindex-0.1.44 → cocoindex-0.1.45}/PKG-INFO

@@ -1,12 +1,14 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.44
+Version: 0.1.45
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: ruff ; extra == 'dev'
 Provides-Extra: test
+Provides-Extra: dev
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Author-email: CocoIndex <cocoindex.io@gmail.com>

{cocoindex-0.1.44 → cocoindex-0.1.45}/docs/docs/core/settings.mdx

@@ -6,7 +6,7 @@ description: Provide settings for CocoIndex, e.g. database connection, app names
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# CocoIndex Settings
+# CocoIndex Setting
 
 Certain settings need to be provided for CocoIndex to work, e.g. database connections, app namespace, etc.
 

{cocoindex-0.1.44 → cocoindex-0.1.45}/examples/amazon_s3_embedding/main.py

@@ -3,8 +3,11 @@ from dotenv import load_dotenv
 import cocoindex
 import os
 
+
 @cocoindex.flow_def(name="AmazonS3TextEmbedding")
-def amazon_s3_text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+def amazon_s3_text_embedding_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+):
     """
     Define an example flow that embeds text from Amazon S3 into a vector database.
     """
@@ -18,21 +21,32 @@ def amazon_s3_text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scop
             prefix=prefix,
             included_patterns=["*.md", "*.txt", "*.docx"],
             binary=False,
-            sqs_queue_url=sqs_queue_url))
+            sqs_queue_url=sqs_queue_url,
+        )
+    )
 
     doc_embeddings = data_scope.add_collector()
 
     with data_scope["documents"].row() as doc:
         doc["chunks"] = doc["content"].transform(
             cocoindex.functions.SplitRecursively(),
-            language="markdown", chunk_size=2000, chunk_overlap=500)
+            language="markdown",
+            chunk_size=2000,
+            chunk_overlap=500,
+        )
 
         with doc["chunks"].row() as chunk:
             chunk["embedding"] = chunk["text"].transform(
                 cocoindex.functions.SentenceTransformerEmbed(
-                 model="sentence-transformers/all-MiniLM-L6-v2")) 
-            doc_embeddings.collect(filename=doc["filename"], location=chunk["location"],
-                                   text=chunk["text"], embedding=chunk["embedding"])
+                    model="sentence-transformers/all-MiniLM-L6-v2"
+                )
+            )
+            doc_embeddings.collect(
+                filename=doc["filename"],
+                location=chunk["location"],
+                text=chunk["text"],
+                embedding=chunk["embedding"],
+            )
 
     doc_embeddings.export(
         "doc_embeddings",
@@ -41,7 +55,11 @@ def amazon_s3_text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scop
         vector_indexes=[
             cocoindex.VectorIndexDef(
                 field_name="embedding",
-                metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
+                metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+            )
+        ],
+    )
+
 
 query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
     name="SemanticsSearch",
@@ -49,27 +67,29 @@ query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
     target_name="doc_embeddings",
     query_transform_flow=lambda text: text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
-            model="sentence-transformers/all-MiniLM-L6-v2")),
-    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    ),
+    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+)
+
 
 def _main():
     # Use a `FlowLiveUpdater` to keep the flow data updated.
     with cocoindex.FlowLiveUpdater(amazon_s3_text_embedding_flow):
         # Run queries in a loop to demonstrate the query capabilities.
         while True:
-            try:
-                query = input("Enter search query (or Enter to quit): ")
-                if query == '':
-                    break
-                results, _ = query_handler.search(query, 10)
-                print("\nSearch results:")
-                for result in results:
-                    print(f"[{result.score:.3f}] {result.data['filename']}")
-                    print(f"    {result.data['text']}")
-                    print("---")
-                print()
-            except KeyboardInterrupt:
+            query = input("Enter search query (or Enter to quit): ")
+            if query == "":
                 break
+            results, _ = query_handler.search(query, 10)
+            print("\nSearch results:")
+            for result in results:
+                print(f"[{result.score:.3f}] {result.data['filename']}")
+                print(f"    {result.data['text']}")
+                print("---")
+            print()
+
 
 if __name__ == "__main__":
     load_dotenv()

{cocoindex-0.1.44 → cocoindex-0.1.45}/examples/code_embedding/main.py

@@ -3,40 +3,59 @@ from psycopg_pool import ConnectionPool
 import cocoindex
 import os
 
+
 @cocoindex.op.function()
 def extract_extension(filename: str) -> str:
     """Extract the extension of a filename."""
     return os.path.splitext(filename)[1]
 
+
 @cocoindex.transform_flow()
-def code_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
+def code_to_embedding(
+    text: cocoindex.DataSlice[str],
+) -> cocoindex.DataSlice[list[float]]:
     """
     Embed the text using a SentenceTransformer model.
     """
     return text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
-            model="sentence-transformers/all-MiniLM-L6-v2"))
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    )
+
 
 @cocoindex.flow_def(name="CodeEmbedding")
-def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+def code_embedding_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+):
     """
     Define an example flow that embeds files into a vector database.
     """
     data_scope["files"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="../..",
-                                    included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx"],
-                                    excluded_patterns=["**/.*", "target", "**/node_modules"]))
+        cocoindex.sources.LocalFile(
+            path="../..",
+            included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx"],
+            excluded_patterns=["**/.*", "target", "**/node_modules"],
+        )
+    )
     code_embeddings = data_scope.add_collector()
 
     with data_scope["files"].row() as file:
         file["extension"] = file["filename"].transform(extract_extension)
         file["chunks"] = file["content"].transform(
             cocoindex.functions.SplitRecursively(),
-            language=file["extension"], chunk_size=1000, chunk_overlap=300)
+            language=file["extension"],
+            chunk_size=1000,
+            chunk_overlap=300,
+        )
         with file["chunks"].row() as chunk:
             chunk["embedding"] = chunk["text"].call(code_to_embedding)
-            code_embeddings.collect(filename=file["filename"], location=chunk["location"],
-                                    code=chunk["text"], embedding=chunk["embedding"])
+            code_embeddings.collect(
+                filename=file["filename"],
+                location=chunk["location"],
+                code=chunk["text"],
+                embedding=chunk["embedding"],
+            )
 
     code_embeddings.export(
         "code_embeddings",
@@ -45,26 +64,35 @@ def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
         vector_indexes=[
             cocoindex.VectorIndexDef(
                 field_name="embedding",
-                metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
+                metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+            )
+        ],
+    )
 
 
 def search(pool: ConnectionPool, query: str, top_k: int = 5):
     # Get the table name, for the export target in the code_embedding_flow above.
-    table_name = cocoindex.utils.get_target_storage_default_name(code_embedding_flow, "code_embeddings")
+    table_name = cocoindex.utils.get_target_storage_default_name(
+        code_embedding_flow, "code_embeddings"
+    )
     # Evaluate the transform flow defined above with the input query, to get the embedding.
     query_vector = code_to_embedding.eval(query)
     # Run the query and get the results.
     with pool.connection() as conn:
         with conn.cursor() as cur:
-            cur.execute(f"""
+            cur.execute(
+                f"""
                 SELECT filename, code, embedding <=> %s::vector AS distance
                 FROM {table_name} ORDER BY distance LIMIT %s
-            """, (query_vector, top_k))
+            """,
+                (query_vector, top_k),
+            )
             return [
                 {"filename": row[0], "code": row[1], "score": 1.0 - row[2]}
                 for row in cur.fetchall()
             ]
 
+
 def _main():
     # Make sure the flow is built and up-to-date.
     stats = code_embedding_flow.update()
@@ -74,20 +102,18 @@ def _main():
     pool = ConnectionPool(os.getenv("COCOINDEX_DATABASE_URL"))
     # Run queries in a loop to demonstrate the query capabilities.
     while True:
-        try:
-            query = input("Enter search query (or Enter to quit): ")
-            if query == '':
-                break
-            # Run the query function with the database connection pool and the query.
-            results = search(pool, query)
-            print("\nSearch results:")
-            for result in results:
-                print(f"[{result['score']:.3f}] {result['filename']}")
-                print(f"    {result['code']}")
-                print("---")
-            print()
-        except KeyboardInterrupt:
+        query = input("Enter search query (or Enter to quit): ")
+        if query == "":
             break
+        # Run the query function with the database connection pool and the query.
+        results = search(pool, query)
+        print("\nSearch results:")
+        for result in results:
+            print(f"[{result['score']:.3f}] {result['filename']}")
+            print(f"    {result['code']}")
+            print("---")
+        print()
+
 
 if __name__ == "__main__":
     load_dotenv()

{cocoindex-0.1.44 → cocoindex-0.1.45}/examples/docs_to_knowledge_graph/main.py

@@ -1,42 +1,71 @@
 """
 This example shows how to extract relationships from documents and build a knowledge graph.
 """
+
 import dataclasses
 import cocoindex
 
+
 @dataclasses.dataclass
 class DocumentSummary:
     """Describe a summary of a document."""
+
     title: str
     summary: str
 
+
 @dataclasses.dataclass
 class Relationship:
     """
     Describe a relationship between two entities.
     Subject and object should be Core CocoIndex concepts only, should be nouns. For example, `CocoIndex`, `Incremental Processing`, `ETL`,  `Data` etc.
     """
+
     subject: str
     predicate: str
     object: str
 
+
+neo4j_conn_spec = cocoindex.add_auth_entry(
+    "Neo4jConnection",
+    cocoindex.storages.Neo4jConnection(
+        uri="bolt://localhost:7687",
+        user="neo4j",
+        password="cocoindex",
+    ),
+)
+kuzu_conn_spec = cocoindex.add_auth_entry(
+    "KuzuConnection",
+    cocoindex.storages.KuzuConnection(
+        api_server_url="http://localhost:8123",
+    ),
+)
+
+# Use Neo4j as the graph database
+GraphDbSpec = cocoindex.storages.Neo4j
+GraphDbConnection = cocoindex.storages.Neo4jConnection
+GraphDbDeclaration = cocoindex.storages.Neo4jDeclaration
+conn_spec = neo4j_conn_spec
+
+# Use Kuzu as the graph database
+#  GraphDbSpec = cocoindex.storages.Kuzu
+#  GraphDbConnection = cocoindex.storages.KuzuConnection
+#  GraphDbDeclaration = cocoindex.storages.KuzuDeclaration
+#  conn_spec = kuzu_conn_spec
+
+
 @cocoindex.flow_def(name="DocsToKG")
-def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+def docs_to_kg_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+) -> None:
     """
     Define an example flow that extracts relationship from files and build knowledge graph.
     """
-    # configure neo4j connection
-    conn_spec = cocoindex.add_auth_entry(
-        "Neo4jConnection",
-        cocoindex.storages.Neo4jConnection(
-            uri="bolt://localhost:7687",
-            user="neo4j",
-            password="cocoindex",
-    ))
-
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="../../docs/docs/core",
-                                    included_patterns=["*.md", "*.mdx"]))
+        cocoindex.sources.LocalFile(
+            path="../../docs/docs/core", included_patterns=["*.md", "*.mdx"]
+        )
+    )
 
     document_node = data_scope.add_collector()
     entity_relationship = data_scope.add_collector()
@@ -48,24 +77,34 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
             cocoindex.functions.ExtractByLlm(
                 llm_spec=cocoindex.LlmSpec(
                     # Supported LLM: https://cocoindex.io/docs/ai/llm
-                    api_type=cocoindex.LlmApiType.OPENAI, model="gpt-4o"),
+                    api_type=cocoindex.LlmApiType.OPENAI,
+                    model="gpt-4o",
+                ),
                 output_type=DocumentSummary,
-                instruction="Please summarize the content of the document."))
+                instruction="Please summarize the content of the document.",
+            )
+        )
         document_node.collect(
-            filename=doc["filename"], title=doc["summary"]["title"],
-            summary=doc["summary"]["summary"])
+            filename=doc["filename"],
+            title=doc["summary"]["title"],
+            summary=doc["summary"]["summary"],
+        )
 
         # extract relationships from document
         doc["relationships"] = doc["content"].transform(
             cocoindex.functions.ExtractByLlm(
                 llm_spec=cocoindex.LlmSpec(
                     # Supported LLM: https://cocoindex.io/docs/ai/llm
-                    api_type=cocoindex.LlmApiType.OPENAI, model="gpt-4o"),
-                    output_type=list[Relationship],
-                    instruction=(
-                        "Please extract relationships from CocoIndex documents. "
-                        "Focus on concepts and ignore examples and code. "
-                        )))
+                    api_type=cocoindex.LlmApiType.OPENAI,
+                    model="gpt-4o",
+                ),
+                output_type=list[Relationship],
+                instruction=(
+                    "Please extract relationships from CocoIndex documents. "
+                    "Focus on concepts and ignore examples and code. "
+                ),
+            )
+        )
 
         with doc["relationships"].row() as relationship:
             # relationship between two entities
@@ -77,27 +116,28 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
             )
             # mention of an entity in a document, for subject
             entity_mention.collect(
-                id=cocoindex.GeneratedField.UUID, entity=relationship["subject"],
+                id=cocoindex.GeneratedField.UUID,
+                entity=relationship["subject"],
                 filename=doc["filename"],
             )
             # mention of an entity in a document, for object
             entity_mention.collect(
-                id=cocoindex.GeneratedField.UUID, entity=relationship["object"],
+                id=cocoindex.GeneratedField.UUID,
+                entity=relationship["object"],
                 filename=doc["filename"],
             )
 
-
     # export to neo4j
     document_node.export(
         "document_node",
-        cocoindex.storages.Neo4j(
-            connection=conn_spec,
-            mapping=cocoindex.storages.Nodes(label="Document")),
+        GraphDbSpec(
+            connection=conn_spec, mapping=cocoindex.storages.Nodes(label="Document")
+        ),
         primary_key_fields=["filename"],
     )
     # Declare reference Node to reference entity node in a relationship
     flow_builder.declare(
-        cocoindex.storages.Neo4jDeclaration(
+        GraphDbDeclaration(
             connection=conn_spec,
             nodes_label="Entity",
             primary_key_fields=["value"],
@@ -105,7 +145,7 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
     )
     entity_relationship.export(
         "entity_relationship",
-        cocoindex.storages.Neo4j(
+        GraphDbSpec(
             connection=conn_spec,
             mapping=cocoindex.storages.Relationships(
                 rel_type="RELATIONSHIP",
@@ -113,15 +153,17 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
                     label="Entity",
                     fields=[
                         cocoindex.storages.TargetFieldMapping(
-                            source="subject", target="value"),
-                    ]
+                            source="subject", target="value"
+                        ),
+                    ],
                 ),
                 target=cocoindex.storages.NodeFromFields(
                     label="Entity",
                     fields=[
                         cocoindex.storages.TargetFieldMapping(
-                            source="object", target="value"),
-                    ]
+                            source="object", target="value"
+                        ),
+                    ],
                 ),
             ),
         ),
@@ -129,7 +171,7 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
     )
     entity_mention.export(
         "entity_mention",
-        cocoindex.storages.Neo4j(
+        GraphDbSpec(
             connection=conn_spec,
             mapping=cocoindex.storages.Relationships(
                 rel_type="MENTION",
@@ -139,8 +181,11 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
                 ),
                 target=cocoindex.storages.NodeFromFields(
                     label="Entity",
-                    fields=[cocoindex.storages.TargetFieldMapping(
-                        source="entity", target="value")],
+                    fields=[
+                        cocoindex.storages.TargetFieldMapping(
+                            source="entity", target="value"
+                        )
+                    ],
                 ),
             ),
         ),

cocoindex-0.1.45/examples/fastapi_server_docker/.env

@@ -0,0 +1,5 @@
+# for docker
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@coco_db:5436/cocoindex
+
+# For local testing
+# COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.45/examples/fastapi_server_docker/README.md

@@ -0,0 +1,65 @@
+## Run docker container with a simple query endpoint via fastapi
+
+In this example, we will build index for text embedding from local markdown files, and provide a simple query endpoint via fastapi.
+We provide a simple docker container using docker compose to build pgvector17 along with a simple python fastapi script
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+
+## Run locally without docker
+
+In the `.env` file, use local Postgres URL
+
+```
+# For local testing
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+```
+
+- Install dependencies:
+
+    ```bash
+    pip install -e .
+    ```
+
+- Setup:
+
+    ```bash
+    cocoindex setup main.py
+    ```
+
+- Update index:
+
+    ```bash
+    cocoindex update main.py
+    ```
+
+- Run:
+
+    ```bash
+    uvicorn main:fastapi_app --reload --host 0.0.0.0 --port 8000
+    ```
+ 
+ ## Query the endpoint
+
+    ```bash
+    curl "http://localhost:8000/search?q=model&limit=3"
+    ```
+
+
+## Run Docker
+
+In the `.env` file, use Docker Postgres URL
+
+```
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@coco_db:5436/cocoindex
+```
+
+Build the docker container via: 
+```bash
+docker compose up --build
+```
+
+Test the endpoint:
+```bash
+curl "http://0.0.0.0:8080/search?q=model&limit=3"
+```

{cocoindex-0.1.44 → cocoindex-0.1.45}/examples/fastapi_server_docker/compose.yaml

@@ -6,8 +6,10 @@ services:
       POSTGRES_USER: cocoindex
       POSTGRES_PASSWORD: cocoindex
       POSTGRES_DB: cocoindex
+      POSTGRES_PORT: 5436
     ports:
-      - "5432:5432"
+      - "5436:5436"
+    command: postgres -p 5436
 
   coco_api:
     build: