cocoindex 0.2.3__tar.gz → 0.2.5__tar.gz

{cocoindex-0.2.3 → cocoindex-0.2.5}/.github/workflows/_test.yml

@@ -62,4 +62,25 @@ jobs:
           if: ${{ startsWith(matrix.platform.runner, 'windows') }}
           shell: cmd  # Use `cmd` to run test for Windows, as PowerShell doesn't detect exit code by `os._exit(0)` correctly.
           run: |
-            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests
+
+    validate-3p-notices:
+        runs-on: ubuntu-latest
+        steps:
+        - uses: actions/checkout@v4
+        - name: Rust Cache
+          uses: Swatinem/rust-cache@v2
+        - name: Install Rust toolchain
+          uses: dtolnay/rust-toolchain@stable
+        - name: Install cargo-about
+          run: cargo install --locked cargo-about
+        - name: Validate third-party notices (dry-run)
+          shell: bash
+          run: |
+            set +e
+            cargo about generate about.hbs > /dev/null
+            status=$?
+            if [ $status -ne 0 ]; then
+              echo "::error::Third-party notices validation failed. Please update /about.toml and rerun."
+              exit $status
+            fi

{cocoindex-0.2.3 → cocoindex-0.2.5}/.github/workflows/release.yml

@@ -94,10 +94,32 @@ jobs:
           name: wheels-sdist
           path: dist
 
+  generate-3p-notices:
+    runs-on: ubuntu-latest
+    needs: [create-versioned-toml]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: Cargo.toml
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install cargo-about
+        run: cargo install --locked cargo-about
+      - name: Generate THIRD_PARTY_NOTICES.html
+        run: cargo about generate -o THIRD_PARTY_NOTICES.html about.hbs
+      - name: Upload THIRD_PARTY_NOTICES.html artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: THIRD_PARTY_NOTICES.html
+          path: THIRD_PARTY_NOTICES.html
+
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: [create-versioned-toml, build, test-abi3, sdist]
+    needs: [create-versioned-toml, build, test-abi3, sdist, generate-3p-notices]
     permissions:
       # Use to sign the release artifacts
       id-token: write
@@ -111,6 +133,9 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: Cargo.toml
+      - uses: actions/download-artifact@v4
+        with:
+          name: THIRD_PARTY_NOTICES.html
       - uses: actions/download-artifact@v4
         with:
           pattern: wheels-*

{cocoindex-0.2.3 → cocoindex-0.2.5}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.3"
+version = "0.2.5"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -5091,6 +5091,7 @@ dependencies = [
  "memchr",
  "once_cell",
  "percent-encoding",
+ "rustls 0.23.31",
  "serde",
  "serde_json",
  "sha2",
@@ -5101,6 +5102,7 @@ dependencies = [
  "tracing",
  "url",
  "uuid",
+ "webpki-roots 0.26.11",
 ]
 
 [[package]]

{cocoindex-0.2.3 → cocoindex-0.2.5}/Cargo.toml

@@ -2,9 +2,10 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.3"
+version = "0.2.5"
 edition = "2024"
 rust-version = "1.88"
+license = "Apache-2.0"
 readme = "README.md"
 
 [profile.release]
@@ -43,6 +44,7 @@ sqlx = { version = "0.8.6", features = [
     "postgres",
     "runtime-tokio",
     "uuid",
+    "tls-rustls-aws-lc-rs",
 ] }
 tokio = { version = "1.47.1", features = [
     "macros",

{cocoindex-0.2.3 → cocoindex-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.3
+Version: 0.2.5
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
@@ -34,7 +34,6 @@ Provides-Extra: dev
 Provides-Extra: embeddings
 Provides-Extra: colpali
 Provides-Extra: all
-License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Keywords: indexing,real-time,incremental,pipeline,search,ai,etl,rag,dataflow,context-engineering
 Author-email: CocoIndex <cocoindex.io@gmail.com>

cocoindex-0.2.5/about.hbs

@@ -0,0 +1,70 @@
+<html>
+
+<head>
+    <style>
+        @media (prefers-color-scheme: dark) {
+            body {
+                background: #333;
+                color: white;
+            }
+            a {
+                color: skyblue;
+            }
+        }
+        .container {
+            font-family: sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+        }
+        .intro {
+            text-align: center;
+        }
+        .licenses-list {
+            list-style-type: none;
+            margin: 0;
+            padding: 0;
+        }
+        .license-used-by {
+            margin-top: -10px;
+        }
+        .license-text {
+            max-height: 200px;
+            overflow-y: scroll;
+            white-space: pre-wrap;
+        }
+    </style>
+</head>
+
+<body>
+    <main class="container">
+        <div class="intro">
+            <h1>Third Party Licenses</h1>
+            <p>This page lists the licenses of the projects used in cargo-about.</p>
+        </div>
+
+        <h2>Overview of licenses:</h2>
+        <ul class="licenses-overview">
+            {{#each overview}}
+            <li><a href="#{{id}}">{{name}}</a> ({{count}})</li>
+            {{/each}}
+        </ul>
+
+        <h2>All license text:</h2>
+        <ul class="licenses-list">
+            {{#each licenses}}
+            <li class="license">
+                <h3 id="{{id}}">{{name}}</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    {{#each used_by}}
+                    <li><a href="{{#if crate.repository}} {{crate.repository}} {{else}} https://crates.io/crates/{{crate.name}} {{/if}}">{{crate.name}} {{crate.version}}</a></li>
+                    {{/each}}
+                </ul>
+                <pre class="license-text">{{text}}</pre>
+            </li>
+            {{/each}}
+        </ul>
+    </main>
+</body>
+
+</html>

cocoindex-0.2.5/about.toml

@@ -0,0 +1,12 @@
+accepted = [
+    "Apache-2.0",
+    "Apache-2.0 WITH LLVM-exception",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "CDLA-Permissive-2.0",
+    "ISC",
+    "MIT",
+    "OpenSSL",
+    "Unicode-3.0",
+    "Zlib",
+]

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/core/cli.mdx

@@ -17,11 +17,16 @@ Once CocoIndex is installed, you can invoke the CLI directly using the `cocoinde
 ### APP_TARGET Format
 
 The `APP_TARGET` can be:
-1.  A **path to a Python file** defining your flows (e.g., `main.py`, `path/to/my_flows.py`).
-2.  An **installed Python module name** that contains your flow definitions (e.g., `my_package.flows`).
+1.  An **Python module name** that contains your flow definitions (e.g., `main`, `my_package.flows`).
+    You can also use `--app-dir <path>` to specify the base directory to load the module from.
+
+2.  A **path to a Python file** defining your flows (e.g., `main.py`, `path/to/my_flows.py`).
+
+    The file will be loaded as a top-level Python module, e.g. relative imports will not work as its parent package is not defined (similar to how `python main.py` resolves imports).
+
 3.  For commands that operate on a *specific flow* (like `show`, `update`, `evaluate`), you can combine the application reference with a flow name:
-    * `path/to/my_flows.py:MyFlow`
     * `my_package.flows:MyFlow`
+    * `path/to/my_flows.py:MyFlow`
 
 ### Environment Variables
 
@@ -44,6 +49,7 @@ If no file is found, only existing system environment variables are used.
 CocoIndex CLI supports the following global options:
 
 * `--env-file <path>`: Load environment variables from a specified `.env` file. If not provided, `.env` in the current directory is loaded if it exists.
+* `--app-dir <path>`: Load apps from the specified directory. It will be treated as part of `PYTHONPATH`. Default to the current directory.
 * `--version`: Show the CocoIndex version and exit.
 * `--help`: Show the main help message and exit.
 

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/core/flow_methods.mdx

@@ -190,7 +190,7 @@ A data source may enable one or multiple *change capture mechanisms*:
 *   Configured with a [refresh interval](flow_def#refresh-interval), which is generally applicable to all data sources.
 
 *   Specific data sources also provide their specific change capture mechanisms.
-    For example, [`AmazonS3` source](../ops/sources/#amazons3) watches S3 bucket's change events, and [`GoogleDrive` source](../ops/sources#googledrive) allows polling recent modified files.
+    For example, [`Postgres` source](../ops/sources/#postgres) listens to PostgreSQL's change notifications, [`AmazonS3` source](../ops/sources/#amazons3) watches S3 bucket's change events, and [`GoogleDrive` source](../ops/sources#googledrive) allows polling recent modified files.
     See documentations for specific data sources.
 
 Change capture mechanisms enable CocoIndex to continuously capture changes from the source data and update the target data accordingly, under live update mode.

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/academic_papers_index.md

@@ -342,7 +342,7 @@ You can walk through the project step by step in [CocoInsight](https://www.youtu
 
 
 ```sh
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 
 Follow the url `https://cocoindex.io/cocoinsight`.  It connects to your local CocoIndex server, with zero pipeline data retention.

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/docs_to_knowledge_graph.md

@@ -345,7 +345,7 @@ It creates relationships by:
 I used CocoInsight to troubleshoot the index generation and understand the data lineage of the pipeline.  It is in free beta now, you can give it a try. 
 
 ```sh
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 
 And then open the url `https://cocoindex.io/cocoinsight`.  It just connects to your local CocoIndex server, with zero pipeline data retention.

cocoindex-0.2.5/docs/docs/examples/examples/document_ai.md

@@ -0,0 +1,156 @@
+---
+title: Bring your own parser as building block with Google Document AI
+description: Use Google Document AI to parse document, embed the resulting text, and store it in a vectorized database for semantic search.
+sidebar_class_name: hidden
+slug: /examples/document_ai
+canonicalUrl: '/examples/document_ai'
+sidebar_custom_props:
+  image: /img/examples/document_ai/cover.png
+  tags: [vector-index, custom-building-block]
+tags: [vector-index, custom-building-block]
+---
+import { GitHubButton, DocumentationButton, ExampleButton } from '../../../src/components/GitHubButton';
+
+
+<GitHubButton url="https://github.com/cocoindex-io/cocoindex/tree/main/examples/document_ai" margin="0 0 24px 0" />
+
+![Document AI](/img/examples/document_ai/cover.png)
+
+CocoIndex is a flexible ETL framework with incremental processing.  We don’t build parser ourselves, and users can bring in any open source or commercial parser that works best for their scenarios.  In this example, we show how to use **Google Document AI to parse document**, embed the resulting text, and store it in a vectorized database for semantic search.
+
+## Set up
+- [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+- Configure Project and Processor ID for Document AI API 
+    - [Official Google document AI API](https://cloud.google.com/document-ai/docs/try-docai) with free live demo.
+    - Sign in to [Google Cloud Console](https://console.cloud.google.com/), create or open a project, and enable Document AI API.
+      - ![image.png](/img/examples/document_ai/document_ai.png)
+      - ![image.png](/img/examples/document_ai/processor.png)
+- update `.env` with `GOOGLE_CLOUD_PROJECT_ID` and `GOOGLE_CLOUD_PROCESSOR_ID`.
+
+
+## Create Your building block to convert PDFs to Markdown
+
+We define a `ToMarkdown` custom function spec, which leverages Google Document AI to parse PDF content:
+
+```python
+class ToMarkdown(cocoindex.op.FunctionSpec):
+    """Convert a PDF to markdown using Google Document AI."""
+```
+
+The corresponding executor class handles API initialization and parsing logic:
+
+```python
+@cocoindex.op.executor_class(cache=True, behavior_version=1)
+class DocumentAIExecutor:
+    """Executor for Google Document AI to parse PDF files."""
+
+    spec: ToMarkdown
+    _client: documentai.DocumentProcessorServiceClient
+    _processor_name: str
+
+    def prepare(self):
+        # Initialize the Document AI client
+        project_id = os.environ.get("GOOGLE_CLOUD_PROJECT_ID")
+        location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us")
+        processor_id = os.environ.get("GOOGLE_CLOUD_PROCESSOR_ID")
+
+        opts = ClientOptions(api_endpoint=f"{location}-documentai.googleapis.com")
+        self._client = documentai.DocumentProcessorServiceClient(client_options=opts)
+        self._processor_name = self._client.processor_path(project_id, location, processor_id)
+
+    async def __call__(self, content: bytes) -> str:
+        """Parse PDF content and convert to markdown text."""
+        request = documentai.ProcessRequest(
+            name=self._processor_name,
+            raw_document=documentai.RawDocument(content=content, mime_type="application/pdf")
+        )
+        response = self._client.process_document(request=request)
+        return response.document.text
+```
+
+Make sure you configure the `cache` and `behavior_version` parameters for heavy operations like this.
+
+- `cache`: Whether the executor will enable cache for this function. When True, the executor will cache the result of the function for reuse during reprocessing. We recommend to set this to True for any function that is computationally intensive.
+
+- `behavior_version`: The version of the behavior of the function. When the version is changed, the function will be re-executed even if cache is enabled. It's required to be set if cache is True.
+
+
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_functions#option-2-by-a-function-spec-and-an-executor" text="Custom Functions" margin="0 0 16px 0" />
+
+<DocumentationButton url="https://cocoindex.io/docs/custom_ops/custom_functions#parameters-for-custom-functions" text="Parameters for Custom Functions" margin="0 0 16px 0" />
+
+## Define the flow
+
+```python
+@cocoindex.flow_def(name="DocumentAiPdfEmbedding")
+def pdf_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    # flow definition
+```
+
+### Add source & collector
+
+```python
+data_scope["documents"] = flow_builder.add_source(
+    cocoindex.sources.LocalFile(path="pdf_files", binary=True)
+)
+
+doc_embeddings = data_scope.add_collector()
+```
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Source" margin="0 0 16px 0" />
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/collectors" text="Collector" margin="0 0 16px 0" />
+
+### Process each document
+
+```python
+with data_scope["documents"].row() as doc:
+    doc["markdown"] = doc["content"].transform(ToMarkdown())
+    doc["chunks"] = doc["markdown"].transform(
+        cocoindex.functions.SplitRecursively(),
+        language="markdown",
+        chunk_size=2000,
+        chunk_overlap=500
+    )
+    with doc["chunks"].row() as chunk:
+        chunk["embedding"] = chunk["text"].call(text_to_embedding)
+        doc_embeddings.collect(
+            id=cocoindex.GeneratedField.UUID,
+            filename=doc["filename"],
+            location=chunk["location"],
+            text=chunk["text"],
+            embedding=chunk["embedding"]
+        )
+```
+
+1. Convert them to Markdown using Document AI.
+2. Split the Markdown into chunks.
+3. Embed each chunk.
+
+## Export to Postgres
+
+```python
+doc_embeddings.export(
+    "doc_embeddings",
+    cocoindex.storages.Postgres(),
+    primary_key_fields=["id"],
+    vector_indexes=[
+        cocoindex.VectorIndexDef(
+            field_name="embedding",
+            metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY
+        )
+    ]
+)
+```
+
+## End to End Example
+
+For a step-by-step walkthrough of each indexing stage and the query path, check out this example:
+
+<ExampleButton href="https://cocoindex.io/docs/examples/simple_vector_index" text="Simple Vector Index" margin="0 0 16px 0" />
+
+## Other sources
+
+CocoIndex natively supports Google Drive, Amazon S3, Azure Blob Storage, and more with native incremental processing out of box - when new or updated files are detected, the pipeline will capture the changes and only process what's changed.
+
+<DocumentationButton url="https://cocoindex.io/docs/ops/sources" text="Sources" margin="0 0 16px 0" />

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/manual_extraction.md

@@ -245,7 +245,7 @@ SELECT filename, module_info->'title' AS title, module_summary FROM modules_info
 [CocoInsight](https://www.youtube.com/watch?v=ZnmyoHslBSc) is a really cool tool to help you understand your data pipeline and data index. It is in Early Access now (Free).
 
 ```sh
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 CocoInsight dashboard is here `https://cocoindex.io/cocoinsight`.  It connects to your local CocoIndex server with zero data retention.
 

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/multi_format_index.md

@@ -193,7 +193,7 @@ You can walk through the project step by step in [CocoInsight](https://www.youtu
 
 
 ```sh
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 
 Follow the url `https://cocoindex.io/cocoinsight`.  It connects to your local CocoIndex server, with zero pipeline data retention. You can use it to view extracted pages, see embedding vectors and metadata.

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/patient_form_extraction.md

@@ -283,7 +283,7 @@ We could troubleshoot in two steps:
 I also use CocoInsight to help me troubleshoot.
 
 ```bash
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 
 Go to `https://cocoindex.io/cocoinsight`. You could see an interactive UI to explore the data.

{cocoindex-0.2.3 → cocoindex-0.2.5}/docs/docs/examples/examples/photo_search.md

@@ -206,7 +206,7 @@ CocoInsight is a tool to help you understand your data pipeline and data index.
 You can walk through the project step by step in [CocoInsight](https://www.youtube.com/watch?v=MMrpUfUcZPk) to see exactly how each field is constructed and what happens behind the scenes.
 
 ```sh
-cocoindex server -ci main.py
+cocoindex server -ci main
 ```
 
 Follow the url `https://cocoindex.io/cocoinsight`.  It connects to your local CocoIndex server, with zero pipeline data retention.