cocoindex 0.2.20__tar.gz → 0.2.21__tar.gz

{cocoindex-0.2.20 → cocoindex-0.2.21}/.github/workflows/_docs_release.yml

@@ -13,6 +13,8 @@ jobs:
     environment: docs-release
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/setup-node@v4
         with:
           cache: yarn

{cocoindex-0.2.20 → cocoindex-0.2.21}/.github/workflows/_test.yml

@@ -17,37 +17,48 @@ jobs:
                 platform:
                   - { runner: ubuntu-latest, python_exec: ".venv/bin/python" }
                   - { runner: ubuntu-24.04-arm, python_exec: ".venv/bin/python" }
-                  - { runner: macos-latest, python_exec: ".venv/bin/python" }
-                  - { runner: macos-13, python_exec: ".venv/bin/python" }
+                  - { runner: macos-latest, python_exec: ".venv/bin/python", extra_rustflags: "-C split-debuginfo=off"}
+                  - { runner: macos-13, python_exec: ".venv/bin/python", extra_rustflags: "-C split-debuginfo=off"}
                   - { runner: windows-latest, python_exec: ".venv\\Scripts\\python" }
         runs-on: ${{ matrix.platform.runner }}
+        env:
+          RUSTFLAGS: "-C debuginfo=0 ${{ matrix.platform.extra_rustflags }}"
+          CARGO_INCREMENTAL: "0"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"
+
         steps:
         - uses: actions/checkout@v4
+          with:
+            fetch-depth: 1
+
+        - name: Run sccache-cache
+          uses: mozilla-actions/sccache-action@v0.0.9
 
         - uses: actions/setup-python@v5
           id: setup_python
           with:
             python-version: ${{ matrix.python-version }}
-            cache: 'pip'
+            # Python cache is huge and saves little time, so disabled for now, until we have higher cache size limits
+            # cache: 'pip'
 
         - run: rustup toolchain install stable --profile minimal
-        - name: Rust Cache
-          uses: Swatinem/rust-cache@v2
-          with:
-            key: rust-${{ matrix.platform.runner }}-${{ matrix.python-version }}
+
+        # Disable Rust cache until we have higher cache size limits
+        # - name: Rust Cache
+        #   uses: Swatinem/rust-cache@v2
+        #   with:
+        #     key: rust-${{ matrix.platform.runner }}-${{ matrix.python-version }}
+        #     cache-targets: 'false'
+        #     cache-all-crates: 'false'
+        #     cache-workspace-crates: 'false'
+
         - name: Rust tests (no default features)
           run: cargo test --no-default-features --verbose
 
         - name: Rust tests
           run: cargo test --verbose
 
-        - uses: actions/cache@v4
-          with:
-            path: .venv
-            key: pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
-            restore-keys: |
-              pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-
-
         - name: Setup venv
           run: |
             python -m venv .venv
@@ -56,7 +67,7 @@ jobs:
             ${{ matrix.platform.python_exec }} -m pip install maturin mypy pytest pytest-asyncio
         - name: Python build
           run: |
-            ${{ matrix.platform.python_exec }} -m maturin develop -E all
+            ${{ matrix.platform.python_exec }} -m maturin develop --strip -E all
         - name: Python type check (mypy)
           run: |
             ${{ matrix.platform.python_exec }} -m mypy python
@@ -68,6 +79,8 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
+          with:
+            fetch-depth: 1
         - name: Install Rust toolchain
           uses: dtolnay/rust-toolchain@stable
         - uses: taiki-e/install-action@v2

{cocoindex-0.2.20 → cocoindex-0.2.21}/.github/workflows/docs_test.yml

@@ -15,6 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/setup-node@v4
         with:
           cache: yarn

{cocoindex-0.2.20 → cocoindex-0.2.21}/.github/workflows/format.yml

@@ -29,6 +29,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
@@ -41,6 +43,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/setup-python@v5
         with:
           python-version: 3.11

{cocoindex-0.2.20 → cocoindex-0.2.21}/.github/workflows/release.yml

@@ -19,6 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - run: ./.github/scripts/update_version.sh
       - uses: actions/upload-artifact@v4
         with:
@@ -30,6 +32,8 @@ jobs:
     needs: [create-versioned-toml]
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/download-artifact@v4
         with:
           name: Cargo.toml
@@ -61,6 +65,8 @@ jobs:
           - { os: windows, runner: windows-latest, target: x64 }
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/download-artifact@v4
         with:
           name: Cargo.toml
@@ -107,6 +113,8 @@ jobs:
     needs: [create-versioned-toml, generate-3p-notices]
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/download-artifact@v4
         with:
           name: Cargo.toml
@@ -138,6 +146,8 @@ jobs:
     environment: release
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - uses: actions/download-artifact@v4
         with:
           name: Cargo.toml

{cocoindex-0.2.20 → cocoindex-0.2.21}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.20"
+version = "0.2.21"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -1305,6 +1305,7 @@ dependencies = [
  "config",
  "const_format",
  "derivative",
+ "encoding_rs",
  "env_logger",
  "expect-test",
  "futures",

{cocoindex-0.2.20 → cocoindex-0.2.21}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.20"
+version = "0.2.21"
 edition = "2024"
 rust-version = "1.89"
 license = "Apache-2.0"
@@ -10,6 +10,7 @@ readme = "README.md"
 
 [profile.release]
 codegen-units = 1
+strip = "symbols"
 lto = true
 
 [lib]
@@ -153,3 +154,4 @@ azure_storage_blobs = { version = "0.21.0", default-features = false, features =
 ] }
 serde_path_to_error = "0.1.17"
 expect-test = "1.5.0"
+encoding_rs = "0.8.35"

{cocoindex-0.2.20 → cocoindex-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.20
+Version: 0.2.21
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent

{cocoindex-0.2.20 → cocoindex-0.2.21}/THIRD_PARTY_NOTICES.html

@@ -2428,7 +2428,7 @@ Software.
                 <h3 id="Apache-2.0">Apache License 2.0</h3>
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
-                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.20</a></li>
+                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.21</a></li>
                     <li><a href=" https://github.com/awesomized/crc-fast-rust ">crc-fast 1.3.0</a></li>
                     <li><a href=" https://github.com/qdrant/rust-client ">qdrant-client 1.15.0</a></li>
                 </ul>
@@ -10677,6 +10677,38 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.</pre>
+            </li>
+            <li class="license">
+                <h3 id="MIT">MIT License</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    <li><a href=" https://github.com/tree-sitter/tree-sitter-scala ">tree-sitter-scala 0.24.0</a></li>
+                </ul>
+                <pre class="license-text">(The MIT License)
+
+Copyright (c) 2014 Nathan Rajlich &lt;nathan@tootallnate.net&gt;
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the &quot;Software&quot;), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+</pre>
             </li>
             <li class="license">
                 <h3 id="MIT">MIT License</h3>
@@ -12300,32 +12332,6 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
-</pre>
-            </li>
-            <li class="license">
-                <h3 id="MIT">MIT License</h3>
-                <h4>Used by:</h4>
-                <ul class="license-used-by">
-                    <li><a href=" https://github.com/tree-sitter/tree-sitter-scala ">tree-sitter-scala 0.24.0</a></li>
-                </ul>
-                <pre class="license-text">This software is released under the MIT license:
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the &quot;Software&quot;), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 </pre>
             </li>
             <li class="license">

cocoindex-0.2.21/docs/docs/custom_ops/custom_sources.mdx

@@ -0,0 +1,275 @@
+---
+title: Custom Sources
+description: Learn how to create custom sources in CocoIndex to read data from any system including APIs, databases, file systems, and external services. Build source specs and connectors with list and get_value methods for flexible data ingestion operations.
+toc_max_heading_level: 4
+---
+
+A custom source allows you to read data from any system you want, such as APIs, databases, file systems, cloud storage, or other external services.
+You can stream data incrementally and provide ordinal information for efficient updates and change tracking.
+
+## Overview
+
+Custom sources are defined by two components:
+
+* A **source spec** that configures the behavior and connection parameters for the source.
+* A **source connector** that handles the actual data reading operations. It provides the following required methods:
+  * `create()`: Create a connector instance from the source spec.
+  * `list()`: List all available data items. Return keys.
+  * `get_value()`: Get the full content for a specific data item by given key.
+
+## Source Spec
+
+The source spec defines the configuration parameters for your custom source. When you use this source in a flow (typically by calling [`add_source()`](/docs/core/flow_def#import-from-source)), you instantiate this source spec with specific parameter values.
+
+A source spec is defined as a class that inherits from `cocoindex.op.SourceSpec`.
+
+```python
+class CustomSource(cocoindex.op.SourceSpec):
+    """
+    Documentation for the source.
+    """
+    param1: str
+    param2: int | None = None
+    ...
+```
+
+Notes:
+
+* All fields of the spec must have a type serializable / deserializable by the `json` module.
+* All subclasses of `SourceSpec` can be instantiated similar to a dataclass, i.e. `ClassName(param1=value1, param2=value2, ...)`.
+
+## Source Connector
+
+A source connector handles the actual data reading operations for your custom source. It defines how data should be accessed from your source system.
+
+Source connectors implement methods to list available data and retrieve specific data items.
+
+A source connector is defined as a class decorated by `@cocoindex.op.source_connector(spec_cls=CustomSource, key_type=KeyType, value_type=ValueType)`.
+This is a full-featured source connector definition:
+
+```python
+@cocoindex.op.source_connector(
+    spec_cls=CustomSource,
+    key_type=DataKeyType,
+    value_type=DataValueType
+)
+class CustomSourceConnector:
+    @staticmethod
+    async def create(spec: CustomSource) -> "CustomSourceConnector":
+        """Required. Create a connector instance from the spec."""
+        ...
+
+    async def list(self, options: SourceReadOptions) -> AsyncIterator[PartialSourceRow[DataKeyType, DataValueType]]:
+        """Required. List all available data items. `options` is optional."""
+        ...
+
+    async def get_value(self, key: DataKeyType, options: SourceReadOptions) -> PartialSourceRowData[DataValueType]:
+        """Required. Get the full content for a specific data item. `options` is optional."""
+        ...
+
+    def provides_ordinal(self) -> bool:
+        """Optional. Return True if this source provides ordinal information."""
+        return False
+```
+
+Your source connector class doesn't have to have everything above:
+
+* `options` arguments are optional. It provides additional hints to tell if the engine needs a certain property currently.
+   You don't have to take this argument and always provide available properties.
+* `provides_ordinal()` is optional. It's a hint to tell if the engine needs ordinal information.
+* `create()`, `list()` and `get_value()` methods can be async or sync.
+
+The following data types are involved in the method definitions above: `CustomSource`, `DataKeyType`, `DataValueType`, `PartialSourceRow`, `PartialSourceRowData`. They should be replaced with the actual types in your implementation. We will explain each of them below.
+
+### Data Access Methods
+
+Data access methods handle the actual reading operations - discovering available data and retrieving specific content.
+
+#### `async? def create(spec)` (Required)
+
+This static method creates a connector instance from the source spec. It should return a connector that can be used to access data from the source.
+
+```python
+@staticmethod
+async def create(spec: CustomSource) -> "CustomSourceConnector":
+    """
+    Create a connector instance. This is where you initialize connections,
+    validate configuration, authenticate, etc.
+    """
+    # Initialize connections, authenticate, etc.
+    return CustomSourceConnector(spec, ...)
+```
+
+#### `async? def list(options?)` (Required)
+
+This method enumerates all available data items from the source. It should yield `PartialSourceRow` objects containing:
+
+* `key`: A unique identifier for the data item
+* `data`: A `PartialSourceRowData` object with metadata (typically just `ordinal` information)
+
+The optional `options` parameter provides hints about what information is needed:
+
+```python
+async def list(self, options: SourceReadOptions) -> AsyncIterator[PartialSourceRow[DataKeyType, DataValueType]]:
+    """
+    List all available data items. This method is used by CocoIndex to
+    discover what data is available and track changes.
+    """
+    # Enumerate data items from your source
+    for item_metadata in await self._fetch_item_list():
+        data = PartialSourceRowData[DataValueType]()
+
+        # Include ordinal if requested and available.
+        # Must provide if `provides_ordinal()` returns True and `include_ordinal` is True.
+        if options.include_ordinal and item_metadata.timestamp:
+            data.ordinal = item_metadata.timestamp
+
+        # Optionally include full value if it's cheap to fetch and requested
+        if options.include_value and self._can_fetch_cheaply(item_metadata):
+            data.value = await self._fetch_item_content(item_metadata.id)
+
+        yield PartialSourceRow(
+            key=DataKeyType(id=item_metadata.id),
+            data=data
+        )
+```
+
+`options` is a hint. You don't have to take this argument.
+Without it, provide all fields as long as they're available.
+
+#### `async? def get_value(key, options?)` (Required)
+
+This method retrieves the full content for a specific data item given its key. It should return a `PartialSourceRowData` object containing the actual data.
+
+The optional `options` parameter provides hints about what information is needed:
+
+```python
+async def get_value(self, key: DataKeyType, options: SourceReadOptions) -> PartialSourceRowData[DataValueType]:
+    """
+    Get the full content for a specific data item. CocoIndex calls this
+    when it needs the actual data content for processing.
+    """
+    # Fetch the full content for the given key
+    content = await self._fetch_item_content(key.id)
+    if content is None:
+        return PartialSourceRowData(
+            value=NON_EXISTENCE,
+            ordinal=NO_ORDINAL,
+            content_version_fp=None
+        )
+
+    data = PartialSourceRowData(
+        value=DataValueType(
+            title=content.title,
+            content=content.body,
+            author=content.author
+        )
+    )
+
+    # Include ordinal if requested and available
+    if options.include_ordinal and content.timestamp:
+        data.ordinal = content.timestamp
+
+    # Include content version fingerprint if requested and easily available
+    if options.include_content_version_fp and content.etag:
+        data.content_version_fp = content.etag.encode()
+
+    return data
+```
+
+`options` is a hint. You don't have to take this argument.
+Without it, provide all fields as long as they're available.
+
+#### `def provides_ordinal()` (Optional)
+
+Returns `True` if this source provides ordinal information (such as timestamps) that can be used for efficient change detection. If `True`, CocoIndex can use this information to optimize updates.
+
+```python
+def provides_ordinal(self) -> bool:
+    """
+    Return True if this source provides ordinal information.
+    """
+    return True  # If your source provides timestamps or version numbers
+```
+
+### Data Types
+
+#### SourceReadOptions
+
+The `SourceReadOptions` class provides hints to source connectors about what information is needed. This allows connectors to optimize their data fetching:
+
+```python
+@dataclasses.dataclass
+class SourceReadOptions:
+    """Options for reading source data."""
+    include_ordinal: bool = False
+    include_content_version_fp: bool = False
+    include_value: bool = False
+```
+
+**Fields:**
+
+* `include_ordinal`: Whether to include ordinal information (timestamps, version numbers). Required in `list()` when `provides_ordinal()` returns `True` and this is `True`.
+* `include_content_version_fp`: Whether to include content version fingerprints for change detection. Always optional.
+* `include_value`: Whether to include full values. Required in `get_value()` when this is `True`. Optional in `list()` - useful when fetching values is cheap.
+
+#### PartialSourceRow
+
+Represents a single data item from the source:
+
+```python
+@dataclasses.dataclass
+class PartialSourceRow(Generic[K, V]):
+    key: K                              # Unique identifier for the data item
+    data: PartialSourceRowData[V]       # Metadata and optionally the value
+```
+
+#### PartialSourceRowData
+
+Contains the data and metadata for a source item:
+
+```python
+@dataclasses.dataclass
+class PartialSourceRowData(Generic[V]):
+    value: V | Literal["NON_EXISTENCE"] | None = None        # The actual data content
+    ordinal: int | Literal["NO_ORDINAL"] | None = None       # Timestamp or version number
+    content_version_fp: bytes | None = None                  # Content fingerprint for change detection
+```
+
+**Fields:**
+
+* `value`: The actual data content, or `cocoindex.op.NON_EXISTENCE` if the item doesn't exist, or `None` if not included.
+* `ordinal`: Timestamp or version number for change tracking, or `cocoindex.op.NO_ORDINAL` if not available, or `None` if not included. An ordinal needs to be monotonically increasing.
+* `content_version_fp`: Content fingerprint (hash, ETag, etc.) for efficient change detection, or `None` if not available.
+
+#### Key Type (`DataKeyType`) and Value Type (`DataValueType`) for Data
+
+The data type of custom source is a [*KTable*](/docs/core/data_types#ktable), so the key type and value type also follows the same requirements as key and value of a *KTable*. Specifically:
+
+* The key type uniquely identifies each data item in your source. It can be:
+  * A `NamedTuple` or frozen dataclass with one or multiple fields.
+
+    ```python
+    class DataKeyType(NamedTuple):
+        id: str
+        category: str | None = None
+    ```
+
+  * When there's a single key part, you can also use a simple type like `str` or `int` for single-field keys.
+    The key field name will be `cocoindex.KEY_FIELD_NAME` for this case.
+
+* The value type represents the actual data content.
+  It's typically a dataclass containing all the value fields the source exposes.
+
+  ```python
+  @dataclasses.dataclass
+  class DataValueType:
+      title: str
+      content: str
+      author: str | None
+      created_at: datetime | None
+  ```
+
+## Example
+
+See the [custom_source_hn](https://github.com/cocoindex-io/cocoindex/blob/main/examples/custom_source_hn/main.py) example for a complete end-to-end implementation.

{cocoindex-0.2.20 → cocoindex-0.2.21}/docs/sidebars.ts

@@ -70,6 +70,7 @@ const sidebars: SidebarsConfig = {
       label: 'Custom Operations',
       collapsed: false,
       items: [
+        'custom_ops/custom_sources',
         'custom_ops/custom_functions',
         'custom_ops/custom_targets',
       ],

cocoindex-0.2.21/examples/custom_source_hn/.env.example

@@ -0,0 +1,6 @@
+# Example environment variables for HackerNews Custom Source Example
+# Copy this to .env and fill in your actual database connection string
+
+# PostgreSQL database connection URL
+# Format: postgresql://username:password@host:port/database
+COCOINDEX_DATABASE_URL=postgresql://username:password@localhost:5432/cocoindex_hackernews

cocoindex-0.2.21/examples/custom_source_hn/README.md

@@ -0,0 +1,52 @@
+# HackerNews Custom Source Example
+
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+In this example, we use [CocoIndex Custom Source](https://cocoindex.io/docs/custom_ops/custom_targets) to define a source to get HackerNews recent content, by calling [HackerNews API](https://hn.algolia.com/api).
+We build index for HackerNews threads and their comments, and provides a lightweight query handler to search by keywords.
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+## Steps
+
+### Indexing Flow
+
+1. We define a custom source connector `HackerNews` to get HackerNews recent threads by calling HackerNews API.
+2. We build index for HackerNews threads and their comments.
+
+## Prerequisite
+
+[Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+## Run
+
+Install dependencies:
+
+```bash
+pip install -e .
+```
+
+Update the target:
+
+```bash
+cocoindex update --setup main
+```
+
+Each time when you run the `update` command, cocoindex will only re-process threads that have changed, and keep the target in sync with the recent 500 threads from HackerNews.
+
+You can also run `update` command in live mode, which will keep the target in sync with the source continuously:
+
+```bash
+cocoindex update --setup -L main.py
+```
+
+## CocoInsight
+
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline.
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
+
+```
+cocoindex server -ci -L main
+```
+
+Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).