cocoindex 0.1.73__tar.gz → 0.1.75__tar.gz

{cocoindex-0.1.73 → cocoindex-0.1.75}/.github/workflows/_doc_release.yml

@@ -11,7 +11,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 18
           cache: yarn
           cache-dependency-path: docs/yarn.lock
       - uses: webfactory/ssh-agent@v0.5.0

{cocoindex-0.1.73 → cocoindex-0.1.75}/.github/workflows/_test.yml

@@ -43,7 +43,7 @@ jobs:
         - name: Install Python toolchains
           run: |
             source .venv/bin/activate
-            pip install maturin pytest mypy
+            pip install maturin mypy pytest pytest-asyncio
         - name: Python build
           run: |
             source .venv/bin/activate

{cocoindex-0.1.73 → cocoindex-0.1.75}/.github/workflows/docs.yml

@@ -19,7 +19,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 18
           cache: yarn
           cache-dependency-path: docs/yarn.lock
       - name: Install dependencies

{cocoindex-0.1.73 → cocoindex-0.1.75}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.73"
+version = "0.1.75"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.73 → cocoindex-0.1.75}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.73"
+version = "0.1.75"
 edition = "2024"
 rust-version = "1.88"
 

{cocoindex-0.1.73 → cocoindex-0.1.75}/PKG-INFO

@@ -1,24 +1,25 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.73
+Version: 0.1.75
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: watchfiles>=1.1.0
 Requires-Dist: numpy>=1.23.2
 Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: pytest-asyncio ; extra == 'dev'
 Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: mypy ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
-Requires-Dist: cocoindex[embeddings] ; extra == 'all'
+Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'all'
 Provides-Extra: dev
 Provides-Extra: embeddings
 Provides-Extra: all
 License-File: LICENSE
 Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
 Author-email: CocoIndex <cocoindex.io@gmail.com>
-License: Apache-2.0
+License-Expression: Apache-2.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Homepage, https://cocoindex.io/
@@ -210,6 +211,7 @@ It defines an index flow like this:
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
+| [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
 
 More coming and stay tuned 👀!
 

{cocoindex-0.1.73 → cocoindex-0.1.75}/README.md

@@ -185,6 +185,7 @@ It defines an index flow like this:
 | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
 | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
 | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
+| [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
 
 More coming and stay tuned 👀!
 

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/docs/core/data_types.mdx

@@ -19,7 +19,7 @@ You don't need to spell out any data type explicitly when you define the flow.
 All you need to do is to make sure the data passed to functions and targets are compatible with them.
 
 Each type in CocoIndex type system is mapped to one or multiple types in Python.
-When you define a [custom function](/docs/core/custom_function), you need to annotate the data types of arguments and return values.
+When you define a [custom function](/docs/custom_ops/custom_functions), you need to annotate the data types of arguments and return values.
 
 *   When you pass a Python value to the engine (e.g. return values of a custom function), a specific type annotation is required.
     The type annotation needs to be specific in describing the target data type, as it provides the ground truth of the data type in the flow.

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/docs/core/flow_def.mdx

@@ -33,23 +33,23 @@ It takes two arguments:
 *   `flow_builder`: a `FlowBuilder` object to help build the flow.
 *   `data_scope`: a `DataScope` object, representing the top-level data scope. Any data created by the flow should be added to it.
 
-Alternatively, for more flexibility (e.g. you want to do this conditionally or generate dynamic name), you can explicitly call the `cocoindex.add_flow_def()` method:
+Alternatively, for more flexibility (e.g. you want to do this conditionally or generate dynamic name), you can explicitly call the `cocoindex.open_flow()` method:
 
 ```python
 def demo_flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
     ...
 
 # Add the flow definition to the flow registry.
-demo_flow = cocoindex.add_flow_def("DemoFlow", demo_flow_def)
+demo_flow = cocoindex.open_flow("DemoFlow", demo_flow_def)
 ```
 
 In both cases, `demo_flow` will be an object with `cocoindex.Flow` class type.
 See [Flow Running](/docs/core/flow_methods) for more details on it.
 
-Sometimes you no longer want to keep states of the flow in memory. We provide a `cocoindex.remove_flow()` method for this purpose:
+Sometimes you no longer want to keep states of the flow in memory. We provide a `close()` method for this purpose:
 
 ```python
-cocoindex.remove_flow(demo_flow)
+demo_flow.close()
 ```
 
 After it's called, `demo_flow` becomes an invalid object, and you should not call any methods of it.

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/docs/core/flow_methods.mdx

@@ -39,7 +39,7 @@ It creates a `demo_flow` object in `cocoindex.Flow` type.
 For a flow, its persistent backends need to be ready before it can run, including:
 
 *   [Internal storage](/docs/core/basics#internal-storage) for CocoIndex.
-*   Backend entities for targets exported by the flow, e.g. a table (in relational databases), a collection (in some vector databases), etc.
+*   Backend resources for targets exported by the flow, e.g. a table (in relational databases), a collection (in some vector databases), etc.
 
 The desired state of the backends for a flow is derived based on the flow definition itself.
 CocoIndex supports two types of actions to manage the persistent backends automatically:
@@ -104,7 +104,7 @@ cocoindex.drop_all_flows(report_to_stdout=True)
 
 After dropping the flow, the in-memory `cocoindex.Flow` instance is still valid, and you can call setup methods on it again.
 
-If you want to remove the flow from the current process, you can call `cocoindex.remove_flow(demo_flow)` to do so (see [related doc](/docs/core/flow_def#entry-point)).
+If you want to remove the flow from the current process, you can call `demo_flow.close()` to do so (see [related doc](/docs/core/flow_def#entry-point)).
 
 :::
 

cocoindex-0.1.73/docs/docs/core/custom_function.mdx → cocoindex-0.1.75/docs/docs/custom_ops/custom_functions.mdx

@@ -1,6 +1,6 @@
 ---
 title: Custom Functions
-description: Build Custom Functions
+description: Build powerful custom functions in CocoIndex for data transformation and processing. Create standalone functions or advanced function specs with executors, including caching, GPU support, and configurable behavior for scalable data operations.
 ---
 
 import Tabs from '@theme/Tabs';

cocoindex-0.1.75/docs/docs/custom_ops/custom_targets.mdx

@@ -0,0 +1,178 @@
+---
+title: Custom Targets
+description: Learn how to create custom targets in CocoIndex to export data to any destination including databases, cloud storage, file systems, and APIs. Build target specs and connectors with setup and data methods for flexible data export operations.
+toc_max_heading_level: 4
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+A custom target allows you to export data to any destination you want, such as databases, cloud storage, file systems, APIs, or other external systems.
+
+Custom targets are defined by two components:
+
+*   A **target spec** that configures the behavior and connection parameters for the target.
+*   A **target connector** that handles the actual data export operations.
+
+## Target Spec
+
+The target spec defines the configuration parameters for your custom target. When you use this target in a flow (typically by calling [`export()`](/docs/core/flow_def#export)), you instantiate this target spec with specific parameter values.
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+A target spec is defined as a class that inherits from `cocoindex.op.TargetSpec`.
+
+```python
+class CustomTarget(cocoindex.op.TargetSpec):
+    """
+    Documentation for the target.
+    """
+    param1: str
+    param2: int | None = None
+    ...
+```
+
+Notes:
+*   All fields of the spec must have a type serializable / deserializable by the `json` module.
+*   All subclasses of `TargetSpec` can be instantiated similar to a dataclass, i.e. `ClassName(param1=value1, param2=value2, ...)`.
+
+</TabItem>
+</Tabs>
+
+## Target Connector
+
+A target connector handles the actual data export operations for your custom target. It defines how data should be written to your target destination.
+
+Target connectors implement two categories of methods: **setup methods** for managing target infrastructure (similar to DDL operations in databases), and **data methods** for handling specific data operations (similar to DML operations).
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+A target connector is defined as a class decorated by `@cocoindex.op.target_connector(spec_cls=CustomTarget)`.
+
+```python
+@cocoindex.op.target_connector(spec_cls=CustomTarget)
+class CustomTargetConnector:
+    # Setup methods
+    @staticmethod
+    def get_persistent_key(spec: CustomTarget, target_name: str) -> PersistentKey:
+        """Required. Return a persistent key that uniquely identifies this target instance."""
+        ...
+
+    @staticmethod
+    def apply_setup_change(
+        key: PersistentKey, previous: CustomTarget | None, current: CustomTarget | None
+    ) -> None:
+        """Required. Apply setup changes to the target."""
+        ...
+
+    @staticmethod
+    def describe(key: PersistentKey) -> str:
+        """Optional. Return a human-readable description of the target."""
+        ...
+
+    # Data methods
+    @staticmethod
+    def prepare(spec: CustomTarget) -> PreparedCustomTarget:
+        """Optional. Prepare for execution before applying mutations."""
+        ...
+
+    @staticmethod
+    def mutate(
+        *all_mutations: tuple[PreparedCustomTarget, dict[DataKeyType, DataValueType | None]],
+    ) -> None:
+        """Required. Apply data mutations to the target."""
+        ...
+```
+
+</TabItem>
+</Tabs>
+
+The following data types are involved in the method definitions above:  `CustomTarget`, `PersistentKey`, `PreparedCustomTarget`, `DataKeyType`, `DataValueType`. They should be replaced with the actual types in your implementation. We will explain each of them below.
+
+### Setup Methods
+Setup methods manage the target infrastructure - creating, configuring, and cleaning up target resources.
+
+#### `get_persistent_key(spec, target_name) -> PersistentKey` (Required)
+
+This method returns a unique identifier for the target instance. This key is used by CocoIndex to keep track of target state and drive target spec changes.
+
+The key should be stable across different runs. If a previously existing key no longer exists, CocoIndex will assume the target is gone, and will drop it by calling `apply_setup_change` with `current` set to `None`.
+
+The return type of this method should be serializable by the `json` module. It will be passed to other setup methods.
+
+#### `apply_setup_change(key, previous, current) -> None` (Required)
+
+This method is called when the target configuration changes. It receives:
+- `key`: The persistent key for this target
+- `previous`: The previous target spec (or `None` if this is a new target)
+- `current`: The current target spec (or `None` if the target is being removed)
+
+This method should be implemented to:
+- Create resources when a target is first added (`previous` is `None`)
+- Update configuration when a target spec changes
+- Clean up resources when a target is removed (`current` is `None`)
+
+#### `describe(key) -> str` (Optional)
+
+Returns a human-readable description of the target for logging and debugging purposes.
+
+
+### Data Methods
+
+Data methods handle the actual data operations - inserting, updating, and deleting records in the target.
+
+#### `mutate(*all_mutations) -> None` (Required)
+
+This method applies data changes to the target. It receives multiple mutation batches, where each batch is a tuple containing:
+
+- The target spec (`PreparedCustomTarget`, or `CustomTarget` if `prepare` is not provided).
+
+- A dictionary of mutations (`dict[DataKeyType, DataValueType | None]`).
+  Each entry represents a mutation for a single row. When the value is `None`, it represents a deletion for the row, otherwise it's an upsert.
+
+  It represented in the same way as [*KTable*](/docs/core/data_types#ktable), except the value can be `None`.
+  In particular:
+
+  - Since both `DataKeyType` and `DataValueType` can have multiple columns, they're [*Struct*](/docs/core/data_types#struct-types).
+    - `DataKeyType` can be represented by a frozen dataclass (i.e. `@dataclass(frozen=True)`) or a `NamedTuple`, as it needs to be immutable.
+    - `DataValueType` can be represented by a `dataclass`, a `NamedTuple` or a `dict[str, Any]`.
+
+  - For simplicity, when there're a single primary key column with basic type, we allow using type of this column (e.g. `str`, `int` etc.) as the key type, and a wrapper *Struct* type can be omitted.
+  You can still use a `@dataclass(frozen=True)` or a `NamedTuple` to represent the key for this case though, if you want to handle both cases consistently.
+
+#### `prepare(spec) -> PreparedCustomTarget` (Optional)
+
+Prepares for execution by performing common operations before applying mutations. The returned value will be passed as the first element of tuples in the `mutate` method instead of the original spec.
+
+```python
+@staticmethod
+def prepare(spec: CustomTarget) -> PreparedCustomTarget:
+    """
+    Prepare for execution. Called once before mutations.
+    """
+    # Initialize connections, validate configuration, etc.
+    return PreparedCustomTarget(...)
+```
+
+If not provided, the original spec will be passed directly to `mutate`.
+
+## Best Practices
+
+### Idempotency of Methods with Side Effects
+
+`apply_setup_change()` and `mutate()` are the two methods that are expected to produce side effects.
+We expect them to be idempotent, i.e. when calling them with the same arguments multiple times, the effect should remain the same.
+
+For example,
+- For `apply_setup_change()`, if the target is a directory, it should be a no-op if we try to create it (`previous` is `None`) when the directory already exists, and also a no-op if we try to delete it (`current` is `None`) when the directory does not exist.
+- For `mutate()`, if a mutation is a deletion, it should be a no-op if the row does not exist.
+
+This is to make sure when the system if left in an intermediate state, e.g. interrupted in the middle between a change is made and CocoIndex notes down the change is completed, the targets can still be gracefully rolled forward to the desired states after the system is resumed.
+
+## Examples
+
+The cocoindex repository contains the following examples of custom targets:
+
+*   In the [custom_output_files](https://github.com/cocoindex-io/cocoindex/blob/main/examples/custom_output_files/main.py) example, `LocalFileTarget` exports data to local HTML files.

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/docusaurus.config.ts

@@ -57,6 +57,10 @@ const config: Config = {
             from: '/core/initialization',
             to: '/core/settings',
           },
+          {
+            from: '/core/custom_function',
+            to: '/custom_ops/custom_functions',
+          },
           {
             from: '/ops/storages',
             to: '/ops/targets',

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/sidebars.ts

@@ -31,7 +31,6 @@ const sidebars: SidebarsConfig = {
         'core/settings',
         'core/flow_methods',
         'core/cli',
-        'core/custom_function',
       ],
     },
     {
@@ -44,6 +43,15 @@ const sidebars: SidebarsConfig = {
         'ops/targets',
       ],
     },
+    {
+      type: 'category',
+      label: 'Custom Operations',
+      collapsed: false,
+      items: [
+        'custom_ops/custom_functions',
+        'custom_ops/custom_targets',
+      ],
+    },
     {
       type: 'category',
       label: 'AI Support',

{cocoindex-0.1.73 → cocoindex-0.1.75}/docs/src/css/custom.css

@@ -61,6 +61,13 @@
   --theme-color-keyline: #374151;
 }
 
+
+.table-of-contents {
+  code {
+    border: none;
+  }
+}
+
 .markdown {
   line-height: 150%;
 
@@ -92,13 +99,15 @@
 
 .navbar__title {
   font-family: 'Questrial', sans-serif;
-  font-size: 1.125rem; /* 18px */
+  font-size: 1.125rem;
+  /* 18px */
   color: var(--my-color-text-black);
 }
 
 .navbar {
   box-shadow: none;
-  font-size: 0.875rem; /* 14px */
+  font-size: 0.875rem;
+  /* 14px */
   border-bottom: 1px solid var(--ifm-color-emphasis-200);
 }
 
@@ -145,11 +154,13 @@
 }
 
 .theme-doc-sidebar-menu {
-  font-size: 0.875rem; /* 14px */
+  font-size: 0.875rem;
+  /* 14px */
 }
 
 .table-of-contents {
-  font-size: 0.8125rem; /* 13px */
+  font-size: 0.8125rem;
+  /* 13px */
 }
 
 .breadcrumbs {
@@ -161,6 +172,7 @@
   .breadcrumbs__item:first-child {
     display: none;
   }
+
   .breadcrumbs__link {
     padding: 0;
     background: none;
@@ -173,7 +185,7 @@
   background: var(--ifm-menu-link-sublist-icon) 50% / 1rem 1rem;
 }
 
-.navbar__item.navbar-github-link{
+.navbar__item.navbar-github-link {
   display: block;
   width: 120px;
 }