cocoindex 0.1.71__tar.gz → 0.1.72__tar.gz

{cocoindex-0.1.71 → cocoindex-0.1.72}/Cargo.lock

@@ -1297,7 +1297,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.71"
+version = "0.1.72"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.71 → cocoindex-0.1.72}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.71"
+version = "0.1.72"
 edition = "2024"
 rust-version = "1.88"
 

{cocoindex-0.1.71 → cocoindex-0.1.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.71
+Version: 0.1.72
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.1.71 → cocoindex-0.1.72}/docs/docs/core/data_types.mdx

@@ -21,12 +21,13 @@ All you need to do is to make sure the data passed to functions and targets are
 Each type in CocoIndex type system is mapped to one or multiple types in Python.
 When you define a [custom function](/docs/core/custom_function), you need to annotate the data types of arguments and return values.
 
-*   When you pass a Python value to the engine (e.g. return values of a custom function), type annotation is required,
-    as it provides the ground truth of the data type in the flow.
+*   When you pass a Python value to the engine (e.g. return values of a custom function), a specific type annotation is required.
+    The type annotation needs to be specific in describing the target data type, as it provides the ground truth of the data type in the flow.
 
 *   When you use a Python variable to bind to an engine value (e.g. arguments of a custom function),
-    we use the type annotation as a guidance to construct the Python value.
-    Type annotation is optional for basic types and struct types, and required for table types.
+    the engine already knows the specific data type, so we don't require a specific type annotation, e.g. type annotations can be omitted, or you can use `Any` at any level.
+    When a specific type annotation is provided, it's still used as a guidance to construct the Python value with compatible type.
+    Otherwise, we will bind to a default Python type.
 
 ### Basic Types
 
@@ -54,7 +55,7 @@ This is the list of all primitive types supported by CocoIndex:
 Notes:
 
 *   For some CocoIndex types, we support multiple Python types. You can annotate with any of these Python types.
-    The first one is the default type, i.e. CocoIndex will create a value with this type when the type annotation is not provided (e.g. for arguments of a custom function).
+    The first one is the default type, i.e. CocoIndex will create a value with this type when a specific type annotation is not provided (e.g. for arguments of a custom function).
 
 *   All Python types starting with `cocoindex.` are type aliases exported by CocoIndex. They're annotated types based on certain Python types:
 
@@ -136,7 +137,7 @@ Both `Person` and `PersonTuple` are valid Struct types in CocoIndex, with identi
 Choose `dataclass` for mutable objects or when you need additional methods, and `NamedTuple` for immutable, lightweight structures.
 
 Besides, for arguments of custom functions, CocoIndex also supports using dictionaries (`dict[str, Any]`) to represent a *Struct* type.
-It's the default Python type if you don't annotate the function argument.
+It's the default Python type if you don't annotate the function argument with a specific type.
 
 ### Table Types
 
@@ -152,11 +153,16 @@ The row order of a *KTable* is not preserved.
 Type of the first column (key column) must be a [key type](#key-types).
 
 In Python, a *KTable* type is represented by `dict[K, V]`. 
-The `V` should be a *Struct* type, either a `dataclass` or `NamedTuple`, representing the value fields of each row.
+The `K` should be the type binding to a key type,
+and the `V` should be the type binding to a *Struct* type representing the value fields of each row.
+When the specific type annotation is not provided,
+the key type is bound to a tuple with its key parts when it's a *Struct* type, the value type is bound to `dict[str, Any]`.
+
+
 For example, you can use `dict[str, Person]` or `dict[str, PersonTuple]` to represent a *KTable*, with 4 columns: key (*Str*), `first_name` (*Str*), `last_name` (*Str*), `dob` (*Date*).
+It's bound to `dict[str, dict[str, Any]]` if you don't annotate the function argument with a specific type.
 
 Note that if you want to use a *Struct* as the key, you need to ensure its value in Python is immutable. For `dataclass`, annotate it with `@dataclass(frozen=True)`. For `NamedTuple`, immutability is built-in. For example:
-For example:
 
 ```python
 @dataclass(frozen=True)
@@ -170,14 +176,16 @@ class PersonKeyTuple(NamedTuple):
 ```
 
 Then you can use `dict[PersonKey, Person]` or `dict[PersonKeyTuple, PersonTuple]` to represent a KTable keyed by `PersonKey` or `PersonKeyTuple`.
+It's bound to `dict[(str, str), dict[str, Any]]` if you don't annotate the function argument with a specific type.
 
 
 #### LTable
 
 *LTable* is a *Table* type whose row order is preserved. *LTable* has no key column.
 
-In Python, a *LTable* type is represented by `list[R]`, where `R` is a dataclass representing a row.
+In Python, a *LTable* type is represented by `list[R]`, where `R` is the type binding to the *Struct* type representing the value fields of each row.
 For example, you can use `list[Person]` to represent a *LTable* with 3 columns: `first_name` (*Str*), `last_name` (*Str*), `dob` (*Date*).
+It's bound to `list[dict[str, Any]]` if you don't annotate the function argument with a specific type.
 
 ## Key Types
 

{cocoindex-0.1.71 → cocoindex-0.1.72}/docs/docs/ops/targets.md

@@ -32,6 +32,13 @@ Here's how CocoIndex data elements map to Postgres elements during export:
 For example, if you have a data collector that collects rows with fields `id`, `title`, and `embedding`, it will be exported to a Postgres table with corresponding columns.
 It should be a unique table, meaning that no other export target should export to the same table.
 
+:::warning vector type mapping to Postgres
+
+Since vectors in pgvector must have fixed dimension, we only map vectors of number types with fixed dimension (i.e. *Vector[cocoindex.Float32, N]*, *Vector[cocoindex.Float64, N]*, and *Vector[cocoindex.Int64, N]*) to `vector(N)` columns.
+For all other vector types, we map them to `jsonb` columns.
+
+:::
+
 #### Spec
 
 The spec takes the following fields:
@@ -58,6 +65,13 @@ Here's how CocoIndex data elements map to Qdrant elements during export:
 
 *Vector[Float32, N]*, *Vector[Float64, N]* and *Vector[Int64, N]* types fit into Qdrant vector.
 
+:::warning vector type mapping to Qdrant
+
+Since vectors in Qdrant must have fixed dimension, we only map vectors of number types with fixed dimension (i.e. *Vector[cocoindex.Float32, N]*, *Vector[cocoindex.Float64, N]*, and *Vector[cocoindex.Int64, N]*) to Qdrant vectors.
+For all other vector types, we map to Qdrant payload as JSON arrays.
+
+:::
+
 #### Spec
 
 The spec takes the following fields:

cocoindex-0.1.72/docs/docs/tutorials/live_updates.md

@@ -0,0 +1,156 @@
+---
+title: Live Updates
+description: "Keep your indexes up-to-date with live updates in CocoIndex."
+---
+
+# Live Updates
+
+CocoIndex is designed to keep your indexes synchronized with your data sources. This is achieved through a feature called **live updates**, which automatically detects changes in your sources and updates your indexes accordingly. This ensures that your search results and data analysis are always based on the most current information.
+
+## How Live Updates Work
+
+Live updates in CocoIndex can be triggered in two main ways:
+
+1.  **Refresh Interval:** You can configure a `refresh_interval` for any data source. CocoIndex will then periodically check the source for any new, updated, or deleted data. This is a simple and effective way to keep your index fresh, especially for sources that don't have a built-in change notification system.
+
+2.  **Change Capture Mechanisms:** Some data sources offer more sophisticated ways to track changes. For example:
+    *   **Amazon S3:** You can configure an SQS queue to receive notifications whenever a file is added, modified, or deleted in your S3 bucket. CocoIndex can listen to this queue and trigger an update instantly.
+    *   **Google Drive:** The Google Drive source can be configured to poll for recent changes, which is more efficient than a full refresh.
+
+When a change is detected, CocoIndex performs an **incremental update**. This means it only re-processes the data that has been affected by the change, without having to re-index your entire dataset. This makes the update process fast and efficient.
+
+Here's an example of how to set up a source with a `refresh_interval`:
+
+```python
+@cocoindex.flow_def(name="LiveUpdateExample")
+def live_update_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    # Source: local files in the 'data' directory
+    data_scope["documents"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="data"),
+        refresh_interval=cocoindex.timedelta(seconds=5),
+    )
+    # ...
+```
+
+By setting `refresh_interval` to 5 seconds, we're telling CocoIndex to check for changes in the `data` directory every 5 seconds.
+
+## Implementing Live Updates
+
+You can enable live updates using either the CocoIndex CLI or the Python library.
+
+### Using the CLI
+
+To start a live update process from the command line, use the `update` command with the `-L` or `--live` flag:
+
+```bash
+cocoindex update -L your_flow_definition_file.py
+```
+
+This will start a long-running process that continuously monitors your data sources for changes and updates your indexes in real-time. You can stop the process by pressing `Ctrl+C`.
+
+### Using the Python Library
+
+For more control over the live update process, you can use the `FlowLiveUpdater` class in your Python code. This is particularly useful when you want to integrate CocoIndex into a larger application.
+
+The `FlowLiveUpdater` can be used as a context manager, which automatically starts the updater when you enter the `with` block and stops it when you exit. The `wait()` method will block until the updater is aborted (e.g., by pressing `Ctrl+C`).
+
+Here's how you can use `FlowLiveUpdater` to start and manage a live update process:
+
+```python
+import cocoindex
+
+# Create a FlowLiveUpdater instance
+with cocoindex.FlowLiveUpdater(live_update_flow, cocoindex.FlowLiveUpdaterOptions(print_stats=True)) as updater:
+    print("Live updater started. Press Ctrl+C to stop.")
+    # The updater runs in the background.
+    # The wait() method blocks until the updater is stopped.
+    updater.wait()
+
+print("Live updater stopped.")
+```
+
+#### Getting Status Updates
+
+You can also get status updates from the `FlowLiveUpdater` to monitor the update process. The `next_status_updates()` method blocks until there is a new status update.
+
+```python
+import cocoindex
+
+updater = cocoindex.FlowLiveUpdater(live_update_flow)
+updater.start()
+
+while True:
+    updates = updater.next_status_updates()
+
+    if not updates.active_sources:
+        print("All sources have finished processing.")
+        break
+
+    for source_name in updates.updated_sources:
+        print(f"Source '{source_name}' has been updated.")
+
+updater.wait()
+```
+
+This allows you to react to updates in your application, for example, by notifying users or triggering downstream processes.
+
+## Example
+
+Let's walk through an example of how to set up a live update flow. For the complete, runnable code, see the [live updates example](https://github.com/cocoindex-io/cocoindex/tree/main/examples/live_updates) in the CocoIndex repository.
+
+### 1. Setting up the Source
+
+The first step is to define a source and configure a `refresh_interval`. In this example, we'll use a `LocalFile` source to monitor a directory named `data`.
+
+```python
+@cocoindex.flow_def(name="LiveUpdateExample")
+def live_update_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    # Source: local files in the 'data' directory
+    data_scope["documents"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="data"),
+        refresh_interval=cocoindex.timedelta(seconds=5),
+    )
+
+    # Collector
+    collector = data_scope.add_collector()
+    with data_scope["documents"].row() as doc:
+        collector.collect(filename=doc["filename"], content=doc["content"])
+
+    # Target: Postgres database
+    collector.export(
+        "documents_index",
+        cocoindex.targets.Postgres(),
+        primary_key_fields=["filename"]
+    )
+```
+
+By setting `refresh_interval` to 5 seconds, we're telling CocoIndex to check for changes in the `data` directory every 5 seconds.
+
+### 2. Running the Live Updater
+
+Once the flow is defined, you can use the `FlowLiveUpdater` to start the live update process.
+
+```python
+def main():
+    # Initialize CocoIndex
+    cocoindex.init()
+
+    # Setup the flow
+    live_update_flow.setup(report_to_stdout=True)
+
+    # Start the live updater
+    with cocoindex.FlowLiveUpdater(live_update_flow, cocoindex.FlowLiveUpdaterOptions(print_stats=True)) as updater:
+        print("Live updater started. Watching for changes in the 'data' directory.")
+        updater.wait()
+
+if __name__ == "__main__":
+    main()
+```
+
+The `FlowLiveUpdater` will run in the background, and the `updater.wait()` call will block until the process is stopped.
+
+## Conclusion
+
+Live updates is a powerful feature of CocoIndex that ensures your indexes are always fresh. By using a combination of refresh intervals and source-specific change capture mechanisms, you can build responsive, real-time applications that are always in sync with your data.
+
+For more detailed information on the `FlowLiveUpdater` and other live update options, please refer to the [Run a Flow documentation](https://cocoindex.io/docs/core/flow_methods#live-update).

{cocoindex-0.1.71 → cocoindex-0.1.72}/docs/sidebars.ts

@@ -12,6 +12,14 @@ const sidebars: SidebarsConfig = {
         'getting_started/installation',
       ],
     },
+    {
+      type: 'category',
+      label: 'Tutorials',
+      collapsed: false,
+      items: [
+        'tutorials/live_updates',
+      ],
+    },
     {
       type: 'category',
       label: 'CocoIndex Core',

{cocoindex-0.1.71 → cocoindex-0.1.72}/examples/face_recognition/main.py

@@ -11,10 +11,10 @@ import numpy as np
 
 @dataclasses.dataclass
 class ImageRect:
-    top: int
-    left: int
-    bottom: int
-    right: int
+    min_x: int
+    min_y: int
+    max_x: int
+    max_y: int
 
 
 @dataclasses.dataclass
@@ -28,7 +28,12 @@ class FaceBase:
 MAX_IMAGE_WIDTH = 1280
 
 
-@cocoindex.op.function(cache=True, behavior_version=1, gpu=True)
+@cocoindex.op.function(
+    cache=True,
+    behavior_version=1,
+    gpu=True,
+    arg_relationship=(cocoindex.op.ArgRelationship.RECTS_BASE_IMAGE, "content"),
+)
 def extract_faces(content: bytes) -> list[FaceBase]:
     """Extract the first pages of a PDF."""
     orig_img = Image.open(io.BytesIO(content)).convert("RGB")
@@ -48,17 +53,17 @@ def extract_faces(content: bytes) -> list[FaceBase]:
     locs = face_recognition.face_locations(np.array(img), model="cnn")
 
     faces: list[FaceBase] = []
-    for top, right, bottom, left in locs:
+    for min_y, max_x, max_y, min_x in locs:
         rect = ImageRect(
-            left=int(left * ratio),
-            top=int(top * ratio),
-            right=int(right * ratio),
-            bottom=int(bottom * ratio),
+            min_x=int(min_x * ratio),
+            min_y=int(min_y * ratio),
+            max_x=int(max_x * ratio),
+            max_y=int(max_y * ratio),
         )
 
         # Crop the face and save it as a PNG.
         buf = io.BytesIO()
-        orig_img.crop((rect.left, rect.top, rect.right, rect.bottom)).save(
+        orig_img.crop((rect.min_x, rect.min_y, rect.max_x, rect.max_y)).save(
             buf, format="PNG"
         )
         face = buf.getvalue()
@@ -70,7 +75,7 @@ def extract_faces(content: bytes) -> list[FaceBase]:
 @cocoindex.op.function(cache=True, behavior_version=1, gpu=True)
 def extract_face_embedding(
     face: bytes,
-) -> cocoindex.Vector[cocoindex.Float32, typing.Literal[128]]:
+) -> cocoindex.Vector[cocoindex.Float32]:
     """Extract the embedding of a face."""
     img = Image.open(io.BytesIO(face)).convert("RGB")
     embedding = face_recognition.face_encodings(

{cocoindex-0.1.71 → cocoindex-0.1.72}/examples/face_recognition/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Build index for papers with both metadata and content embeddings"
 requires-python = ">=3.11"
 dependencies = [
-    "cocoindex>=0.1.67",
+    "cocoindex>=0.1.71",
     "face-recognition>=1.3.0",
     "pillow>=10.0.0",
     "numpy>=1.26.0",

cocoindex-0.1.72/examples/live_updates/.env

@@ -0,0 +1 @@
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.72/examples/live_updates/README.md

@@ -0,0 +1,58 @@
+# Applying Live Updates to CocoIndex Flow Example
+[![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+We appreciate a star ⭐ at [CocoIndex Github](https://github.com/cocoindex-io/cocoindex) if this is helpful.
+
+This example demonstrates how to use CocoIndex's live update feature to keep an index synchronized with a local directory.
+
+## How it Works
+
+The `main.py` script defines a CocoIndex flow that:
+
+1.  **Sources** data from a local directory named `data`. It uses a `refresh_interval` of 5 seconds to check for changes.
+2.  **Collects** the `filename` and `content` of each file.
+3.  **Exports** the collected data to a Postgres database table.
+
+The script then starts a `FlowLiveUpdater`, which runs in the background and continuously monitors the `data` directory for changes.
+
+## Running the Example
+
+1.  [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+2. **Install the dependencies:**
+
+    ```bash
+    pip install -e .
+    ```
+
+3.  **Run the example:**
+
+    You can run the live update example in two ways:
+
+    **Option 1: Using the Python script**
+
+    This method uses CocoIndex [Library API](https://cocoindex.io/docs/core/flow_methods#library-api-2) to perform live updates.
+
+    ```bash
+    python main.py
+    ```
+
+    **Option 2: Using the CocoIndex CLI**
+
+    This method is useful for managing your indexes from the command line, through CocoIndex [CLI](https://cocoindex.io/docs/core/flow_methods#cli-2).
+
+    ```bash
+    cocoindex update main.py -L --setup
+    ```
+
+4.  **Test the live updates:**
+
+    While the script is running, you can try adding, modifying, or deleting files in the `data` directory. You will see the changes reflected in the logs as CocoIndex updates the index.
+
+## Cleaning Up
+
+To remove the database table created by this example, you can run:
+
+```bash
+cocoindex drop main.py
+```

cocoindex-0.1.72/examples/live_updates/data/bizarre_animals.md

@@ -0,0 +1,21 @@
+In the spirit of Project Zeta’s innovative chaos, here’s a collection of absurdly true facts about the weirdest animals you’ve never heard of:
+
+1. **Tardigrade (Water Bear)**: This microscopic beast can survive outer space, radiation, and being boiled alive. It once crashed a team meeting by stowing away in Bob’s coffee mug and demanding admin access to the server.
+
+2. **Aye-Aye**: A Madagascar primate with a creepy long finger it uses to tap trees for grubs. It tried to “debug” our codebase by tapping the keyboard, resulting in 47 nested for-loops.
+
+3. **Saiga Antelope**: This goofy-nosed critter looks like it’s auditioning for a sci-fi flick. Its sneezes are so powerful they once blew out the office Wi-Fi during a sprint review.
+
+4. **Glaucus Atlanticus (Blue Dragon Sea Slug)**: This tiny ocean dragon steals venom from jellyfish and uses it like a borrowed superpower. It infiltrated our water cooler and left behind a sparkly, toxic trail.
+
+5. **Pink Fairy Armadillo**: A palm-sized digger that looks like a cotton candy tank. It burrowed into the office carpet, mistaking it for a desert, and now we have a “no armadillos” policy.
+
+6. **Dumbo Octopus**: A deep-sea octopus with ear-like fins, flapping around like it’s late for a Zoom call. It once rewired our projector to display memes of itself across the office.
+
+7. **Jerboa**: A hopping desert rodent with kangaroo vibes. It stole the team’s snacks and leaped over three cubicles before anyone noticed, earning the codename "Snack Bandit."
+
+8. **Mantis Shrimp**: This crustacean sees more colors than our graphic designer and punches harder than a failing CI pipeline. It shattered a monitor when we tried to pair-program with it.
+
+9. **Okapi**: A zebra-giraffe hybrid that looks like a Photoshop error. It wandered into our sprint planning and suggested we pivot to a “forest-themed” microservices architecture.
+
+10. **Blobfish**: The ocean’s saddest-looking blob, voted “Most Likely to Crash a Stand-Up” by the team. Its mere presence caused our morale bot to send 200 crying emojis.

cocoindex-0.1.72/examples/live_updates/data/chunk_norris.md

@@ -0,0 +1,19 @@
+# Chuck Norris Project Facts
+Date: 2025-07-20
+Author: Anonymous (because Chuck Norris knows who you are)
+
+Here are some totally true facts about Chuck Norris's involvement in Project Omega:
+
+1. Chuck Norris doesn't write code; he stares at the computer until it writes itself out of fear.
+2. The project deadline was yesterday, but time rescheduled itself to accommodate Chuck Norris.
+3. Chuck Norris's code never has bugs—just "features" that are too scared to misbehave.
+4. When the database crashed, Chuck Norris roundhouse-kicked the server, and it apologized.
+5. The team tried to use Agile, but Chuck Norris declared, "I am the only methodology you need."
+6. Version control? Chuck Norris is the only version that matters.
+7. The project scope expanded because Chuck Norris added "world domination" as a deliverable.
+8. When the CI/CD pipeline failed, Chuck Norris rebuilt it with a single grunt.
+9. The codebase is 100% documented because no one dares ask Chuck Norris, "What does this do?"
+10. Chuck Norris doesn't deploy to production; production deploys to Chuck Norris.
+
+Last updated: 2025-07-20 06:36 AM MST
+Note: If you modify this file, Chuck Norris will know... and he’ll find you.

cocoindex-0.1.72/examples/live_updates/main.py

@@ -0,0 +1,55 @@
+import datetime
+
+import cocoindex
+from dotenv import load_dotenv
+
+
+# Define the flow
+@cocoindex.flow_def(name="LiveUpdates")
+def live_update_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+) -> None:
+    # Source: local files in the 'data' directory
+    data_scope["documents"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="data"),
+        refresh_interval=datetime.timedelta(seconds=5),
+    )
+
+    # Collector
+    collector = data_scope.add_collector()
+    with data_scope["documents"].row() as doc:
+        collector.collect(
+            filename=doc["filename"],
+            content=doc["content"],
+        )
+
+    # Target: Postgres database
+    collector.export(
+        "documents_index",
+        cocoindex.targets.Postgres(),
+        primary_key_fields=["filename"],
+    )
+
+
+def main() -> None:
+    # Setup the flow
+    live_update_flow.setup(report_to_stdout=True)
+
+    # Start the live updater
+    print("Starting live updater...")
+    with cocoindex.FlowLiveUpdater(
+        live_update_flow, cocoindex.FlowLiveUpdaterOptions(print_stats=True)
+    ) as updater:
+        print("Live updater started. Watching for changes in the 'data' directory.")
+        print("Try adding, modifying, or deleting files in the 'data' directory.")
+        print("Press Ctrl+C to stop.")
+        try:
+            updater.wait()
+        except KeyboardInterrupt:  # handle graceful shutdown
+            print("Stopping live updater...")
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    cocoindex.init()
+    main()

cocoindex-0.1.72/examples/live_updates/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "live-updates-example"
+version = "0.1.0"
+description = "Simple example for cocoindex: perform live updates based on local markdown files."
+requires-python = ">=3.11"
+dependencies = [
+    "cocoindex>=0.1.70",
+    "python-dotenv>=1.1.0",
+]
+
+[tools.setuptools]
+packages = []

{cocoindex-0.1.71 → cocoindex-0.1.72}/python/cocoindex/flow.py

@@ -10,6 +10,13 @@ import functools
 import inspect
 import re
 
+from .validation import (
+    validate_flow_name,
+    NamingError,
+    validate_full_flow_name,
+    validate_target_name,
+)
+
 from dataclasses import dataclass
 from enum import Enum
 from threading import Lock
@@ -300,6 +307,9 @@ class DataScope:
         )
 
     def __setitem__(self, field_name: str, value: DataSlice[T]) -> None:
+        from .validation import validate_field_name
+
+        validate_field_name(field_name)
         value._state.attach_to_scope(self._engine_data_scope, field_name)
 
     def __enter__(self) -> DataScope:
@@ -367,7 +377,7 @@ class DataCollector:
 
     def export(
         self,
-        name: str,
+        target_name: str,
         target_spec: op.TargetSpec,
         /,
         *,
@@ -381,6 +391,8 @@ class DataCollector:
 
         `vector_index` is for backward compatibility only. Please use `vector_indexes` instead.
         """
+
+        validate_target_name(target_name)
         if not isinstance(target_spec, op.TargetSpec):
             raise ValueError(
                 "export() can only be called on a CocoIndex target storage"
@@ -398,7 +410,7 @@ class DataCollector:
             vector_indexes=vector_indexes,
         )
         self._flow_builder_state.engine_flow_builder.export(
-            name,
+            target_name,
             _spec_kind(target_spec),
             dump_engine_object(target_spec),
             dump_engine_object(index_options),
@@ -660,6 +672,8 @@ class Flow:
     def __init__(
         self, name: str, full_name: str, engine_flow_creator: Callable[[], _engine.Flow]
     ):
+        validate_flow_name(name)
+        validate_full_flow_name(full_name)
         self._name = name
         self._full_name = full_name
         engine_flow = None
@@ -831,11 +845,6 @@ def get_flow_full_name(name: str) -> str:
 
 
 def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
-    """Add a flow definition to the cocoindex library."""
-    if not all(c.isalnum() or c == "_" for c in name):
-        raise ValueError(
-            f"Flow name '{name}' contains invalid characters. Only alphanumeric characters and underscores are allowed."
-        )
     with _flows_lock:
         if name in _flows:
             raise KeyError(f"Flow with name {name} already exists")

{cocoindex-0.1.71 → cocoindex-0.1.72}/python/cocoindex/setting.py

@@ -6,6 +6,7 @@ import os
 
 from typing import Callable, Self, Any, overload
 from dataclasses import dataclass
+from .validation import validate_app_namespace_name
 
 _app_namespace: str = ""
 
@@ -27,6 +28,8 @@ def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
 
 def set_app_namespace(app_namespace: str) -> None:
     """Set the application namespace."""
+    if app_namespace:
+        validate_app_namespace_name(app_namespace)
     global _app_namespace  # pylint: disable=global-statement
     _app_namespace = app_namespace