cocoindex 0.2.14__tar.gz → 0.2.16__tar.gz

{cocoindex-0.2.14 → cocoindex-0.2.16}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.14"
+version = "0.2.16"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.2.14 → cocoindex-0.2.16}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.14"
+version = "0.2.16"
 edition = "2024"
 rust-version = "1.89"
 license = "Apache-2.0"

{cocoindex-0.2.14 → cocoindex-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.14
+Version: 0.2.16
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent

{cocoindex-0.2.14 → cocoindex-0.2.16}/THIRD_PARTY_NOTICES.html

@@ -2428,7 +2428,7 @@ Software.
                 <h3 id="Apache-2.0">Apache License 2.0</h3>
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
-                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.14</a></li>
+                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.16</a></li>
                     <li><a href=" https://github.com/awesomized/crc-fast-rust ">crc-fast 1.3.0</a></li>
                     <li><a href=" https://github.com/qdrant/rust-client ">qdrant-client 1.15.0</a></li>
                 </ul>

{cocoindex-0.2.14 → cocoindex-0.2.16}/docs/docs/examples/examples/codebase_index.md

@@ -55,10 +55,12 @@ The flow is composed of the following steps:
 We will index the CocoIndex codebase. Here we use the `LocalFile` source to ingest files from the CocoIndex codebase root directory.
 
 ```python
+import os
+
 @cocoindex.flow_def(name="CodeEmbedding")
 def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
     data_scope["files"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="../..",
+        cocoindex.sources.LocalFile(path=os.path.join('..', '..'),
                                     included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx"],
                                     excluded_patterns=[".*", "target", "**/node_modules"]))
     code_embeddings = data_scope.add_collector()

{cocoindex-0.2.14 → cocoindex-0.2.16}/docs/docs/examples/examples/docs_to_knowledge_graph.md

@@ -54,10 +54,12 @@ and then build a knowledge graph.
 We will process CocoIndex documentation markdown files (`.md`, `.mdx`) from the `docs/core` directory ([markdown files](https://github.com/cocoindex-io/cocoindex/tree/main/docs/docs/core), [deployed docs](https://cocoindex.io/docs/core/basics)).
 
 ```python
+import os
+
 @cocoindex.flow_def(name="DocsToKG")
 def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="../../docs/docs/core",
+        cocoindex.sources.LocalFile(path=os.path.join('..', '..', 'docs', 'docs', 'core'),
                                     included_patterns=["*.md", "*.mdx"]))
 ```
 

{cocoindex-0.2.14 → cocoindex-0.2.16}/docs/docs/examples/examples/patient_form_extraction.md

@@ -50,6 +50,8 @@ Alternatively, we have native support for Gemini, Ollama, LiteLLM. You can choos
 Add source from local files.
 
 ```python
+import os
+
 @cocoindex.flow_def(name="PatientIntakeExtraction")
 def patient_intake_extraction_flow(
     flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
@@ -58,7 +60,7 @@ def patient_intake_extraction_flow(
     Define a flow that extracts patient information from intake forms.
     """
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="data/patient_forms", binary=True)
+        cocoindex.sources.LocalFile(path=os.path.join('data', 'patient_forms'), binary=True)
     )
 ```
 

{cocoindex-0.2.14 → cocoindex-0.2.16}/examples/code_embedding/main.py

@@ -45,7 +45,7 @@ def code_embedding_flow(
     """
     data_scope["files"] = flow_builder.add_source(
         cocoindex.sources.LocalFile(
-            path="../..",
+            path=os.path.join("..", ".."),
             included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx"],
             excluded_patterns=["**/.*", "target", "**/node_modules"],
         )

{cocoindex-0.2.14 → cocoindex-0.2.16}/examples/docs_to_knowledge_graph/main.py

@@ -4,6 +4,7 @@ This example shows how to extract relationships from documents and build a knowl
 
 import dataclasses
 import cocoindex
+import os
 
 neo4j_conn_spec = cocoindex.add_auth_entry(
     "Neo4jConnection",
@@ -66,7 +67,8 @@ def docs_to_kg_flow(
     """
     data_scope["documents"] = flow_builder.add_source(
         cocoindex.sources.LocalFile(
-            path="../../docs/docs/core", included_patterns=["*.md", "*.mdx"]
+            path=os.path.join("..", "..", "docs", "docs", "core"),
+            included_patterns=["*.md", "*.mdx"],
         )
     )
 

{cocoindex-0.2.14 → cocoindex-0.2.16}/examples/patient_intake_extraction/main.py

@@ -118,7 +118,9 @@ def patient_intake_extraction_flow(
     Define a flow that extracts patient information from intake forms.
     """
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="data/patient_forms", binary=True)
+        cocoindex.sources.LocalFile(
+            path=os.path.join("data", "patient_forms"), binary=True
+        )
     )
 
     patients_index = data_scope.add_collector()

{cocoindex-0.2.14 → cocoindex-0.2.16}/examples/text_embedding_lancedb/main.py

@@ -89,8 +89,6 @@ async def search(query: str) -> cocoindex.QueryOutput:
     search = await table.search(query_embedding, vector_column_name="text_embedding")
     search_results = await search.limit(5).to_list()
 
-    print(search_results)
-
     return cocoindex.QueryOutput(
         results=[
             {

{cocoindex-0.2.14 → cocoindex-0.2.16}/python/cocoindex/__init__.py

@@ -2,6 +2,7 @@
 Cocoindex is a framework for building and running indexing pipelines.
 """
 
+from . import _engine  # type: ignore
 from . import functions, sources, targets, cli, utils
 
 from . import targets as storages  # Deprecated: Use targets instead
@@ -42,6 +43,8 @@ from .typing import (
     Json,
 )
 
+_engine.init_pyo3_runtime()
+
 __all__ = [
     # Submodules
     "_engine",

{cocoindex-0.2.14 → cocoindex-0.2.16}/python/cocoindex/flow.py

@@ -17,7 +17,6 @@ from typing import (
     Callable,
     Generic,
     Iterable,
-    NamedTuple,
     Sequence,
     TypeVar,
     cast,
@@ -575,7 +574,8 @@ class FlowLiveUpdaterOptions:
     print_stats: bool = False
 
 
-class FlowUpdaterStatusUpdates(NamedTuple):
+@dataclass
+class FlowUpdaterStatusUpdates:
     """
     Status updates for a flow updater.
     """
@@ -1060,12 +1060,14 @@ def _get_data_slice_annotation_type(
 _transform_flow_name_builder = _NameBuilder()
 
 
-class TransformFlowInfo(NamedTuple):
+@dataclass
+class TransformFlowInfo(Generic[T]):
     engine_flow: _engine.TransientFlow
     result_decoder: Callable[[Any], T]
 
 
-class FlowArgInfo(NamedTuple):
+@dataclass
+class FlowArgInfo:
     name: str
     type_hint: Any
     encoder: Callable[[Any], Any]
@@ -1081,7 +1083,7 @@ class TransformFlow(Generic[T]):
     _args_info: list[FlowArgInfo]
 
     _lazy_lock: asyncio.Lock
-    _lazy_flow_info: TransformFlowInfo | None = None
+    _lazy_flow_info: TransformFlowInfo[T] | None = None
 
     def __init__(
         self,
@@ -1123,12 +1125,12 @@ class TransformFlow(Generic[T]):
         return self._flow_fn(*args, **kwargs)
 
     @property
-    def _flow_info(self) -> TransformFlowInfo:
+    def _flow_info(self) -> TransformFlowInfo[T]:
         if self._lazy_flow_info is not None:
             return self._lazy_flow_info
         return execution_context.run(self._flow_info_async())
 
-    async def _flow_info_async(self) -> TransformFlowInfo:
+    async def _flow_info_async(self) -> TransformFlowInfo[T]:
         if self._lazy_flow_info is not None:
             return self._lazy_flow_info
         async with self._lazy_lock:
@@ -1136,7 +1138,7 @@ class TransformFlow(Generic[T]):
                 self._lazy_flow_info = await self._build_flow_info_async()
             return self._lazy_flow_info
 
-    async def _build_flow_info_async(self) -> TransformFlowInfo:
+    async def _build_flow_info_async(self) -> TransformFlowInfo[T]:
         flow_builder_state = _FlowBuilderState(self._flow_name)
         kwargs: dict[str, DataSlice[T]] = {}
         for arg_info in self._args_info:

{cocoindex-0.2.14 → cocoindex-0.2.16}/python/cocoindex/functions.py

@@ -2,13 +2,13 @@
 
 import dataclasses
 import functools
-from typing import Annotated, Any, Literal
+from typing import Any, Literal
 
 import numpy as np
 from numpy.typing import NDArray
 
 from . import llm, op
-from .typing import TypeAttr, Vector
+from .typing import Vector
 
 
 class ParseJson(op.FunctionSpec):
@@ -40,6 +40,24 @@ class SplitRecursively(op.FunctionSpec):
     custom_languages: list[CustomLanguageSpec] = dataclasses.field(default_factory=list)
 
 
+class SplitBySeparators(op.FunctionSpec):
+    """
+    Split text by specified regex separators only.
+    Output schema matches SplitRecursively for drop-in compatibility:
+        KTable rows with fields: location (Range), text (Str), start, end.
+    Args:
+        separators_regex: list[str]  # e.g., [r"\\n\\n+"]
+        keep_separator: Literal["NONE", "LEFT", "RIGHT"] = "NONE"
+        include_empty: bool = False
+        trim: bool = True
+    """
+
+    separators_regex: list[str] = dataclasses.field(default_factory=list)
+    keep_separator: Literal["NONE", "LEFT", "RIGHT"] = "NONE"
+    include_empty: bool = False
+    trim: bool = True
+
+
 class EmbedText(op.FunctionSpec):
     """Embed a text into a vector space."""
 

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/builder/analyzer.rs

@@ -646,6 +646,7 @@ struct ExportDataFieldsInfo {
     primary_key_schema: Box<[FieldSchema]>,
     value_fields_idx: Vec<u32>,
     value_stable: bool,
+    output_value_fingerprinter: Fingerprinter,
 }
 
 impl AnalyzerContext {
@@ -866,6 +867,8 @@ impl AnalyzerContext {
                             .as_ref()
                             .map(|uuid_idx| pk_fields_idx.contains(uuid_idx))
                             .unwrap_or(false);
+                        let output_value_fingerprinter =
+                            Fingerprinter::default().with(&value_fields_schema)?;
                         (
                             value_fields_schema,
                             ExportDataFieldsInfo {
@@ -874,6 +877,7 @@ impl AnalyzerContext {
                                 primary_key_schema,
                                 value_fields_idx,
                                 value_stable,
+                                output_value_fingerprinter,
                             },
                         )
                     }
@@ -937,6 +941,7 @@ impl AnalyzerContext {
                         primary_key_schema: data_fields_info.primary_key_schema,
                         value_fields: data_fields_info.value_fields_idx,
                         value_stable: data_fields_info.value_stable,
+                        output_value_fingerprinter: data_fields_info.output_value_fingerprinter,
                     })
                 })
             })

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/builder/exec_ctx.rs

@@ -10,6 +10,7 @@ pub struct ImportOpExecutionContext {
 
 pub struct ExportOpExecutionContext {
     pub target_id: i32,
+    pub schema_version_id: usize,
 }
 
 pub struct FlowSetupExecutionContext {
@@ -107,12 +108,12 @@ fn build_import_op_exec_ctx(
     Ok(ImportOpExecutionContext { source_id })
 }
 
-fn build_target_id(
+fn build_export_op_exec_ctx(
     analyzed_target_ss: &AnalyzedTargetSetupState,
     existing_target_states: &HashMap<&setup::ResourceIdentifier, Vec<&setup::TargetSetupState>>,
     metadata: &mut setup::FlowSetupMetadata,
     target_states: &mut IndexMap<setup::ResourceIdentifier, setup::TargetSetupState>,
-) -> Result<i32> {
+) -> Result<ExportOpExecutionContext> {
     let target_factory = get_target_factory(&analyzed_target_ss.target_kind)?;
 
     let resource_id = setup::ResourceIdentifier {
@@ -121,7 +122,7 @@ fn build_target_id(
     };
     let existing_target_states = existing_target_states.get(&resource_id);
     let mut compatible_target_ids = HashSet::<Option<i32>>::new();
-    let mut reusable_schema_version_ids = HashSet::<Option<i32>>::new();
+    let mut reusable_schema_version_ids = HashSet::<Option<usize>>::new();
     for existing_state in existing_target_states.iter().flat_map(|v| v.iter()) {
         let compatibility = if let Some(key_type) = &analyzed_target_ss.key_type
             && let Some(existing_key_type) = &existing_state.common.key_type
@@ -196,7 +197,10 @@ fn build_target_id(
             });
         }
     }
-    Ok(target_id)
+    Ok(ExportOpExecutionContext {
+        target_id,
+        schema_version_id,
+    })
 }
 
 pub fn build_flow_setup_execution_context(
@@ -276,18 +280,17 @@ pub fn build_flow_setup_execution_context(
         .targets
         .iter()
         .map(|analyzed_target_ss| {
-            let target_id = build_target_id(
+            build_export_op_exec_ctx(
                 analyzed_target_ss,
                 &target_states_by_name_type,
                 &mut metadata,
                 &mut target_states,
-            )?;
-            Ok(ExportOpExecutionContext { target_id })
+            )
         })
         .collect::<Result<Vec<_>>>()?;
 
     for analyzed_target_ss in analyzed_ss.declarations.iter() {
-        build_target_id(
+        build_export_op_exec_ctx(
             analyzed_target_ss,
             &target_states_by_name_type,
             &mut metadata,

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/builder/plan.rs

@@ -111,6 +111,8 @@ pub struct AnalyzedExportOp {
     /// If true, value is never changed on the same primary key.
     /// This is guaranteed if the primary key contains auto-generated UUIDs.
     pub value_stable: bool,
+    /// Fingerprinter of the output value.
+    pub output_value_fingerprinter: Fingerprinter,
 }
 
 pub struct AnalyzedExportTargetOpGroup {

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/execution/row_indexer.rs

@@ -599,6 +599,10 @@ impl<'a> RowIndexer<'a> {
                 let mut keys_info = Vec::new();
                 let collected_values =
                     &data.evaluate_output.collected_values[export_op.input.collector_idx as usize];
+                let value_fingerprinter = export_op
+                    .output_value_fingerprinter
+                    .clone()
+                    .with(&export_op_exec_ctx.schema_version_id)?;
                 for value in collected_values.iter() {
                     let primary_key =
                         extract_primary_key_for_export(&export_op.primary_key_def, value)?;
@@ -629,7 +633,8 @@ impl<'a> RowIndexer<'a> {
 
                     let curr_fp = if !export_op.value_stable {
                         Some(
-                            Fingerprinter::default()
+                            value_fingerprinter
+                                .clone()
                                 .with(&field_values)?
                                 .into_fingerprint(),
                         )

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/lib_context.rs

@@ -162,6 +162,13 @@ impl FlowContext {
 static TOKIO_RUNTIME: LazyLock<Runtime> = LazyLock::new(|| Runtime::new().unwrap());
 static AUTH_REGISTRY: LazyLock<Arc<AuthRegistry>> = LazyLock::new(|| Arc::new(AuthRegistry::new()));
 
+pub fn get_runtime() -> &'static Runtime {
+    &TOKIO_RUNTIME
+}
+pub fn get_auth_registry() -> &'static Arc<AuthRegistry> {
+    &AUTH_REGISTRY
+}
+
 type PoolKey = (String, Option<String>);
 type PoolValue = Arc<tokio::sync::OnceCell<PgPool>>;
 
@@ -271,21 +278,10 @@ impl LibContext {
     }
 }
 
-pub fn get_runtime() -> &'static Runtime {
-    &TOKIO_RUNTIME
-}
-
-pub fn get_auth_registry() -> &'static Arc<AuthRegistry> {
-    &AUTH_REGISTRY
-}
-
 static LIB_INIT: OnceLock<()> = OnceLock::new();
 pub async fn create_lib_context(settings: settings::Settings) -> Result<LibContext> {
     LIB_INIT.get_or_init(|| {
         let _ = env_logger::try_init();
-
-        pyo3_async_runtimes::tokio::init_with_runtime(get_runtime()).unwrap();
-
         let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
     });
 

{cocoindex-0.2.14 → cocoindex-0.2.16}/src/ops/functions/mod.rs

@@ -1,6 +1,7 @@
 pub mod embed_text;
 pub mod extract_by_llm;
 pub mod parse_json;
+pub mod split_by_separators;
 pub mod split_recursively;
 
 #[cfg(test)]