cocoindex 0.1.22__tar.gz → 0.1.23__tar.gz

{cocoindex-0.1.22 → cocoindex-0.1.23}/Cargo.lock

@@ -533,7 +533,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.22"
+version = "0.1.23"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -568,7 +568,7 @@ dependencies = [
  "pyo3-async-runtimes",
  "pythonize",
  "qdrant-client",
- "rand 0.9.0",
+ "rand 0.9.1",
  "regex",
  "reqwest",
  "rustls",
@@ -2697,7 +2697,7 @@ checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc"
 dependencies = [
  "bytes",
  "getrandom 0.3.2",
- "rand 0.9.0",
+ "rand 0.9.1",
  "ring",
  "rustc-hash 2.1.1",
  "rustls",
@@ -2751,13 +2751,12 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.3",
- "zerocopy 0.8.24",
 ]
 
 [[package]]

{cocoindex-0.1.22 → cocoindex-0.1.23}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.22"
+version = "0.1.23"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

{cocoindex-0.1.22 → cocoindex-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.22
+Version: 0.1.23
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: pytest ; extra == 'test'

cocoindex-0.1.23/dev/neo4j.yaml

@@ -0,0 +1,25 @@
+name: cocoindex-neo4j
+services:
+  neo4j:
+    image: neo4j:5-enterprise
+    volumes:
+        - /$HOME/neo4j/logs:/logs
+        - /$HOME/neo4j/config:/config
+        - /$HOME/neo4j/data:/data
+        - /$HOME/neo4j/plugins:/plugins
+    environment:
+        - NEO4J_AUTH=neo4j/cocoindex
+        - NEO4J_PLUGINS='["graph-data-science"]'
+        - NEO4J_ACCEPT_LICENSE_AGREEMENT=eval
+
+        # Uncomment to enable query logging
+        # - NEO4J_db_logs_query_enabled=VERBOSE
+        # - NEO4J_db_logs_query_transaction_enabled=VERBOSE
+        # - NEO4J_db_logs_query_parameter__logging__enabled=true
+        # - NEO4J_dbms_logs_http_enabled=true
+        # - NEO4J_server_logs_debug_enabled=true
+
+    ports:
+      - "7474:7474"
+      - "7687:7687"
+    restart: always

{cocoindex-0.1.22 → cocoindex-0.1.23}/examples/docs_to_kg/main.py

@@ -96,35 +96,35 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
         "document_node",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.Neo4jNode(label="Document")),
+            mapping=cocoindex.storages.GraphNode(label="Document")),
         primary_key_fields=["filename"],
     )
     entity_relationship.export(
         "entity_relationship",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.Neo4jRelationship(
+            mapping=cocoindex.storages.GraphRelationship(
                 rel_type="RELATIONSHIP",
-                source=cocoindex.storages.Neo4jRelationshipEnd(
+                source=cocoindex.storages.GraphRelationshipEnd(
                     label="Entity",
                     fields=[
-                        cocoindex.storages.Neo4jFieldMapping(
+                        cocoindex.storages.GraphFieldMapping(
                             field_name="subject", node_field_name="value"),
-                        cocoindex.storages.Neo4jFieldMapping(
+                        cocoindex.storages.GraphFieldMapping(
                             field_name="subject_embedding", node_field_name="embedding"),
                     ]
                 ),
-                target=cocoindex.storages.Neo4jRelationshipEnd(
+                target=cocoindex.storages.GraphRelationshipEnd(
                     label="Entity",
                     fields=[
-                        cocoindex.storages.Neo4jFieldMapping(
+                        cocoindex.storages.GraphFieldMapping(
                             field_name="object", node_field_name="value"),
-                        cocoindex.storages.Neo4jFieldMapping(
+                        cocoindex.storages.GraphFieldMapping(
                             field_name="object_embedding", node_field_name="embedding"),
                     ]
                 ),
                 nodes={
-                    "Entity": cocoindex.storages.Neo4jRelationshipNode(
+                    "Entity": cocoindex.storages.GraphRelationshipNode(
                         primary_key_fields=["value"],
                         vector_indexes=[
                             cocoindex.VectorIndexDef(
@@ -142,15 +142,15 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
         "entity_mention",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.Neo4jRelationship(
+            mapping=cocoindex.storages.GraphRelationship(
                 rel_type="MENTION",
-                source=cocoindex.storages.Neo4jRelationshipEnd(
+                source=cocoindex.storages.GraphRelationshipEnd(
                     label="Document",
-                    fields=[cocoindex.storages.Neo4jFieldMapping("filename")],
+                    fields=[cocoindex.storages.GraphFieldMapping("filename")],
                 ),
-                target=cocoindex.storages.Neo4jRelationshipEnd(
+                target=cocoindex.storages.GraphRelationshipEnd(
                     label="Entity",
-                    fields=[cocoindex.storages.Neo4jFieldMapping(
+                    fields=[cocoindex.storages.GraphFieldMapping(
                         field_name="entity", node_field_name="value")],
                 ),
             ),

{cocoindex-0.1.22 → cocoindex-0.1.23}/python/cocoindex/functions.py

@@ -5,6 +5,9 @@ import sentence_transformers
 from .typing import Float32, Vector, TypeAttr
 from . import op, llm
 
+class ParseJson(op.FunctionSpec):
+    """Parse a text into a JSON object."""
+
 class SplitRecursively(op.FunctionSpec):
     """Split a document (in string) recursively."""
 

{cocoindex-0.1.22 → cocoindex-0.1.23}/python/cocoindex/storages.py

@@ -29,7 +29,7 @@ class Neo4jConnection:
     db: str | None = None
 
 @dataclass
-class Neo4jFieldMapping:
+class GraphFieldMapping:
     """Mapping for a Neo4j field."""
     field_name: str
     # Field name for the node in the Knowledge Graph.
@@ -37,36 +37,36 @@ class Neo4jFieldMapping:
     node_field_name: str | None = None
 
 @dataclass
-class Neo4jRelationshipEnd:
+class GraphRelationshipEnd:
     """Spec for a Neo4j node type."""
     label: str
-    fields: list[Neo4jFieldMapping]
+    fields: list[GraphFieldMapping]
 
 @dataclass
-class Neo4jRelationshipNode:
+class GraphRelationshipNode:
     """Spec for a Neo4j node type."""
     primary_key_fields: Sequence[str]
     vector_indexes: Sequence[index.VectorIndexDef] = ()
 
 @dataclass
-class Neo4jNode:
+class GraphNode:
     """Spec for a Neo4j node type."""
     kind = "Node"
 
     label: str
 
 @dataclass
-class Neo4jRelationship:
+class GraphRelationship:
     """Spec for a Neo4j relationship."""
     kind = "Relationship"
 
     rel_type: str
-    source: Neo4jRelationshipEnd
-    target: Neo4jRelationshipEnd
-    nodes: dict[str, Neo4jRelationshipNode] | None = None
+    source: GraphRelationshipEnd
+    target: GraphRelationshipEnd
+    nodes: dict[str, GraphRelationshipNode] | None = None
 
 class Neo4j(op.StorageSpec):
     """Graph storage powered by Neo4j."""
 
     connection: AuthEntryReference
-    mapping: Neo4jNode | Neo4jRelationship
+    mapping: GraphNode | GraphRelationship

{cocoindex-0.1.22 → cocoindex-0.1.23}/python/cocoindex/typing.py

@@ -183,11 +183,7 @@ def _encode_type(type_info: AnalyzedTypeInfo) -> dict[str, Any]:
         if type_info.elem_type is None:
             raise ValueError(f"{type_info.kind} type must have an element type")
         row_type_info = analyze_type_info(type_info.elem_type)
-        if row_type_info.dataclass_type is None:
-            raise ValueError(f"{type_info.kind} type must have a dataclass type")
-        encoded_type['row'] = {
-            'fields': _encode_fields_schema(row_type_info.dataclass_type),
-        }
+        encoded_type['row'] = _encode_type(row_type_info)
 
     return encoded_type
 

{cocoindex-0.1.22 → cocoindex-0.1.23}/src/builder/analyzer.rs

@@ -714,6 +714,9 @@ impl AnalyzerContext<'_> {
                             .data
                             .add_field(reactive_op.name.clone(), &output_type)?;
                         let reactive_op = reactive_op.clone();
+                        let logic_fingerprinter = Fingerprinter::default()
+                            .with(&op.op)?
+                            .with(&output_type.without_attrs())?;
                         async move {
                             trace!("Start building executor for transform op `{}`", reactive_op.name);
                             let executor = executor.await.with_context(|| {
@@ -725,11 +728,8 @@ impl AnalyzerContext<'_> {
                             let function_exec_info = AnalyzedFunctionExecInfo {
                                 enable_cache,
                                 behavior_version,
-                                fingerprinter: Fingerprinter::default()
-                                    .with(&reactive_op.name)?
-                                    .with(&reactive_op.spec)?
-                                    .with(&behavior_version)?
-                                    .with(&output_type.without_attrs())?,
+                                fingerprinter: logic_fingerprinter
+                                    .with(&behavior_version)?,
                                 output_type: output_type.typ.clone(),
                             };
                             if function_exec_info.enable_cache

{cocoindex-0.1.22 → cocoindex-0.1.23}/src/execution/db_tracking_setup.rs

@@ -58,7 +58,7 @@ pub struct TrackingTableSetupStatusCheck {
 
     pub legacy_table_names: Vec<String>,
 
-    pub min_existing_version_ids: Option<i32>,
+    pub min_existing_version_id: Option<i32>,
     pub source_ids_to_delete: Vec<i32>,
 }
 
@@ -75,7 +75,10 @@ impl TrackingTableSetupStatusCheck {
                 .into_iter()
                 .cloned()
                 .collect(),
-            min_existing_version_ids: existing.possible_versions().map(|v| v.version_id).min(),
+            min_existing_version_id: existing
+                .always_exists()
+                .then(|| existing.possible_versions().map(|v| v.version_id).min())
+                .flatten(),
             source_ids_to_delete,
         }
     }
@@ -102,7 +105,7 @@ impl ResourceSetupStatusCheck for TrackingTableSetupStatusCheck {
                 self.legacy_table_names.join(", ")
             ));
         }
-        match (self.min_existing_version_ids, &self.desired_state) {
+        match (self.min_existing_version_id, &self.desired_state) {
             (None, Some(state)) => {
                 changes.push(format!("Create the tracking table: {}. ", state.table_name))
             }
@@ -131,7 +134,7 @@ impl ResourceSetupStatusCheck for TrackingTableSetupStatusCheck {
     }
 
     fn change_type(&self) -> SetupChangeType {
-        match (self.min_existing_version_ids, &self.desired_state) {
+        match (self.min_existing_version_id, &self.desired_state) {
             (None, Some(_)) => SetupChangeType::Create,
             (Some(min_version_id), Some(desired)) => {
                 if min_version_id == desired.version_id && self.legacy_table_names.is_empty() {
@@ -158,11 +161,11 @@ impl ResourceSetupStatusCheck for TrackingTableSetupStatusCheck {
                 sqlx::query(&query).execute(pool).await?;
             }
 
-            if self.min_existing_version_ids != Some(desired.version_id) {
+            if self.min_existing_version_id != Some(desired.version_id) {
                 upgrade_tracking_table(
                     pool,
                     &desired.table_name,
-                    self.min_existing_version_ids.unwrap_or(0),
+                    self.min_existing_version_id.unwrap_or(0),
                     desired.version_id,
                 )
                 .await?;

{cocoindex-0.1.22 → cocoindex-0.1.23}/src/ops/functions/mod.rs

@@ -1,2 +1,3 @@
 pub mod extract_by_llm;
+pub mod parse_json;
 pub mod split_recursively;

cocoindex-0.1.23/src/ops/functions/parse_json.rs

@@ -0,0 +1,104 @@
+use crate::ops::sdk::*;
+use anyhow::Result;
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock};
+use unicase::UniCase;
+
+pub struct Args {
+    text: ResolvedOpArg,
+    language: Option<ResolvedOpArg>,
+}
+
+type ParseFn = fn(&str) -> Result<serde_json::Value>;
+struct LanguageConfig {
+    parse_fn: ParseFn,
+}
+
+fn add_language<'a>(
+    output: &'a mut HashMap<UniCase<&'static str>, Arc<LanguageConfig>>,
+    name: &'static str,
+    aliases: impl IntoIterator<Item = &'static str>,
+    parse_fn: ParseFn,
+) {
+    let lang_config = Arc::new(LanguageConfig { parse_fn });
+    for name in std::iter::once(name).chain(aliases.into_iter()) {
+        if output.insert(name.into(), lang_config.clone()).is_some() {
+            panic!("Language `{name}` already exists");
+        }
+    }
+}
+
+fn parse_json(text: &str) -> Result<serde_json::Value> {
+    Ok(serde_json::from_str(text)?)
+}
+
+static PARSE_FN_BY_LANG: LazyLock<HashMap<UniCase<&'static str>, Arc<LanguageConfig>>> =
+    LazyLock::new(|| {
+        let mut map = HashMap::new();
+        add_language(&mut map, "json", [".json"], parse_json);
+        map
+    });
+
+struct Executor {
+    args: Args,
+}
+
+#[async_trait]
+impl SimpleFunctionExecutor for Executor {
+    async fn evaluate(&self, input: Vec<value::Value>) -> Result<value::Value> {
+        let text = self.args.text.value(&input)?.as_str()?;
+        let lang_config = {
+            let language = self.args.language.value(&input)?;
+            language
+                .optional()
+                .map(|v| anyhow::Ok(v.as_str()?.as_ref()))
+                .transpose()?
+                .and_then(|lang| PARSE_FN_BY_LANG.get(&UniCase::new(lang)))
+        };
+        let parse_fn = lang_config.map(|c| c.parse_fn).unwrap_or(parse_json);
+        let parsed_value = parse_fn(text)?;
+        Ok(value::Value::Basic(value::BasicValue::Json(Arc::new(
+            parsed_value,
+        ))))
+    }
+}
+
+pub struct Factory;
+
+#[async_trait]
+impl SimpleFunctionFactoryBase for Factory {
+    type Spec = EmptySpec;
+    type ResolvedArgs = Args;
+
+    fn name(&self) -> &str {
+        "ParseJson"
+    }
+
+    fn resolve_schema(
+        &self,
+        _spec: &EmptySpec,
+        args_resolver: &mut OpArgsResolver<'_>,
+        _context: &FlowInstanceContext,
+    ) -> Result<(Args, EnrichedValueType)> {
+        let args = Args {
+            text: args_resolver
+                .next_arg("text")?
+                .expect_type(&ValueType::Basic(BasicValueType::Str))?,
+            language: args_resolver
+                .next_optional_arg("language")?
+                .expect_type(&ValueType::Basic(BasicValueType::Str))?,
+        };
+
+        let output_schema = make_output_type(BasicValueType::Json);
+        Ok((args, output_schema))
+    }
+
+    async fn build_executor(
+        self: Arc<Self>,
+        _spec: EmptySpec,
+        args: Args,
+        _context: Arc<FlowInstanceContext>,
+    ) -> Result<Box<dyn SimpleFunctionExecutor>> {
+        Ok(Box::new(Executor { args }))
+    }
+}

{cocoindex-0.1.22 → cocoindex-0.1.23}/src/ops/registration.rs

@@ -9,6 +9,7 @@ fn register_executor_factories(registry: &mut ExecutorFactoryRegistry) -> Result
     sources::local_file::Factory.register(registry)?;
     sources::google_drive::Factory.register(registry)?;
 
+    functions::parse_json::Factory.register(registry)?;
     functions::split_recursively::Factory.register(registry)?;
     functions::extract_by_llm::Factory.register(registry)?;