cocoindex 0.1.27__tar.gz → 0.1.29__tar.gz

{cocoindex-0.1.27 → cocoindex-0.1.29}/Cargo.lock

@@ -533,7 +533,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.27"
+version = "0.1.29"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.27 → cocoindex-0.1.29}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.27"
+version = "0.1.29"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

{cocoindex-0.1.27 → cocoindex-0.1.29}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.27
+Version: 0.1.29
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
+Requires-Dist: rich>=14.0.0
 Requires-Dist: pytest ; extra == 'test'
 Provides-Extra: test
 License-File: LICENSE

{cocoindex-0.1.27 → cocoindex-0.1.29}/docs/docs/ops/storages.md

@@ -87,24 +87,34 @@ You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoin
 
 ## Property Graph Targets
 
-Property graph is a graph data model where both nodes and relationships can have properties.
+Property graph is a widely-adopted model for knowledge graphs, where both nodes and relationships can have properties.
+[Graph database concepts](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/) has a good introduction to basic concepts of property graphs.
+
+The following concepts will be used in the following sections:
+* [Node](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-node)
+    * [Node label](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-labels), which represents a type of nodes.
+* [Relationship](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-relationship), which describes a connection between two nodes.
+    * [Relationship type](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-relationship-type)
+* [Properties](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-properties), which are key-value pairs associated with nodes and relationships.
 
 ### Data Mapping
 
-In CocoIndex, you can export data to property graph databases.
-This usually involves more than one collectors, and you export them to different types of graph elements (nodes and relationships).
-In particular,
+Data from collectors are mapped to graph elements in various types:
+
+1.  Rows from collectors → Nodes in the graph
+2.  Rows from collectors → Relationships in the graph (including source and target nodes of the relationship)
 
-1.  You can export rows from some collectors to nodes in the graph.
-2.  You can export rows from some other collectors to relationships in the graph.
-3.  Some nodes referenced by relationships exported in 2 may not exist as nodes exported in 1.
-    CocoIndex will automatically create and keep these nodes, as long as they're still referenced by at least one relationship.
-    This guarantees that all relationships exported in 2 are valid.
+This is what you need to provide to define these mappings:
 
-We provide common types `NodeMapping`, `RelationshipMapping`, and `ReferencedNode`, to configure for each situation.
-They're agnostic to specific graph databases.
+*   Specify [nodes to export](#nodes-to-export).
+*   [Declare extra node labels](#declare-extra-node-labels), for labels to appear as source/target nodes of relationships but not exported as nodes.
+*   Specify [relationships to export](#relationships-to-export).
 
-#### Nodes
+In addition, the same node may appear multiple times, from exported nodes and various relationships.
+They should appear as the same node in the target graph database.
+CocoIndex automatically [matches and deduplicates nodes](#nodes-matching-and-deduplicating) based on their primary key values.
+
+#### Nodes to Export
 
 Here's how CocoIndex data elements map to nodes in the graph:
 
@@ -114,9 +124,9 @@ Here's how CocoIndex data elements map to nodes in the graph:
 | a collected row   | a node |
 | a field           | a property of node |
 
-Note that the label used in different `NodeMapping`s should be unique.
+Note that the label used in different `Nodes`s should be unique.
 
-`cocoindex.storages.NodeMapping` is to describe mapping to nodes. It has the following fields:
+`cocoindex.storages.Nodes` is to describe mapping to nodes. It has the following fields:
 
 *   `label` (type: `str`): The label of the node.
 
@@ -138,7 +148,7 @@ document_collector.export(
     ...
     cocoindex.storages.Neo4j(
         ...
-        mapping=cocoindex.storages.NodeMapping(label="Document"),
+        mapping=cocoindex.storages.Nodes(label="Document"),
     ),
     primary_key_fields=["filename"],
 )
@@ -167,7 +177,32 @@ graph TD
   classDef node font-size:8pt,text-align:left,stroke-width:2;
 ```
 
-#### Relationships
+#### Declare Extra Node Labels
+
+If a node label needs to appear as source or target of a relationship, but not exported as a node, you need to [declare](../core/flow_def#target-declarations) the label with necessary configuration.
+
+The dataclass to describe the declaration is specific to each target storage (e.g. `cocoindex.storages.Neo4jDeclarations`),
+while they share the following common fields:
+
+*   `nodes_label` (required): The label of the node.
+*   Options for [storage indexes](../core/flow_def#storage-indexes).
+    *   `primary_key_fields` (required)
+    *   `vector_indexes` (optional)
+
+Continuing the same example above.
+Considering we want to extract relationships from `Document` to `Place` later (i.e. a document mentions a place), but the `Place` label isn't exported as a node, we need to declare it:
+
+```python
+flow_builder.declare(
+    cocoindex.storages.Neo4jDeclarations(
+        connection = ...,
+        nodes_label="Place",
+        primary_key_fields=["name"],
+    ),
+)
+```
+
+#### Relationships to Export
 
 Here's how CocoIndex data elements map to relationships in the graph:
 
@@ -177,12 +212,12 @@ Here's how CocoIndex data elements map to relationships in the graph:
 | a collected row   | a relationship |
 | a field           | a property of relationship, or a property of source/target node, based on configuration |
 
-Note that the type used in different `RelationshipMapping`s should be unique.
+Note that the type used in different `Relationships`s should be unique.
 
-`cocoindex.storages.RelationshipMapping` is to describe mapping to relationships. It has the following fields:
+`cocoindex.storages.Relationships` is to describe mapping to relationships. It has the following fields:
 
 *   `rel_type` (type: `str`): The type of the relationship.
-*   `source`/`target` (type: `cocoindex.storages.NodeReferenceMapping`): Specify how to extract source/target node information from the collected row. It has the following fields:
+*   `source`/`target` (type: `cocoindex.storages.NodeFromFields`): Specify how to extract source/target node information from specific fields in the collected row. It has the following fields:
     *   `label` (type: `str`): The label of the node.
     *   `fields` (type: `Sequence[cocoindex.storages.TargetFieldMapping]`): Specify field mappings from the collected rows to node properties, with the following fields:
         *   `source` (type: `str`): The name of the field in the collected row.
@@ -218,13 +253,13 @@ doc_place_collector.export(
     ...
     cocoindex.storages.Neo4j(
         ...
-        mapping=cocoindex.storages.RelationshipMapping(
+        mapping=cocoindex.storages.Relationships(
             rel_type="MENTION",
-            source=cocoindex.storages.NodeReferenceMapping(
+            source=cocoindex.storages.NodeFromFields(
                 label="Document",
                 fields=[cocoindex.storages.TargetFieldMapping(source="doc_filename", target="filename")],
             ),
-            target=cocoindex.storages.NodeReferenceMapping(
+            target=cocoindex.storages.NodeFromFields(
                 label="Place",
                 fields=[
                     cocoindex.storages.TargetFieldMapping(source="place_name", target="name"),
@@ -250,19 +285,26 @@ graph TD
     classDef: nodeRef
   }
 
-  Doc_Chapter2@{
+  Doc_Chapter2_a@{
     shape: rounded
     label: "**[Document]**
             **filename\\*: chapter2.md**"
     classDef: nodeRef
   }
 
-  Place_CrystalPalace@{
+  Doc_Chapter2_b@{
+    shape: rounded
+    label: "**[Document]**
+            **filename\\*: chapter2.md**"
+    classDef: nodeRef
+  }
+
+  Place_CrystalPalace_a@{
     shape: rounded
     label: "**[Place]**
             **name\\*: Crystal Palace**
             embedding: [0.1, 0.5, ...]"
-    classDef: nodeRef
+    classDef: node
   }
 
   Place_MagicForest@{
@@ -270,38 +312,43 @@ graph TD
     label: "**[Place]**
             **name\\*: Magic Forest**
             embedding: [0.4, 0.2, ...]"
-    classDef: nodeRef
+    classDef: node
+  }
+
+  Place_CrystalPalace_b@{
+    shape: rounded
+    label: "**[Place]**
+            **name\\*: Crystal Palace**
+            embedding: [0.1, 0.5, ...]"
+    classDef: node
   }
 
-  Doc_Chapter1:::nodeRef -- **[MENTION]**{location:12} --> Place_CrystalPalace:::nodeRef
-  Doc_Chapter2:::nodeRef -- **[MENTION]**{location:23} --> Place_MagicForest:::nodeRef
-  Doc_Chapter2:::nodeRef -- **[MENTION]**{location:56} --> Place_CrystalPalace:::nodeRef
+
+  Doc_Chapter1:::nodeRef -- **:MENTION** (location:12) --> Place_CrystalPalace_a:::node
+  Doc_Chapter2_a:::nodeRef -- **:MENTION** (location:23) --> Place_MagicForest:::node
+  Doc_Chapter2_b:::nodeRef -- **:MENTION** (location:56) --> Place_CrystalPalace_b:::node
 
   classDef nodeRef font-size:8pt,text-align:left,fill:transparent,stroke-width:1,stroke-dasharray:5 5;
+  classDef node font-size:8pt,text-align:left,stroke-width:2;
 
 ```
 
+#### Nodes Matching and Deduplicating
 
-#### Nodes only referenced by relationships
+The nodes and relationships we got above are discrete elements.
+To fit them into a connected property graph, CocoIndex will match and deduplicate nodes automatically:
 
-If a node appears as source or target of a relationship, but not exported using `NodeMapping`, CocoIndex will automatically create and keep these nodes until they're no longer referenced by any relationships.
+*   Match nodes based on their primary key values. Nodes with the same primary key values are considered as the same node.
+*   For non-primary key fields (a.k.a. value fields), CocoIndex will pick the values from an arbitrary one.
+    If multiple nodes (before deduplication) with the same primary key provide value fields, an arbitrary one will be picked.
 
-:::note Merge of node values
+:::note
 
-If the same node (as identified by primary key values) appears multiple times (e.g. they're referenced by different relationships),
-CocoIndex uses value fields provided by an arbitrary one of them.
 The best practice is to make the value fields consistent across different appearances of the same node, to avoid non-determinism in the exported graph.
 
 :::
 
-If a node's label specified in `NodeReferenceMapping` doesn't exist in any `NodeMapping`, you need to [declare](../core/flow_def#target-declarations) a `ReferencedNode` to configure [storage indexes](../core/flow_def#storage-indexes) for nodes with this label.
-The following options are supported:
-
-*   `primary_key_fields` (required)
-*   `vector_indexes` (optional)
-
-Using the same example above.
-After combining exported nodes and relationships, we get the knowledge graph with all information:
+After matching and deduplication, we get the final graph:
 
 ```mermaid
 graph TD
@@ -326,7 +373,7 @@ graph TD
     label: "**[Place]**
             **name\\*: Crystal Palace**
             embedding: [0.1, 0.5, ...]"
-    classDef: nodeRef
+    classDef: node
   }
 
   Place_MagicForest@{
@@ -334,30 +381,14 @@ graph TD
     label: "**[Place]**
             **name\\*: Magic Forest**
             embedding: [0.4, 0.2, ...]"
-    classDef: nodeRef
+    classDef: node
   }
 
-  Doc_Chapter1:::node -- **[MENTION]**{location:12} --> Place_CrystalPalace:::nodeRef
-  Doc_Chapter2:::node -- **[MENTION]**{location:23} --> Place_MagicForest:::nodeRef
-  Doc_Chapter2:::node -- **[MENTION]**{location:56} --> Place_CrystalPalace:::nodeRef
+  Doc_Chapter1:::node -- **:MENTION** (location:12) --> Place_CrystalPalace:::node
+  Doc_Chapter2:::node -- **:MENTION** (location:23) --> Place_MagicForest:::node
+  Doc_Chapter2:::node -- **:MENTION** (location:56) --> Place_CrystalPalace:::node
 
   classDef node font-size:8pt,text-align:left,stroke-width:2;
-  classDef nodeRef font-size:8pt,text-align:left,fill:transparent,stroke-width:1,stroke-dasharray:5 5;
-
-```
-
-Nodes with `Place` label in the example aren't exported explicitly using `NodeMapping`, so CocoIndex will automatically create them as long as they're still referenced by any relationship.
-You need to declare a `ReferencedNode`:
-
-```python
-flow_builder.declare(
-    cocoindex.storages.Neo4jDeclarations(
-        ...
-        referenced_nodes=[
-            cocoindex.storages.ReferencedNode(label="Place", primary_key_fields=["name"]),
-        ],
-    ),
-)
 ```
 
 ### Neo4j
@@ -388,6 +419,9 @@ The `Neo4j` storage exports each row as a relationship to Neo4j Knowledge Graph.
 Neo4j also provides a declaration spec `Neo4jDeclaration`, to configure indexing options for nodes only referenced by relationships. It has the following fields:
 
 *   `connection` (type: auth reference to `Neo4jConnectionSpec`)
-*   `relationships` (type: `Sequence[ReferencedNode]`)
+*   Fields for [nodes to declare](#nodes-to-declare), including
+    *   `nodes_label` (required)
+    *   `primary_key_fields` (required)
+    *   `vector_indexes` (optional)
 
 You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph).

{cocoindex-0.1.27 → cocoindex-0.1.29}/examples/docs_to_knowledge_graph/main.py

@@ -13,7 +13,10 @@ class DocumentSummary:
 
 @dataclasses.dataclass
 class Relationship:
-    """Describe a relationship between two entities."""
+    """
+    Describe a relationship between two entities.
+    Subject and object should be Core CocoIndex concepts only, should be nouns. For example, `CocoIndex`, `Incremental Processing`, `ETL`,  `Data` etc.
+    """
     subject: str
     predicate: str
     object: str
@@ -62,8 +65,8 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
                     output_type=list[Relationship],
                     instruction=(
                         "Please extract relationships from CocoIndex documents. "
-                        "Focus on concepts and ingnore specific examples. "
-                        "Each relationship should be a tuple of (subject, predicate, object).")))
+                        "Focus on concepts and ignore examples and code. "
+                        )))
 
         with doc["relationships"].row() as relationship:
             # relationship between two entities
@@ -90,35 +93,31 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
         "document_node",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.NodeMapping(label="Document")),
+            mapping=cocoindex.storages.Nodes(label="Document")),
         primary_key_fields=["filename"],
     )
     # Declare reference Node to reference entity node in a relationship
     flow_builder.declare(
-        cocoindex.storages.Neo4jDeclarations(
+        cocoindex.storages.Neo4jDeclaration(
             connection=conn_spec,
-            referenced_nodes=[
-                cocoindex.storages.ReferencedNode(
-                    label="Entity",
-                    primary_key_fields=["value"],
-                )
-            ]
+            nodes_label="Entity",
+            primary_key_fields=["value"],
         )
     )
     entity_relationship.export(
         "entity_relationship",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.RelationshipMapping(
+            mapping=cocoindex.storages.Relationships(
                 rel_type="RELATIONSHIP",
-                source=cocoindex.storages.NodeReferenceMapping(
+                source=cocoindex.storages.NodeFromFields(
                     label="Entity",
                     fields=[
                         cocoindex.storages.TargetFieldMapping(
                             source="subject", target="value"),
                     ]
                 ),
-                target=cocoindex.storages.NodeReferenceMapping(
+                target=cocoindex.storages.NodeFromFields(
                     label="Entity",
                     fields=[
                         cocoindex.storages.TargetFieldMapping(
@@ -133,13 +132,13 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
         "entity_mention",
         cocoindex.storages.Neo4j(
             connection=conn_spec,
-            mapping=cocoindex.storages.RelationshipMapping(
+            mapping=cocoindex.storages.Relationships(
                 rel_type="MENTION",
-                source=cocoindex.storages.NodeReferenceMapping(
+                source=cocoindex.storages.NodeFromFields(
                     label="Document",
                     fields=[cocoindex.storages.TargetFieldMapping("filename")],
                 ),
-                target=cocoindex.storages.NodeReferenceMapping(
+                target=cocoindex.storages.NodeFromFields(
                     label="Entity",
                     fields=[cocoindex.storages.TargetFieldMapping(
                         source="entity", target="value")],

{cocoindex-0.1.27 → cocoindex-0.1.29}/pyproject.toml

@@ -9,7 +9,7 @@ description = "With CocoIndex, users declare the transformation, CocoIndex creat
 authors = [{ name = "CocoIndex", email = "cocoindex.io@gmail.com" }]
 readme = "README.md"
 requires-python = ">=3.11"
-dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8"]
+dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8", "rich>=14.0.0"]
 license = "Apache-2.0"
 urls = { Homepage = "https://cocoindex.io/" }
 

{cocoindex-0.1.27 → cocoindex-0.1.29}/python/cocoindex/cli.py

@@ -1,6 +1,7 @@
 import asyncio
 import click
 import datetime
+from rich.console import Console
 
 from . import flow, lib
 from .setup import sync_setup, drop_setup, flow_names_with_setup, apply_setup_changes
@@ -52,11 +53,14 @@ def ls(show_all: bool):
 
 @cli.command()
 @click.argument("flow_name", type=str, required=False)
-def show(flow_name: str | None):
+@click.option("--color/--no-color", default=True)
+def show(flow_name: str | None, color: bool):
     """
-    Show the flow spec.
+    Show the flow spec in a readable format with colored output.
     """
-    click.echo(str(_flow_by_name(flow_name)))
+    flow = _flow_by_name(flow_name)
+    console = Console(no_color=not color)
+    console.print(flow._render_text())
 
 @cli.command()
 def setup():

{cocoindex-0.1.27 → cocoindex-0.1.29}/python/cocoindex/flow.py

@@ -8,11 +8,14 @@ import asyncio
 import re
 import inspect
 import datetime
+import json
 
 from typing import Any, Callable, Sequence, TypeVar
 from threading import Lock
 from enum import Enum
 from dataclasses import dataclass
+from rich.text import Text
+from rich.console import Console
 
 from . import _engine
 from . import index
@@ -451,8 +454,58 @@ class Flow:
             return engine_flow
         self._lazy_engine_flow = _lazy_engine_flow
 
+    def _format_flow(self, flow_dict: dict) -> Text:
+        output = Text()
+
+        def add_line(content, indent=0, style=None, end="\n"):
+            output.append(" " * indent)
+            output.append(content, style=style)
+            output.append(end)
+
+        def format_key_value(key, value, indent):
+            if isinstance(value, (dict, list)):
+                add_line(f"- {key}:", indent, style="green")
+                format_data(value, indent + 2)
+            else:
+                add_line(f"- {key}:", indent, style="green", end="")
+                add_line(f" {value}", style="yellow")
+
+        def format_data(data, indent=0):
+            if isinstance(data, dict):
+                for key, value in data.items():
+                    format_key_value(key, value, indent)
+            elif isinstance(data, list):
+                for i, item in enumerate(data):
+                    format_key_value(f"[{i}]", item, indent)
+            else:
+                add_line(str(data), indent, style="yellow")
+
+        # Header
+        flow_name = flow_dict.get("name", "Unnamed")
+        add_line(f"Flow: {flow_name}", style="bold cyan")
+
+        # Section
+        for section_title, section_key in [
+            ("Sources:", "import_ops"),
+            ("Processing:", "reactive_ops"),
+            ("Targets:", "export_ops"),
+        ]:
+            add_line("")
+            add_line(section_title, style="bold cyan")
+            format_data(flow_dict.get(section_key, []), indent=0)
+
+        return output
+
+    def _render_text(self) -> Text:
+        flow_spec_str = str(self._lazy_engine_flow())
+        try:
+            flow_dict = json.loads(flow_spec_str)
+            return self._format_flow(flow_dict)
+        except json.JSONDecodeError:
+            return Text(flow_spec_str)
+
     def __str__(self):
-        return str(self._lazy_engine_flow())
+        return str(self._render_text())
 
     def __repr__(self):
         return repr(self._lazy_engine_flow())

{cocoindex-0.1.27 → cocoindex-0.1.29}/python/cocoindex/storages.py

@@ -37,7 +37,7 @@ class TargetFieldMapping:
     target: str | None = None
 
 @dataclass
-class NodeReferenceMapping:
+class NodeFromFields:
     """Spec for a referenced graph node, usually as part of a relationship."""
     label: str
     fields: list[TargetFieldMapping]
@@ -50,30 +50,37 @@ class ReferencedNode:
     vector_indexes: Sequence[index.VectorIndexDef] = ()
 
 @dataclass
-class NodeMapping:
+class Nodes:
     """Spec to map a row to a graph node."""
     kind = "Node"
 
     label: str
 
 @dataclass
-class RelationshipMapping:
+class Relationships:
     """Spec to map a row to a graph relationship."""
     kind = "Relationship"
 
     rel_type: str
-    source: NodeReferenceMapping
-    target: NodeReferenceMapping
+    source: NodeFromFields
+    target: NodeFromFields
+
+# For backwards compatibility only
+NodeMapping = Nodes
+RelationshipMapping = Relationships
+NodeReferenceMapping = NodeFromFields
 
 class Neo4j(op.StorageSpec):
     """Graph storage powered by Neo4j."""
 
     connection: AuthEntryReference
-    mapping: NodeMapping | RelationshipMapping
+    mapping: Nodes | Relationships
 
-class Neo4jDeclarations(op.DeclarationSpec):
+class Neo4jDeclaration(op.DeclarationSpec):
     """Declarations for Neo4j."""
 
     kind = "Neo4j"
     connection: AuthEntryReference
-    referenced_nodes: Sequence[ReferencedNode] = ()
+    nodes_label: str
+    primary_key_fields: Sequence[str]
+    vector_indexes: Sequence[index.VectorIndexDef] = ()

{cocoindex-0.1.27 → cocoindex-0.1.29}/src/base/schema.rs

@@ -120,8 +120,10 @@ pub enum TableKind {
     /// An table with unordered rows, without key.
     UTable,
     /// A table's first field is the key.
+    #[serde(alias = "Table")]
     KTable,
     /// A table whose rows orders are preserved.
+    #[serde(alias = "List")]
     LTable,
 }