cocoindex 0.1.30__tar.gz → 0.1.31__tar.gz

{cocoindex-0.1.30 → cocoindex-0.1.31}/Cargo.lock

@@ -533,7 +533,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.30"
+version = "0.1.31"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.30 → cocoindex-0.1.31}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.30"
+version = "0.1.31"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

{cocoindex-0.1.30 → cocoindex-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.30
+Version: 0.1.31
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/code_embedding/README.md

@@ -46,7 +46,7 @@ CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute vi
 Run CocoInsight to understand your RAG data pipeline:
 
 ```
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/code_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.26", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/docs_to_knowledge_graph/README.md

@@ -66,15 +66,11 @@ I used CocoInsight (Free beta now) to troubleshoot the index generation and unde
 It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
 
 ```bash
-python3 main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 And then open the url https://cocoindex.io/cocoinsight. 
 
-```
-python main.py cocoindex server -c https://cocoindex.io
-```
-
 <img width="1430" alt="cocoinsight" src="https://github.com/user-attachments/assets/d5ada581-cceb-42bf-a949-132df674f3dd" />
 
 

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,4 +3,4 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.26", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/gdrive_text_embedding/README.md

@@ -53,13 +53,13 @@ CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute vi
 Run CocoInsight to understand your RAG data pipeline:
 
 ```sh
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 You can also add a `-L` flag to make the server keep updating the index to reflect source changes at the same time:
 
 ```sh
-python main.py cocoindex server -c https://cocoindex.io -L
+python main.py cocoindex server -ci -L
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.26", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/manuals_llm_extraction/README.md

@@ -65,7 +65,7 @@ CocoInsight is a tool to help you understand your data pipeline and data index.
 Run CocoInsight to understand your RAG data pipeline:
 
 ```
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight). It connects to your local CocoIndex server with zero data retention.

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/manuals_llm_extraction/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.26",
+    "cocoindex>=0.1.30",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/pdf_embedding/README.md

@@ -35,7 +35,7 @@ CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute vi
 Run CocoInsight to understand your RAG data pipeline:
 
 ```
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.26",
+    "cocoindex>=0.1.30",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/text_embedding/README.md

@@ -39,7 +39,7 @@ CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute vi
 Run CocoInsight to understand your RAG data pipeline:
 
 ```
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/text_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.26", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/text_embedding_qdrant/README.md

@@ -63,7 +63,7 @@ CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute vi
 Run CocoInsight to understand your RAG data pipeline:
 
 ```bash
-python main.py cocoindex server -c https://cocoindex.io
+python main.py cocoindex server -ci
 ```
 
 Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

{cocoindex-0.1.30 → cocoindex-0.1.31}/examples/text_embedding_qdrant/pyproject.toml

@@ -3,4 +3,4 @@ name = "text-embedding-qdrant"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.26", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]

{cocoindex-0.1.30 → cocoindex-0.1.31}/python/cocoindex/cli.py

@@ -1,7 +1,9 @@
+import asyncio
 import click
 import datetime
 
 from rich.console import Console
+from rich.table import Table
 
 from . import flow, lib, setting
 from .setup import sync_setup, drop_setup, flow_names_with_setup, apply_setup_changes
@@ -56,11 +58,26 @@ def ls(show_all: bool):
 @click.option("--color/--no-color", default=True)
 def show(flow_name: str | None, color: bool):
     """
-    Show the flow spec in a readable format with colored output.
+    Show the flow spec in a readable format with colored output,
+    including the schema.
     """
-    fl = _flow_by_name(flow_name)
+    flow = _flow_by_name(flow_name)
     console = Console(no_color=not color)
-    console.print(fl._render_text())
+    console.print(flow._render_text())
+
+    table = Table(
+        title=f"Schema for Flow: {flow.name}",
+        show_header=True,
+        header_style="bold magenta"
+    )
+    table.add_column("Field", style="cyan")
+    table.add_column("Type", style="green")
+    table.add_column("Attributes", style="yellow")
+
+    for field_name, field_type, attr_str in flow._render_schema():
+        table.add_row(field_name, field_type, attr_str)
+
+    console.print(table)
 
 @cli.command()
 def setup():

{cocoindex-0.1.30 → cocoindex-0.1.31}/python/cocoindex/flow.py

@@ -503,6 +503,9 @@ class Flow:
             return self._format_flow(flow_dict)
         except json.JSONDecodeError:
             return Text(flow_spec_str)
+    
+    def _render_schema(self) -> list[tuple[str, str, str]]:
+        return self._lazy_engine_flow().get_schema()
 
     def __str__(self):
         return str(self._render_text())

{cocoindex-0.1.30 → cocoindex-0.1.31}/src/base/schema.rs

@@ -1,9 +1,7 @@
-use crate::builder::plan::AnalyzedValueMapping;
+use crate::prelude::*;
 
 use super::spec::*;
-use anyhow::Result;
-use serde::{Deserialize, Serialize};
-use std::{collections::BTreeMap, ops::Deref, sync::Arc};
+use crate::builder::plan::AnalyzedValueMapping;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct VectorTypeSchema {
@@ -60,27 +58,26 @@ pub enum BasicValueType {
 impl std::fmt::Display for BasicValueType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            BasicValueType::Bytes => write!(f, "bytes"),
-            BasicValueType::Str => write!(f, "str"),
-            BasicValueType::Bool => write!(f, "bool"),
-            BasicValueType::Int64 => write!(f, "int64"),
-            BasicValueType::Float32 => write!(f, "float32"),
-            BasicValueType::Float64 => write!(f, "float64"),
-            BasicValueType::Range => write!(f, "range"),
-            BasicValueType::Uuid => write!(f, "uuid"),
-            BasicValueType::Date => write!(f, "date"),
-            BasicValueType::Time => write!(f, "time"),
-            BasicValueType::LocalDateTime => write!(f, "local_datetime"),
-            BasicValueType::OffsetDateTime => write!(f, "offset_datetime"),
-            BasicValueType::Json => write!(f, "json"),
-            BasicValueType::Vector(s) => write!(
-                f,
-                "vector({}, {})",
-                s.dimension
-                    .map(|d| d.to_string())
-                    .unwrap_or_else(|| "*".to_string()),
-                s.element_type
-            ),
+            BasicValueType::Bytes => write!(f, "Bytes"),
+            BasicValueType::Str => write!(f, "Str"),
+            BasicValueType::Bool => write!(f, "Bool"),
+            BasicValueType::Int64 => write!(f, "Int64"),
+            BasicValueType::Float32 => write!(f, "Float32"),
+            BasicValueType::Float64 => write!(f, "Float64"),
+            BasicValueType::Range => write!(f, "Range"),
+            BasicValueType::Uuid => write!(f, "Uuid"),
+            BasicValueType::Date => write!(f, "Date"),
+            BasicValueType::Time => write!(f, "Time"),
+            BasicValueType::LocalDateTime => write!(f, "LocalDateTime"),
+            BasicValueType::OffsetDateTime => write!(f, "OffsetDateTime"),
+            BasicValueType::Json => write!(f, "Json"),
+            BasicValueType::Vector(s) => {
+                write!(f, "Vector[{}", s.element_type)?;
+                if let Some(dimension) = s.dimension {
+                    write!(f, ", {}", dimension)?;
+                }
+                write!(f, "]")
+            }
         }
     }
 }
@@ -141,9 +138,6 @@ impl std::fmt::Display for TableKind {
 pub struct TableSchema {
     pub kind: TableKind,
     pub row: StructSchema,
-
-    #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")]
-    pub collectors: Vec<NamedSpec<Arc<CollectorSchema>>>,
 }
 
 impl TableSchema {
@@ -170,36 +164,19 @@ impl TableSchema {
         Self {
             kind: self.kind,
             row: self.row.without_attrs(),
-            collectors: self
-                .collectors
-                .iter()
-                .map(|c| NamedSpec {
-                    name: c.name.clone(),
-                    spec: Arc::from(c.spec.without_attrs()),
-                })
-                .collect(),
         }
     }
 }
 
 impl std::fmt::Display for TableSchema {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}({}", self.kind, self.row)?;
-        for collector in self.collectors.iter() {
-            write!(f, "; COLLECTOR {} ({})", collector.name, collector.spec)?;
-        }
-        write!(f, ")")?;
-        Ok(())
+        write!(f, "{}({})", self.kind, self.row)
     }
 }
 
 impl TableSchema {
     pub fn new(kind: TableKind, row: StructSchema) -> Self {
-        Self {
-            kind,
-            row,
-            collectors: Default::default(),
-        }
+        Self { kind, row }
     }
 
     pub fn key_field(&self) -> Option<&FieldSchema> {
@@ -409,16 +386,27 @@ impl CollectorSchema {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct OpScopeSchema {
+    /// Output schema for transform ops.
+    pub op_output_types: HashMap<FieldName, EnrichedValueType>,
+
+    /// Child op scope for foreach ops.
+    pub op_scopes: HashMap<String, Arc<OpScopeSchema>>,
+
+    /// Collectors for the current scope.
+    pub collectors: Vec<NamedSpec<Arc<CollectorSchema>>>,
+}
+
 /// Top-level schema for a flow instance.
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DataSchema {
+pub struct FlowSchema {
     pub schema: StructSchema,
 
-    #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")]
-    pub collectors: Vec<NamedSpec<Arc<CollectorSchema>>>,
+    pub root_op_scope: OpScopeSchema,
 }
 
-impl Deref for DataSchema {
+impl std::ops::Deref for FlowSchema {
     type Target = StructSchema;
 
     fn deref(&self) -> &Self::Target {

{cocoindex-0.1.30 → cocoindex-0.1.31}/src/builder/analyzed_flow.rs

@@ -9,7 +9,7 @@ use crate::{
 
 pub struct AnalyzedFlow {
     pub flow_instance: spec::FlowInstanceSpec,
-    pub data_schema: schema::DataSchema,
+    pub data_schema: schema::FlowSchema,
     pub desired_state: setup::FlowSetupState<setup::DesiredMode>,
     /// It's None if the flow is not up to date
     pub execution_plan:
@@ -67,7 +67,7 @@ impl AnalyzedFlow {
 
 pub struct AnalyzedTransientFlow {
     pub transient_flow_instance: spec::TransientFlowSpec,
-    pub data_schema: schema::DataSchema,
+    pub data_schema: schema::FlowSchema,
     pub execution_plan: plan::TransientExecutionPlan,
     pub output_type: schema::EnrichedValueType,
 }