cocoindex 0.1.35__tar.gz → 0.1.36__tar.gz

{cocoindex-0.1.35 → cocoindex-0.1.36}/Cargo.lock

@@ -533,7 +533,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.35"
+version = "0.1.36"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.35 → cocoindex-0.1.36}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.35"
+version = "0.1.36"
 edition = "2021"
 
 [profile.release]

{cocoindex-0.1.35 → cocoindex-0.1.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.35
+Version: 0.1.36
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0

{cocoindex-0.1.35 → cocoindex-0.1.36}/docs/docs/core/basics.md

@@ -11,7 +11,7 @@ CocoIndex is an ETL framework for building indexes from specified data sources,
 
 ## Indexing flow
 
-An indexing flow extracts data from speicfied data sources, upon specified transformations, and puts the transformed data into specified storage for later retrieval.
+An indexing flow extracts data from specified data sources, upon specified transformations, and puts the transformed data into specified storage for later retrieval.
 
 An indexing flow has two aspects: data and operations on data.
 

{cocoindex-0.1.35 → cocoindex-0.1.36}/docs/docs/core/flow_methods.mdx

@@ -1,5 +1,6 @@
 ---
 title: Flow Running
+toc_max_heading_level: 4
 description: Run a CocoIndex Flow, including build / update data in the target storage and evaluate the flow without changing the target storage.
 ---
 
@@ -70,8 +71,7 @@ This is to achieve best efficiency.
 
 ### One time update
 
-<Tabs queryString="code_lang">
-<TabItem value="shell" label="Shell" default>
+#### CLI
 
 The `cocoindex update` subcommand creates/updates data in the target storage.
 
@@ -81,7 +81,9 @@ Once it's done, the target data is fresh up to the moment when the function is c
 python main.py cocoindex update
 ```
 
-</TabItem>
+#### Library API
+
+<Tabs>
 <TabItem value="python" label="Python">
 
 The `update()` async method creates/updates data in the target storage.
@@ -108,8 +110,7 @@ A data source may enable one or multiple *change capture mechanisms*:
 
 Change capture mechanisms enable CocoIndex to continuously capture changes from the source data and update the target data accordingly, under live update mode.
 
-<Tabs queryString="code_lang">
-<TabItem value="shell" label="Shell" default>
+#### CLI
 
 To perform live update, run the `cocoindex update` subcommand with `-L` option:
 
@@ -120,7 +121,9 @@ python main.py cocoindex update -L
 If there's at least one data source with change capture mechanism enabled, it will keep running until the aborted (e.g. by `Ctrl-C`).
 Otherwise, it falls back to the same behavior as one time update, and will finish after a one-time update is done.
 
-</TabItem>
+#### Library API
+
+<Tabs>
 <TabItem value="python" label="Python">
 
 To perform live update, you need to create a `cocoindex.FlowLiveUpdater` object using the `cocoindex.Flow` object.
@@ -145,8 +148,10 @@ my_updater = cocoindex.FlowLiveUpdater(
 
 A `FlowLiveUpdater` object supports the following methods:
 
+*   `start()`: Start the updater.
+    CocoIndex will continuously capture changes from the source data and update the target data accordingly in background threads managed by the engine.
 *   `abort()`: Abort the updater.
-*   `wait()` (async): Wait for the updater to finish. It only unblocks in one of the following cases:
+*   `wait()`: Wait for the updater to finish. It only unblocks in one of the following cases:
     *   The updater was aborted.
     *   A one time update is done, and live update is not enabled:
         either `live_mode` is `False`, or all data sources have no change capture mechanisms enabled.
@@ -154,6 +159,8 @@ A `FlowLiveUpdater` object supports the following methods:
 
 ```python
 my_updater = cocoindex.FlowLiveUpdater(demo_flow)
+# Start the updater.
+my_updater.start()
 
 # Perform your own logic (e.g. a query loop).
 ...
@@ -163,26 +170,47 @@ print(my_updater.update_stats())
 # Abort the updater.
 my_updater.abort()
 # Wait for the updater to finish.
-await my_updater.wait()
+my_updater.wait()
 ```
 
 Python SDK also allows you to use the updater as a context manager.
-It will abort and wait for the updater to finish automatically when the context is exited.
-The following code is equivalent to the code above:
+It will automatically start the updater during the context entry, and abort and wait for the updater to finish automatically when the context is exited.
+The following code is equivalent to the code above (if no early return happens):
 
 ```python
-async with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
+with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
     # Perform your own logic (e.g. a query loop).
     ...
     print(my_updater.update_stats())
 ```
 
-Within a synchronous function, remove `async` before `with`, like this:
+CocoIndex also provides asynchronous versions of APIs for blocking operations, including:
 
-```python
-with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
+*   `start_async()` and `wait_async()`, e.g.
+
+    ```python
+    my_updater = cocoindex.FlowLiveUpdater(demo_flow)
+    # Start the updater.
+    await my_updater.start_async()
+    
+    # Perform your own logic (e.g. a query loop).
     ...
-```
+    
+    # Print the update stats.
+    print(my_updater.update_stats())
+    # Abort the updater.
+    my_updater.abort()
+    # Wait for the updater to finish.
+    await my_updater.wait_async()
+    ```
+*   Async context manager, e.g.
+
+    ```python
+    async with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
+        # Perform your own logic (e.g. a query loop).
+        ...
+        print(my_updater.update_stats())
+    ```
 
 </TabItem>
 </Tabs>
@@ -191,8 +219,7 @@ with cocoindex.FlowLiveUpdater(demo_flow) as my_updater:
 
 CocoIndex allows you to run the transformations defined by the flow without updating the target storage.
 
-<Tabs queryString="code_lang">
-<TabItem value="shell" label="Shell" default>
+### CLI
 
 The `cocoindex evaluate` subcommand runs the transformation and dumps flow outputs.
 It takes the following options:
@@ -208,7 +235,9 @@ Example:
 python main.py cocoindex evaluate --output-dir ./eval_output
 ```
 
-</TabItem>
+### Library API
+
+<Tabs>
 <TabItem value="python" label="Python">
 
 The `evaluate_and_dump()` method runs the transformation and dumps flow outputs to files.

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/code_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,4 +3,4 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/fastapi_server_docker/requirements.txt

@@ -1,4 +1,4 @@
-cocoindex>=0.1.26
+cocoindex>=0.1.35
 python-dotenv>=1.0.1
 fastapi==0.115.12
 fastapi-cli==0.0.7

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/image_search_example/requirements.txt

@@ -1,4 +1,4 @@
-cocoindex>=0.1.30
+cocoindex>=0.1.35
 python-dotenv>=1.0.1
 requests>=2.31.0
 uvicorn>=0.29.0

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/manuals_llm_extraction/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.32",
+    "cocoindex>=0.1.35",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.32",
+    "cocoindex>=0.1.35",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/product_taxonomy_knowledge_graph/pyproject.toml

@@ -3,4 +3,4 @@ name = "cocoindex-ecommerce-taxonomy"
 version = "0.1.0"
 description = "Simple example for CocoIndex: extract taxonomy from e-commerce products and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1", "jinja2>=3.1.6"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1", "jinja2>=3.1.6"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/text_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/examples/text_embedding_qdrant/pyproject.toml

@@ -3,4 +3,4 @@ name = "text-embedding-qdrant"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local text files."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.35", "python-dotenv>=1.0.1"]

{cocoindex-0.1.35 → cocoindex-0.1.36}/src/execution/db_tracking.rs

@@ -211,3 +211,28 @@ impl ListTrackedSourceKeyMetadataState {
         sqlx::query_as(&self.query_str).bind(source_id).fetch(pool)
     }
 }
+
+#[derive(sqlx::FromRow, Debug)]
+pub struct SourceLastProcessedInfo {
+    pub processed_source_ordinal: Option<i64>,
+    pub process_logic_fingerprint: Option<Vec<u8>>,
+    pub process_time_micros: Option<i64>,
+}
+
+pub async fn read_source_last_processed_info(
+    source_id: i32,
+    source_key_json: &serde_json::Value,
+    db_setup: &TrackingTableSetupState,
+    pool: &PgPool,
+) -> Result<Option<SourceLastProcessedInfo>> {
+    let query_str = format!(
+        "SELECT processed_source_ordinal, process_logic_fingerprint, process_time_micros FROM {} WHERE source_id = $1 AND source_key = $2",
+        db_setup.table_name
+    );
+    let last_processed_info = sqlx::query_as(&query_str)
+        .bind(source_id)
+        .bind(source_key_json)
+        .fetch_optional(pool)
+        .await?;
+    Ok(last_processed_info)
+}

{cocoindex-0.1.35 → cocoindex-0.1.36}/src/execution/dumper.rs

@@ -11,6 +11,7 @@ use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 use yaml_rust2::YamlEmitter;
 
+use super::evaluator::SourceRowEvaluationContext;
 use super::memoization::EvaluationMemoryOptions;
 use super::row_indexer;
 use crate::base::{schema, value};
@@ -77,10 +78,12 @@ impl<'a> Dumper<'a> {
         'a: 'b,
     {
         let data_builder = row_indexer::evaluate_source_entry_with_memory(
-            self.plan,
-            import_op,
-            self.schema,
-            key,
+            &SourceRowEvaluationContext {
+                plan: self.plan,
+                import_op,
+                schema: self.schema,
+                key,
+            },
             EvaluationMemoryOptions {
                 enable_cache: self.options.use_cache,
                 evaluation_only: true,
@@ -167,7 +170,7 @@ impl<'a> Dumper<'a> {
     async fn evaluate_and_dump_for_source(&self, import_op: &AnalyzedImportOp) -> Result<()> {
         let mut keys_by_filename_prefix: IndexMap<String, Vec<value::KeyValue>> = IndexMap::new();
 
-        let mut rows_stream = import_op.executor.list(SourceExecutorListOptions {
+        let mut rows_stream = import_op.executor.list(&SourceExecutorListOptions {
             include_ordinal: false,
         });
         while let Some(rows) = rows_stream.next().await {

{cocoindex-0.1.35 → cocoindex-0.1.36}/src/execution/evaluator.rs

@@ -446,6 +446,13 @@ async fn evaluate_op_scope(
     Ok(())
 }
 
+pub struct SourceRowEvaluationContext<'a> {
+    pub plan: &'a ExecutionPlan,
+    pub import_op: &'a AnalyzedImportOp,
+    pub schema: &'a schema::FlowSchema,
+    pub key: &'a value::KeyValue,
+}
+
 #[derive(Debug)]
 pub struct EvaluateSourceEntryOutput {
     pub data_scope: ScopeValueBuilder,
@@ -453,23 +460,20 @@ pub struct EvaluateSourceEntryOutput {
 }
 
 pub async fn evaluate_source_entry(
-    plan: &ExecutionPlan,
-    import_op: &AnalyzedImportOp,
-    schema: &schema::FlowSchema,
-    key: &value::KeyValue,
+    src_eval_ctx: &SourceRowEvaluationContext<'_>,
     source_value: value::FieldValues,
     memory: &EvaluationMemory,
 ) -> Result<EvaluateSourceEntryOutput> {
-    let root_schema = &schema.schema;
+    let root_schema = &src_eval_ctx.schema.schema;
     let root_scope_value = ScopeValueBuilder::new(root_schema.fields.len());
     let root_scope_entry = ScopeEntry::new(
         ScopeKey::None,
         &root_scope_value,
         root_schema,
-        &plan.op_scope,
+        &src_eval_ctx.plan.op_scope,
     );
 
-    let table_schema = match &root_schema.fields[import_op.output.field_idx as usize]
+    let table_schema = match &root_schema.fields[src_eval_ctx.import_op.output.field_idx as usize]
         .value_type
         .typ
     {
@@ -482,12 +486,12 @@ pub async fn evaluate_source_entry(
     let scope_value =
         ScopeValueBuilder::augmented_from(&value::ScopeValue(source_value), table_schema)?;
     root_scope_entry.define_field_w_builder(
-        &import_op.output,
-        value::Value::KTable(BTreeMap::from([(key.clone(), scope_value)])),
+        &src_eval_ctx.import_op.output,
+        value::Value::KTable(BTreeMap::from([(src_eval_ctx.key.clone(), scope_value)])),
     );
 
     evaluate_op_scope(
-        &plan.op_scope,
+        &src_eval_ctx.plan.op_scope,
         RefList::Nil.prepend(&root_scope_entry),
         memory,
     )

cocoindex-0.1.36/src/execution/indexing_status.rs

@@ -0,0 +1,59 @@
+use crate::prelude::*;
+
+use super::db_tracking;
+use super::evaluator;
+use futures::try_join;
+
+#[derive(Debug, Serialize)]
+pub struct SourceRowLastProcessedInfo {
+    pub source_ordinal: Option<interface::Ordinal>,
+    pub processing_time: Option<chrono::DateTime<chrono::Utc>>,
+    pub is_logic_current: bool,
+}
+
+#[derive(Debug, Serialize)]
+pub struct SourceRowInfo {
+    pub ordinal: Option<interface::Ordinal>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct SourceRowIndexingStatus {
+    pub last_processed: Option<SourceRowLastProcessedInfo>,
+    pub current: Option<SourceRowInfo>,
+}
+
+pub async fn get_source_row_indexing_status(
+    src_eval_ctx: &evaluator::SourceRowEvaluationContext<'_>,
+    pool: &sqlx::PgPool,
+) -> Result<SourceRowIndexingStatus> {
+    let source_key_json = serde_json::to_value(src_eval_ctx.key)?;
+    let last_processed_fut = db_tracking::read_source_last_processed_info(
+        src_eval_ctx.import_op.source_id,
+        &source_key_json,
+        &src_eval_ctx.plan.tracking_table_setup,
+        pool,
+    );
+    let current_fut = src_eval_ctx.import_op.executor.get_value(
+        &src_eval_ctx.key,
+        &interface::SourceExecutorGetOptions {
+            include_value: false,
+            include_ordinal: true,
+        },
+    );
+    let (last_processed, current) = try_join!(last_processed_fut, current_fut)?;
+
+    let last_processed = last_processed.map(|l| SourceRowLastProcessedInfo {
+        source_ordinal: l.processed_source_ordinal.map(interface::Ordinal),
+        processing_time: l
+            .process_time_micros
+            .map(chrono::DateTime::<chrono::Utc>::from_timestamp_micros)
+            .flatten(),
+        is_logic_current: Some(src_eval_ctx.plan.logic_fingerprint.0.as_slice())
+            == l.process_logic_fingerprint.as_ref().map(|b| b.as_slice()),
+    });
+    let current = current.map(|c| SourceRowInfo { ordinal: c.ordinal });
+    Ok(SourceRowIndexingStatus {
+        last_processed,
+        current,
+    })
+}

{cocoindex-0.1.35 → cocoindex-0.1.36}/src/execution/mod.rs

@@ -1,6 +1,7 @@
 pub(crate) mod db_tracking_setup;
 pub(crate) mod dumper;
 pub(crate) mod evaluator;
+pub(crate) mod indexing_status;
 pub(crate) mod memoization;
 pub(crate) mod query;
 pub(crate) mod row_indexer;