cocoindex 0.1.80__tar.gz → 0.1.81__tar.gz

{cocoindex-0.1.80 → cocoindex-0.1.81}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.80"
+version = "0.1.81"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.80 → cocoindex-0.1.81}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.80"
+version = "0.1.81"
 edition = "2024"
 rust-version = "1.88"
 readme = "README.md"

{cocoindex-0.1.80 → cocoindex-0.1.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.80
+Version: 0.1.81
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.1.80 → cocoindex-0.1.81}/docs/docs/ai/llm.mdx

@@ -20,7 +20,7 @@ We support the following types of LLM APIs:
 | API Name | `LlmApiType` enum | Text Generation | Text Embedding |
 |----------|---------------------|--------------------|--------------------|
 | [OpenAI](#openai) | `LlmApiType.OPENAI` | ✅ | ✅ |
-| [Ollama](#ollama) | `LlmApiType.OLLAMA` | ✅ | ❌ |
+| [Ollama](#ollama) | `LlmApiType.OLLAMA` | ✅ | ✅ |
 | [Google Gemini](#google-gemini) | `LlmApiType.GEMINI` | ✅ | ✅ |
 | [Vertex AI](#vertex-ai) | `LlmApiType.VERTEX_AI` | ✅ | ✅ |
 | [Anthropic](#anthropic) | `LlmApiType.ANTHROPIC` | ✅ | ❌ |
@@ -141,6 +141,29 @@ cocoindex.LlmSpec(
 </TabItem>
 </Tabs>
 
+For text embedding with Ollama, you'll need to pull an embedding model first:
+
+```bash
+ollama pull nomic-embed-text
+```
+
+Then, a spec for Ollama embedding looks like this:
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+```python
+cocoindex.functions.EmbedText(
+    api_type=cocoindex.LlmApiType.OLLAMA,
+    model="nomic-embed-text",
+    # Optional, use Ollama's default port (11434) on localhost if not specified
+    address="http://localhost:11434",
+)
+```
+
+</TabItem>
+</Tabs>
+
 ### Google Gemini
 
 Google exposes Gemini through Google AI Studio APIs.

{cocoindex-0.1.80 → cocoindex-0.1.81}/docs/docs/custom_ops/custom_targets.mdx

@@ -158,6 +158,95 @@ def prepare(spec: CustomTarget) -> PreparedCustomTarget:
 
 If not provided, the original spec will be passed directly to `mutate`.
 
+### Complete Example
+
+In this example, we define a custom target that accepts data with the following fields:
+- `filename` (key field)
+- `author` (value field)
+- `html` (value field)
+
+<Tabs>
+<TabItem value="python" label="Python" default>
+
+```python
+import dataclasses
+import cocoindex
+
+# 1. Define the target spec
+class MyCustomTarget(cocoindex.op.TargetSpec):
+    """Spec of the custom target, to configure the target location etc."""
+    location: str
+
+# 2. Define the value dataclass for exported data
+@dataclasses.dataclass
+class LocalFileTargetValues:
+    """Represents value fields of exported data."""
+    author: str
+    html: str
+
+# 3. Define the target connector
+@cocoindex.op.target_connector(spec_cls=MyCustomTarget)
+class LocalFileTargetConnector:
+    @staticmethod
+    def get_persistent_key(spec: MyCustomTarget, target_name: str) -> str:
+        return spec.location
+
+    @staticmethod
+    def apply_setup_change(
+        key: str, previous: MyCustomTarget | None, current: MyCustomTarget | None
+    ) -> None:
+        # Setup/teardown logic here
+        ...
+
+    @staticmethod
+    def mutate(
+        *all_mutations: tuple[MyCustomTarget, dict[str, LocalFileTargetValues | None]],
+    ) -> None:
+        """Apply data mutations to the target."""
+        for spec, mutations in all_mutations:
+            for filename, mutation in mutations.items():
+                if mutation is None:
+                    # Delete the file
+                    ...
+                else:
+                    # Write the file with author and html content
+                    ...
+
+# 4. Usage in a flow
+@cocoindex.flow_def(name="ExampleFlow")
+def example_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope) -> None:
+    # Add data source
+    data_scope["documents"] = flow_builder.add_source(...)
+
+    # Create collector
+    output_data = data_scope.add_collector()
+
+    # Collect data
+    with data_scope["documents"].row() as doc:
+        # Create the "author" and "fieldname" field
+        ...
+
+        # Collect the data
+        output_data.collect(filename=doc["filename"], author=doc["author"], html=doc["transformed_html"])
+
+    # Export to custom target
+    output_data.export(
+        "OutputData",
+        MyCustomTarget(location=...),
+        primary_key_fields=["filename"],
+    )
+```
+
+In this example, the type for data in `all_mutations` is `dict[str, LocalFileTargetValues | None]`:
+- `str` is the `DataKeyType` (the filename)
+- `LocalFileTargetValues` is the `DataValueType` (containing `html` and `author` fields)
+- The `mutate()` method receives tuples of `(MyCustomTarget, dict[str, LocalFileTargetValues | None])`
+
+For simplicity, the type hints can be omitted and a `dict` will be created instead of a dataclass instance, and `author` and `html` will be the keys of the dict.
+
+</TabItem>
+</Tabs>
+
 ## Best Practices
 
 ### Idempotency of Methods with Side Effects

{cocoindex-0.1.80 → cocoindex-0.1.81}/examples/amazon_s3_embedding/.env.example

@@ -9,3 +9,7 @@ AMAZON_S3_BUCKET_NAME=your-bucket-name
 
 # Optional
 # AMAZON_S3_SQS_QUEUE_URL=
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

{cocoindex-0.1.80 → cocoindex-0.1.81}/examples/azure_blob_embedding/.env.example

@@ -5,3 +5,7 @@ COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
 AZURE_STORAGE_ACCOUNT_NAME=testnamecocoindex1
 AZURE_BLOB_CONTAINER_NAME=testpublic1
 AZURE_BLOB_PREFIX=
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/code_embedding/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/face_recognition/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

{cocoindex-0.1.80 → cocoindex-0.1.81}/examples/fastapi_server_docker/.env

@@ -3,3 +3,7 @@ COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@coco_db:5436/cocoindex
 
 # For local testing
 # COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

{cocoindex-0.1.80 → cocoindex-0.1.81}/examples/gdrive_text_embedding/.env.example

@@ -8,3 +8,7 @@ GOOGLE_SERVICE_ACCOUNT_CREDENTIAL=/path/to/service_account_credential.json
 # Google Drive root folder IDs, comma separated.
 #! PLEASE FILL IN
 GOOGLE_DRIVE_ROOT_FOLDER_IDS=id1,id2
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/image_search/.env

@@ -0,0 +1,5 @@
+export COCOINDEX_DATABASE_URL="postgres://cocoindex:cocoindex@localhost/cocoindex"
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/manuals_llm_extraction/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/multi_format_indexing/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

{cocoindex-0.1.80 → cocoindex-0.1.81}/examples/multi_format_indexing/README.md

@@ -37,6 +37,8 @@ Install dependencies:
 pip install -e .
 ```
 
+**NOTE**: The `pdf2image` requires `poppler` to be installed manually. Please refer to [this document](https://pdf2image.readthedocs.io/en/latest/installation.html#installing-poppler) for the specific installation instructions for your platform.
+
 Setup:
 
 ```bash

{cocoindex-0.1.80/examples/patient_intake_extraction → cocoindex-0.1.81/examples/paper_metadata}/.env.example

@@ -2,3 +2,7 @@
 COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
 
 OPENAI_API_KEY=
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/pdf_embedding/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/text_embedding/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

cocoindex-0.1.81/examples/text_embedding_qdrant/.env

@@ -0,0 +1,6 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+
+# Fallback to CPU for operations not supported by MPS on Mac.
+# It's no-op for other platforms.
+PYTORCH_ENABLE_MPS_FALLBACK=1

{cocoindex-0.1.80 → cocoindex-0.1.81}/src/execution/evaluator.rs

@@ -354,24 +354,31 @@ async fn evaluate_op_scope(
                 let mut input_values = Vec::with_capacity(op.inputs.len());
                 input_values
                     .extend(assemble_input_values(&op.inputs, scoped_entries).collect::<Vec<_>>());
-                let output_value_cell = memory.get_cache_entry(
-                    || {
-                        Ok(op
-                            .function_exec_info
-                            .fingerprinter
-                            .clone()
-                            .with(&input_values)?
-                            .into_fingerprint())
-                    },
-                    &op.function_exec_info.output_type,
-                    /*ttl=*/ None,
-                )?;
-                let output_value = evaluate_with_cell(output_value_cell.as_ref(), move || {
-                    op.executor.evaluate(input_values)
-                })
-                .await
-                .with_context(|| format!("Evaluating Transform op `{}`", op.name,))?;
-                head_scope.define_field(&op.output, &output_value)?;
+                if op.function_exec_info.enable_cache {
+                    let output_value_cell = memory.get_cache_entry(
+                        || {
+                            Ok(op
+                                .function_exec_info
+                                .fingerprinter
+                                .clone()
+                                .with(&input_values)?
+                                .into_fingerprint())
+                        },
+                        &op.function_exec_info.output_type,
+                        /*ttl=*/ None,
+                    )?;
+                    evaluate_with_cell(output_value_cell.as_ref(), move || {
+                        op.executor.evaluate(input_values)
+                    })
+                    .await
+                    .and_then(|v| head_scope.define_field(&op.output, &v))
+                } else {
+                    op.executor
+                        .evaluate(input_values)
+                        .await
+                        .and_then(|v| head_scope.define_field(&op.output, &v))
+                }
+                .with_context(|| format!("Evaluating Transform op `{}`", op.name,))?
             }
 
             AnalyzedReactiveOp::ForEach(op) => {

{cocoindex-0.1.80 → cocoindex-0.1.81}/src/execution/live_updater.rs

@@ -1,4 +1,7 @@
-use crate::{execution::stats::UpdateStats, prelude::*};
+use crate::{
+    execution::{source_indexer::ProcessSourceKeyOptions, stats::UpdateStats},
+    prelude::*,
+};
 
 use super::stats;
 use futures::future::try_join_all;
@@ -191,14 +194,17 @@ impl SourceUpdateTask {
                                         .await?;
                                     tokio::spawn(source_context.clone().process_source_key(
                                         change.key,
-                                        Some(change.key_aux_info),
-                                        change.data,
                                         update_stats.clone(),
                                         concur_permit,
                                         Some(move || async move {
                                             SharedAckFn::ack(&shared_ack_fn).await
                                         }),
                                         pool.clone(),
+                                        ProcessSourceKeyOptions {
+                                            key_aux_info: Some(change.key_aux_info),
+                                            source_data: change.data,
+                                            ..Default::default()
+                                        },
                                     ));
                                 }
                             }

{cocoindex-0.1.80 → cocoindex-0.1.81}/src/execution/source_indexer.rs

@@ -48,6 +48,13 @@ pub struct SourceIndexingContext {
 
 pub const NO_ACK: Option<fn() -> Ready<Result<()>>> = None;
 
+#[derive(Default)]
+pub struct ProcessSourceKeyOptions {
+    /// `key_aux_info` is not available for deletions. It must not be provided if `source_data` is `None`.
+    pub key_aux_info: Option<serde_json::Value>,
+    pub source_data: Option<interface::SourceData>,
+}
+
 impl SourceIndexingContext {
     pub async fn load(
         flow: Arc<builder::AnalyzedFlow>,
@@ -100,31 +107,29 @@ impl SourceIndexingContext {
         })
     }
 
-    /// `key_aux_info` is not available for deletions. It must not be provided if `source_data` is `None`.
     pub async fn process_source_key<
         AckFut: Future<Output = Result<()>> + Send + 'static,
         AckFn: FnOnce() -> AckFut,
     >(
         self: Arc<Self>,
         key: value::KeyValue,
-        key_aux_info: Option<serde_json::Value>,
-        source_data: Option<interface::SourceData>,
         update_stats: Arc<stats::UpdateStats>,
         _concur_permit: concur_control::CombinedConcurrencyControllerPermit,
         ack_fn: Option<AckFn>,
         pool: PgPool,
+        options: ProcessSourceKeyOptions,
     ) {
         let process = async {
             let plan = self.flow.get_execution_plan().await?;
             let import_op = &plan.import_ops[self.source_idx];
             let schema = &self.flow.data_schema;
-            let source_data = match source_data {
+            let source_data = match options.source_data {
                 Some(source_data) => source_data,
                 None => import_op
                     .executor
                     .get_value(
                         &key,
-                        key_aux_info.as_ref().ok_or_else(|| {
+                        options.key_aux_info.as_ref().ok_or_else(|| {
                             anyhow::anyhow!(
                                 "`key_aux_info` must be provided when there's no `source_data`"
                             )
@@ -337,12 +342,14 @@ impl SourceIndexingContext {
                     .await?;
                 join_set.spawn(self.clone().process_source_key(
                     row.key,
-                    Some(row.key_aux_info),
-                    None,
                     update_stats.clone(),
                     concur_permit,
                     NO_ACK,
                     pool.clone(),
+                    ProcessSourceKeyOptions {
+                        key_aux_info: Some(row.key_aux_info),
+                        ..Default::default()
+                    },
                 ));
             }
         }
@@ -372,12 +379,14 @@ impl SourceIndexingContext {
             let concur_permit = import_op.concurrency_controller.acquire(Some(|| 0)).await?;
             join_set.spawn(self.clone().process_source_key(
                 key,
-                None,
-                Some(source_data),
                 update_stats.clone(),
                 concur_permit,
                 NO_ACK,
                 pool.clone(),
+                ProcessSourceKeyOptions {
+                    source_data: Some(source_data),
+                    ..Default::default()
+                },
             ));
         }
         while let Some(result) = join_set.join_next().await {

{cocoindex-0.1.80 → cocoindex-0.1.81}/src/service/error.rs

@@ -47,7 +47,7 @@ impl IntoResponse for ApiError {
     fn into_response(self) -> Response {
         debug!("Internal server error:\n{:?}", self.err);
         let error_response = ErrorResponse {
-            error: self.err.to_string(),
+            error: format!("{:?}", self.err),
         };
         (self.status_code, Json(error_response)).into_response()
     }

{cocoindex-0.1.80 → cocoindex-0.1.81}/src/setup/flow_features.rs

@@ -3,5 +3,5 @@ use crate::prelude::*;
 pub const SOURCE_STATE_TABLE: &str = "source_state_table";
 
 pub fn default_features() -> BTreeSet<String> {
-    vec![SOURCE_STATE_TABLE.to_string()].into_iter().collect()
+    BTreeSet::new()
 }

cocoindex-0.1.80/examples/face_recognition/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/image_search/.env

@@ -1 +0,0 @@
-export COCOINDEX_DATABASE_URL="postgres://cocoindex:cocoindex@localhost/cocoindex"

cocoindex-0.1.80/examples/manuals_llm_extraction/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/multi_format_indexing/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/pdf_embedding/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/product_recommendation/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/text_embedding/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex

cocoindex-0.1.80/examples/text_embedding_qdrant/.env

@@ -1,2 +0,0 @@
-# Postgres database address for cocoindex
-COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex