cocoindex 0.2.0__tar.gz → 0.2.1__tar.gz

{cocoindex-0.2.0 → cocoindex-0.2.1}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.2.0 → cocoindex-0.2.1}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.0"
+version = "0.2.1"
 edition = "2024"
 rust-version = "1.88"
 readme = "README.md"

{cocoindex-0.2.0 → cocoindex-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.0
+Version: 0.2.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.2.0 → cocoindex-0.2.1}/docs/docs/core/settings.mdx

@@ -96,11 +96,11 @@ If not set, all flows are in a default unnamed namespace.
 
     :::
 
-*   `max_connections` (type: `int`, default: `64`): The maximum number of connections to keep in the pool.
+*   `max_connections` (type: `int`, default: `25`): The maximum number of connections to keep in the pool.
 
     *Environment variable* for `Settings.database.max_connections`: `COCOINDEX_DATABASE_MAX_CONNECTIONS`
 
-*   `min_connections` (type: `int`, default: `16`): The minimum number of connections to keep in the pool.
+*   `min_connections` (type: `int`, default: `5`): The minimum number of connections to keep in the pool.
 
     *Environment variable* for `Settings.database.min_connections`: `COCOINDEX_DATABASE_MIN_CONNECTIONS`
 
@@ -134,7 +134,7 @@ This is the list of environment variables, each of which has a corresponding fie
 | `COCOINDEX_DATABASE_URL` | `database.url` | Yes |
 | `COCOINDEX_DATABASE_USER` | `database.user` | No |
 | `COCOINDEX_DATABASE_PASSWORD` | `database.password` | No |
-| `COCOINDEX_DATABASE_MAX_CONNECTIONS` | `database.max_connections` | No (default: `64`) |
-| `COCOINDEX_DATABASE_MIN_CONNECTIONS` | `database.min_connections` | No (default: `16`) |
+| `COCOINDEX_DATABASE_MAX_CONNECTIONS` | `database.max_connections` | No (default: `25`) |
+| `COCOINDEX_DATABASE_MIN_CONNECTIONS` | `database.min_connections` | No (default: `5`) |
 | `COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS` | `global_execution_options.source_max_inflight_rows` | No (default: `1024`) |
 | `COCOINDEX_SOURCE_MAX_INFLIGHT_BYTES` | `global_execution_options.source_max_inflight_bytes` | No |

{cocoindex-0.2.0 → cocoindex-0.2.1}/python/cocoindex/setting.py

@@ -44,8 +44,8 @@ class DatabaseConnectionSpec:
     url: str
     user: str | None = None
     password: str | None = None
-    max_connections: int = 64
-    min_connections: int = 16
+    max_connections: int = 25
+    min_connections: int = 5
 
 
 @dataclass

{cocoindex-0.2.0 → cocoindex-0.2.1}/python/cocoindex/subprocess_exec.py

@@ -22,6 +22,7 @@ import time
 from .user_app_loader import load_user_app
 from .runtime import execution_context
 import logging
+import multiprocessing as mp
 
 WATCHDOG_INTERVAL_SECONDS = 10.0
 
@@ -43,6 +44,7 @@ def _get_pool() -> ProcessPoolExecutor:
                 max_workers=1,
                 initializer=_subprocess_init,
                 initargs=(_user_apps, os.getpid()),
+                mp_context=mp.get_context("spawn"),
             )
         return _pool
 
@@ -69,6 +71,7 @@ def _restart_pool(old_pool: ProcessPoolExecutor | None = None) -> None:
             max_workers=1,
             initializer=_subprocess_init,
             initargs=(_user_apps, os.getpid()),
+            mp_context=mp.get_context("spawn"),
         )
         if prev_pool is not None:
             # Best-effort shutdown of previous pool; letting exceptions bubble up
@@ -124,8 +127,19 @@ def _start_parent_watchdog(
 
 def _subprocess_init(user_apps: list[str], parent_pid: int) -> None:
     _start_parent_watchdog(parent_pid)
+
+    # In case any user app is already in this subprocess, e.g. the subprocess is forked, we need to avoid loading it again.
+    with _pool_lock:
+        already_loaded_apps = set(_user_apps)
+
+    loaded_apps = []
     for app_target in user_apps:
-        load_user_app(app_target)
+        if app_target not in already_loaded_apps:
+            load_user_app(app_target)
+            loaded_apps.append(app_target)
+
+    with _pool_lock:
+        _user_apps.extend(loaded_apps)
 
 
 class _OnceResult:

{cocoindex-0.2.0 → cocoindex-0.2.1}/src/execution/row_indexer.rs

@@ -369,12 +369,10 @@ impl<'a> RowIndexer<'a> {
 
         if let Some(existing_version) = existing_version {
             if output.is_some() {
-                if !source_version.ordinal.is_available()
-                    || source_version.ordinal != existing_version.ordinal
-                {
-                    self.update_stats.num_updates.inc(1);
-                } else {
+                if existing_version.kind == SourceVersionKind::DifferentLogic {
                     self.update_stats.num_reprocesses.inc(1);
+                } else {
+                    self.update_stats.num_updates.inc(1);
                 }
             } else {
                 self.update_stats.num_deletions.inc(1);

{cocoindex-0.2.0 → cocoindex-0.2.1}/src/execution/stats.rs

@@ -115,8 +115,24 @@ impl std::fmt::Display for UpdateStats {
         let num_reprocesses = self.num_reprocesses.get();
         let num_source_rows = num_insertions + num_deletions + num_updates + num_reprocesses;
         if num_source_rows > 0 {
+            let mut sub_messages = Vec::new();
+            if num_insertions > 0 {
+                sub_messages.push(format!("{num_insertions} ADDED"));
+            }
+            if num_deletions > 0 {
+                sub_messages.push(format!("{num_deletions} REMOVED"));
+            }
+            if num_reprocesses > 0 {
+                sub_messages.push(format!(
+                    "{num_reprocesses} REPROCESSED due to flow or logic changes"
+                ));
+            }
+            if num_updates > 0 {
+                sub_messages.push(format!("{num_updates} UPDATED in source content"));
+            }
             messages.push(format!(
-                "{num_source_rows} source rows processed ({num_insertions} ADDED, {num_deletions} REMOVED, {num_updates} UPDATED, {num_reprocesses} REPROCESSED on flow or logic changes)",
+                "{num_source_rows} source rows processed ({})",
+                sub_messages.join(", "),
             ));
         }
 

{cocoindex-0.2.0 → cocoindex-0.2.1}/src/lib_context.rs

@@ -196,9 +196,9 @@ impl DbPools {
                 let pool_options = PgPoolOptions::new()
                     .max_connections(conn_spec.max_connections)
                     .min_connections(conn_spec.min_connections)
-                    .acquire_timeout(Duration::from_secs(5 * 60))
-                    .idle_timeout(Duration::from_secs(10 * 60))
-                    .max_lifetime(Duration::from_secs(60 * 60));
+                    .acquire_slow_level(log::LevelFilter::Info)
+                    .acquire_slow_threshold(Duration::from_secs(10))
+                    .acquire_timeout(Duration::from_secs(5 * 60));
                 let pool = pool_options
                     .connect_with(pg_options)
                     .await