cocoindex 0.1.60__tar.gz → 0.1.62__tar.gz

{cocoindex-0.1.60 → cocoindex-0.1.62}/.gitignore

@@ -18,3 +18,5 @@ dist/
 
 # Output of `cocoindex eval`
 examples/**/eval_*
+
+/.vscode

{cocoindex-0.1.60 → cocoindex-0.1.62}/Cargo.lock

@@ -1040,7 +1040,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.60"
+version = "0.1.62"
 dependencies = [
  "anyhow",
  "async-openai",

{cocoindex-0.1.60 → cocoindex-0.1.62}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.60"
+version = "0.1.62"
 edition = "2024"
 rust-version = "1.86"
 

{cocoindex-0.1.60 → cocoindex-0.1.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.60
+Version: 0.1.62
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0

{cocoindex-0.1.60 → cocoindex-0.1.62}/docs/docs/core/cli.mdx

@@ -47,12 +47,6 @@ CocoIndex CLI supports the following global options:
 * `--version`: Show the CocoIndex version and exit.
 * `--help`: Show the main help message and exit.
 
-:::caution Deprecated Usage
-
-The old method of invoking the CLI using `python main.py cocoindex ...` via the `@cocoindex.main_fn()` decorator is now deprecated. Please remove `@cocoindex.main_fn()` from your scripts and use the standalone cocoindex command as described.
-
-:::
-
 ## Subcommands
 
 The following subcommands are available:

{cocoindex-0.1.60 → cocoindex-0.1.62}/python/cocoindex/__init__.py

@@ -12,7 +12,7 @@ from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
 from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions
 from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
-from .lib import init, start_server, stop, main_fn
+from .lib import init, start_server, stop
 from .llm import LlmSpec, LlmApiType
 from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
 from .setting import DatabaseConnectionSpec, Settings, ServerSettings
@@ -59,7 +59,6 @@ __all__ = [
     "init",
     "start_server",
     "stop",
-    "main_fn",
     # LLM
     "LlmSpec",
     "LlmApiType",

{cocoindex-0.1.60 → cocoindex-0.1.62}/python/cocoindex/cli.py

@@ -408,6 +408,12 @@ def update(
     app_ref, flow_name = _parse_app_flow_specifier(app_flow_specifier)
     _load_user_app(app_ref)
 
+    if live:
+        click.secho(
+            "NOTE: Flow code changes will NOT be reflected in the server until you restart it.",
+            fg="red",
+        )
+
     options = flow.FlowLiveUpdaterOptions(live_mode=live, print_stats=not quiet)
     if flow_name is None:
         if setup:
@@ -597,6 +603,10 @@ def server(
             ),
         )
     else:
+        click.secho(
+            "NOTE: Flow code changes will NOT be reflected in the server until you restart it. Use --reload to enable auto-reload.",
+            fg="red",
+        )
         _run_server(*args)
 
 
@@ -633,9 +643,6 @@ def _run_server(
     if address is not None:
         server_settings.address = address
 
-    if COCOINDEX_HOST in cors_origins:
-        click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
-
     if run_setup:
         _setup_flows(
             flow.flows().values(),
@@ -649,6 +656,9 @@ def _run_server(
 
     lib.start_server(server_settings)
 
+    if COCOINDEX_HOST in cors_origins:
+        click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
+
     click.secho("Press Ctrl+C to stop the server.", fg="yellow")
 
     shutdown_event = threading.Event()

{cocoindex-0.1.60 → cocoindex-0.1.62}/python/cocoindex/flow.py

@@ -416,6 +416,11 @@ class _SourceRefreshOptions:
     refresh_interval: datetime.timedelta | None = None
 
 
+@dataclass
+class _ExecutionOptions:
+    max_inflight_count: int | None = None
+
+
 class FlowBuilder:
     """
     A flow builder is used to build a flow.
@@ -439,6 +444,7 @@ class FlowBuilder:
         *,
         name: str | None = None,
         refresh_interval: datetime.timedelta | None = None,
+        max_inflight_count: int | None = None,
     ) -> DataSlice[T]:
         """
         Import a source to the flow.
@@ -454,9 +460,12 @@ class FlowBuilder:
                 self._state.field_name_builder.build_name(
                     name, prefix=_to_snake_case(_spec_kind(spec)) + "_"
                 ),
-                dump_engine_object(
+                refresh_options=dump_engine_object(
                     _SourceRefreshOptions(refresh_interval=refresh_interval)
                 ),
+                execution_options=dump_engine_object(
+                    _ExecutionOptions(max_inflight_count=max_inflight_count)
+                ),
             ),
             name,
         )

cocoindex-0.1.62/python/cocoindex/lib.py

@@ -0,0 +1,32 @@
+"""
+Library level functions and states.
+"""
+
+import warnings
+from typing import Callable, Any
+
+from . import _engine  # type: ignore
+from . import flow, setting
+from .convert import dump_engine_object
+
+
+def init(settings: setting.Settings | None = None) -> None:
+    """
+    Initialize the cocoindex library.
+
+    If the settings are not provided, they are loaded from the environment variables.
+    """
+    settings = settings or setting.Settings.from_env()
+    _engine.init(dump_engine_object(settings))
+    setting.set_app_namespace(settings.app_namespace)
+
+
+def start_server(settings: setting.ServerSettings) -> None:
+    """Start the cocoindex server."""
+    flow.ensure_all_flows_built()
+    _engine.start_server(settings.__dict__)
+
+
+def stop() -> None:
+    """Stop the cocoindex library."""
+    _engine.stop()

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/base/spec.rs

@@ -253,6 +253,11 @@ impl SpecFormatter for OpSpec {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ExecutionOptions {
+    pub max_inflight_count: Option<u32>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct SourceRefreshOptions {
     pub refresh_interval: Option<std::time::Duration>,
@@ -274,6 +279,9 @@ pub struct ImportOpSpec {
 
     #[serde(default)]
     pub refresh_options: SourceRefreshOptions,
+
+    #[serde(default)]
+    pub execution_options: ExecutionOptions,
 }
 
 impl SpecFormatter for ImportOpSpec {

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/builder/analyzer.rs

@@ -697,6 +697,9 @@ impl AnalyzerContext {
                 primary_key_type,
                 name: op_name,
                 refresh_options: import_op.spec.refresh_options,
+                concurrency_controller: utils::ConcurrencyController::new(
+                    import_op.spec.execution_options.max_inflight_count,
+                ),
             })
         };
         Ok(result_fut)

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/builder/flow_builder.rs

@@ -288,7 +288,7 @@ impl FlowBuilder {
         OpScopeRef(self.root_op_scope.clone())
     }
 
-    #[pyo3(signature = (kind, op_spec, target_scope, name, refresh_options=None))]
+    #[pyo3(signature = (kind, op_spec, target_scope, name, refresh_options=None, execution_options=None))]
     pub fn add_source(
         &mut self,
         py: Python<'_>,
@@ -297,6 +297,7 @@ impl FlowBuilder {
         target_scope: Option<OpScopeRef>,
         name: String,
         refresh_options: Option<py::Pythonized<spec::SourceRefreshOptions>>,
+        execution_options: Option<py::Pythonized<spec::ExecutionOptions>>,
     ) -> PyResult<DataSlice> {
         if let Some(target_scope) = target_scope {
             if *target_scope != self.root_op_scope {
@@ -313,6 +314,9 @@ impl FlowBuilder {
                     spec: op_spec.into_inner(),
                 },
                 refresh_options: refresh_options.map(|o| o.into_inner()).unwrap_or_default(),
+                execution_options: execution_options
+                    .map(|o| o.into_inner())
+                    .unwrap_or_default(),
             },
         };
         let analyzer_ctx = AnalyzerContext {

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/builder/plan.rs

@@ -56,6 +56,7 @@ pub struct AnalyzedImportOp {
     pub output: AnalyzedOpOutput,
     pub primary_key_type: schema::ValueType,
     pub refresh_options: spec::SourceRefreshOptions,
+    pub concurrency_controller: utils::ConcurrencyController,
 }
 
 pub struct AnalyzedFunctionExecInfo {

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/execution/source_indexer.rs

@@ -282,6 +282,7 @@ impl SourceIndexingContext {
             state.scan_generation
         };
         while let Some(row) = rows_stream.next().await {
+            let _ = import_op.concurrency_controller.acquire().await?;
             for row in row? {
                 self.process_source_key_if_newer(
                     row.key,

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/ops/functions/split_recursively.rs

@@ -918,16 +918,14 @@ impl SimpleFunctionExecutor for Executor {
         let table = output
             .into_iter()
             .map(|chunk_output| {
+                let output_start = chunk_output.start_pos.output.unwrap();
+                let output_end = chunk_output.end_pos.output.unwrap();
                 (
-                    RangeValue::new(
-                        chunk_output.start_pos.byte_offset,
-                        chunk_output.end_pos.byte_offset,
-                    )
-                    .into(),
+                    RangeValue::new(output_start.char_offset, output_end.char_offset).into(),
                     fields_value!(
                         Arc::<str>::from(chunk_output.text),
-                        chunk_output.start_pos.output.unwrap().into_output(),
-                        chunk_output.end_pos.output.unwrap().into_output()
+                        output_start.into_output(),
+                        output_end.into_output()
                     )
                     .into(),
                 )

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/ops/targets/postgres.rs

@@ -197,8 +197,7 @@ impl ExportContext {
             .map(|f| format!("\"{}\"", f.name))
             .collect::<Vec<_>>()
             .join(", ");
-        let value_fields = value_fields_schema
-            .iter()
+        let all_fields = (key_fields_schema.iter().chain(value_fields_schema.iter()))
             .map(|f| format!("\"{}\"", f.name))
             .collect::<Vec<_>>()
             .join(", ");
@@ -211,15 +210,15 @@ impl ExportContext {
         Ok(Self {
             db_ref,
             db_pool,
+            upsert_sql_prefix: format!("INSERT INTO {table_name} ({all_fields}) VALUES "),
+            upsert_sql_suffix: if value_fields_schema.is_empty() {
+                format!(" ON CONFLICT ({key_fields}) DO NOTHING;")
+            } else {
+                format!(" ON CONFLICT ({key_fields}) DO UPDATE SET {set_value_fields};")
+            },
+            delete_sql_prefix: format!("DELETE FROM {table_name} WHERE "),
             key_fields_schema,
             value_fields_schema,
-            upsert_sql_prefix: format!(
-                "INSERT INTO {table_name} ({key_fields}, {value_fields}) VALUES "
-            ),
-            upsert_sql_suffix: format!(
-                " ON CONFLICT ({key_fields}) DO UPDATE SET {set_value_fields};"
-            ),
-            delete_sql_prefix: format!("DELETE FROM {table_name} WHERE "),
         })
     }
 }

cocoindex-0.1.62/src/utils/concur_control.rs

@@ -0,0 +1,30 @@
+use crate::prelude::*;
+
+use tokio::sync::{Semaphore, SemaphorePermit};
+
+pub struct ConcurrencyController {
+    inflight_count_sem: Option<Semaphore>,
+}
+
+pub struct ConcurrencyControllerPermit<'a> {
+    _inflight_count_permit: Option<SemaphorePermit<'a>>,
+}
+
+impl ConcurrencyController {
+    pub fn new(max_inflight_count: Option<u32>) -> Self {
+        Self {
+            inflight_count_sem: max_inflight_count.map(|max| Semaphore::new(max as usize)),
+        }
+    }
+
+    pub async fn acquire<'a>(&'a self) -> Result<ConcurrencyControllerPermit<'a>> {
+        let inflight_count_permit = if let Some(sem) = &self.inflight_count_sem {
+            Some(sem.acquire().await?)
+        } else {
+            None
+        };
+        Ok(ConcurrencyControllerPermit {
+            _inflight_count_permit: inflight_count_permit,
+        })
+    }
+}

{cocoindex-0.1.60 → cocoindex-0.1.62}/src/utils/mod.rs

@@ -3,3 +3,6 @@ pub mod fingerprint;
 pub mod immutable;
 pub mod retryable;
 pub mod yaml_ser;
+
+mod concur_control;
+pub use concur_control::ConcurrencyController;

cocoindex-0.1.60/.vscode/settings.json

@@ -1,9 +0,0 @@
-{
-    "cSpell.words": [
-        "cocoindex",
-        "reindexing",
-        "timedelta"
-    ],
-    "editor.formatOnSave": true,
-    "python.formatting.provider": "ruff"
-}

cocoindex-0.1.60/python/cocoindex/lib.py

@@ -1,75 +0,0 @@
-"""
-Library level functions and states.
-"""
-
-import warnings
-from typing import Callable, Any
-
-from . import _engine  # type: ignore
-from . import flow, setting
-from .convert import dump_engine_object
-
-
-def init(settings: setting.Settings | None = None) -> None:
-    """
-    Initialize the cocoindex library.
-
-    If the settings are not provided, they are loaded from the environment variables.
-    """
-    settings = settings or setting.Settings.from_env()
-    _engine.init(dump_engine_object(settings))
-    setting.set_app_namespace(settings.app_namespace)
-
-
-def start_server(settings: setting.ServerSettings) -> None:
-    """Start the cocoindex server."""
-    flow.ensure_all_flows_built()
-    _engine.start_server(settings.__dict__)
-
-
-def stop() -> None:
-    """Stop the cocoindex library."""
-    _engine.stop()
-
-
-def main_fn(
-    settings: Any | None = None,
-    cocoindex_cmd: str | None = None,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    """
-    DEPRECATED: The @cocoindex.main_fn() decorator is obsolete and has no effect.
-    It will be removed in a future version, which will cause an AttributeError.
-
-    Please remove this decorator from your code and use the standalone 'cocoindex' CLI.
-    See the updated CLI usage examples in the warning message.
-    """
-    warnings.warn(
-        "\n\n"
-        "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-        "CRITICAL DEPRECATION NOTICE from CocoIndex:\n"
-        "The @cocoindex.main_fn() decorator in your script is DEPRECATED and IGNORED.\n"
-        "It provides NO functionality and will be REMOVED entirely in a future version.\n"
-        "If not removed, your script will FAIL with an AttributeError in the future.\n\n"
-        "ACTION REQUIRED: Please REMOVE @cocoindex.main_fn() from your Python script.\n\n"
-        "To use CocoIndex, invoke the standalone 'cocoindex' CLI."
-        " Examples of new CLI usage:\n\n"
-        "  To list flows from 'main.py' (previously 'python main.py cocoindex ls'):\n"
-        "    cocoindex ls main.py\n\n"
-        "  To list all persisted flows (previously 'python main.py cocoindex ls --all'):\n"
-        "    cocoindex ls\n\n"
-        "  To show 'MyFlow' defined in 'main.py' (previously 'python main.py cocoindex show MyFlow'):\n"
-        "    cocoindex show main.py:MyFlow\n\n"
-        "  To update all flows in 'my_package.flows_module':\n"
-        "    cocoindex update my_package.flows_module\n\n"
-        "  To update 'SpecificFlow' in 'my_package.flows_module':\n"
-        "    cocoindex update my_package.flows_module:SpecificFlow\n\n"
-        "See cocoindex <command> --help for more details.\n"
-        "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-
-    def _main_wrapper(fn: Callable[..., Any]) -> Callable[..., Any]:
-        return fn
-
-    return _main_wrapper