cocoindex 0.2.9__tar.gz → 0.2.10__tar.gz

{cocoindex-0.2.9 → cocoindex-0.2.10}/.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
 
         - id: mypy-check
           name: mypy type check
-          entry: mypy
+          entry: mypy --cache-fine-grain
           language: system
           types: [python]
           pass_filenames: false

{cocoindex-0.2.9 → cocoindex-0.2.10}/Cargo.lock

@@ -1283,7 +1283,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.2.9"
+version = "0.2.10"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -1367,6 +1367,7 @@ dependencies = [
  "tree-sitter-rust",
  "tree-sitter-scala",
  "tree-sitter-sequel",
+ "tree-sitter-solidity",
  "tree-sitter-swift",
  "tree-sitter-toml-ng",
  "tree-sitter-typescript",
@@ -5904,6 +5905,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-solidity"
+version = "1.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eacf8875b70879f0cb670c60b233ad0b68752d9e1474e6c3ef168eea8a90b25"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree-sitter-swift"
 version = "0.7.1"

{cocoindex-0.2.9 → cocoindex-0.2.10}/Cargo.toml

@@ -2,7 +2,7 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.2.9"
+version = "0.2.10"
 edition = "2024"
 rust-version = "1.88"
 license = "Apache-2.0"
@@ -101,6 +101,7 @@ tree-sitter-toml-ng = "0.7.0"
 tree-sitter-typescript = "0.23.2"
 tree-sitter-xml = "0.7.0"
 tree-sitter-yaml = "0.7.1"
+tree-sitter-solidity = "1.2.13"
 
 globset = "0.4.16"
 unicase = "2.8.1"

{cocoindex-0.2.9 → cocoindex-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.2.9
+Version: 0.2.10
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent

{cocoindex-0.2.9 → cocoindex-0.2.10}/THIRD_PARTY_NOTICES.html

@@ -45,7 +45,7 @@
         <h2>Overview of licenses:</h2>
         <ul class="licenses-overview">
             <li><a href="#Apache-2.0">Apache License 2.0</a> (412)</li>
-            <li><a href="#MIT">MIT License</a> (124)</li>
+            <li><a href="#MIT">MIT License</a> (125)</li>
             <li><a href="#Unicode-3.0">Unicode License v3</a> (19)</li>
             <li><a href="#ISC">ISC License</a> (8)</li>
             <li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (6)</li>
@@ -2428,7 +2428,7 @@ Software.
                 <h3 id="Apache-2.0">Apache License 2.0</h3>
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
-                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.9</a></li>
+                    <li><a href=" https://crates.io/crates/cocoindex ">cocoindex 0.2.10</a></li>
                     <li><a href=" https://github.com/awesomized/crc-fast-rust ">crc-fast 1.3.0</a></li>
                     <li><a href=" https://github.com/qdrant/rust-client ">qdrant-client 1.15.0</a></li>
                 </ul>
@@ -11376,6 +11376,20 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
 IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
 </pre>
+            </li>
+            <li class="license">
+                <h3 id="MIT">MIT License</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    <li><a href=" https://github.com/JoranHonig/tree-sitter-solidity ">tree-sitter-solidity 1.2.13</a></li>
+                </ul>
+                <pre class="license-text">Copyright (c) 2020 Joran Honig
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the &quot;Software&quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</pre>
             </li>
             <li class="license">
                 <h3 id="MIT">MIT License</h3>

{cocoindex-0.2.9 → cocoindex-0.2.10}/docs/docs/core/settings.mdx

@@ -10,49 +10,82 @@ import TabItem from '@theme/TabItem';
 
 Certain settings need to be provided for CocoIndex to work, e.g. database connections, app namespace, etc.
 
-## Launch CocoIndex
+## Configure CocoIndex Settings
 
-You have two ways to launch CocoIndex:
+Note that in general, you have two ways to launch CocoIndex:
 
+*   Call CocoIndex APIs from your own Python application or library.
 *   Use [Cocoindex CLI](cli). It's handy for most routine indexing building and management tasks.
-    It will load settings from environment variables, either already set in your environment, or specified in `.env` file.
-    See [CLI](cli#environment-variables) for more details.
 
-*   Call CocoIndex functionality from your own Python application or library.
-    It's needed when you want to leverage CocoIndex support for query, or have your custom logic to trigger indexing, etc.
 
-    <Tabs>
-    <TabItem value="python" label="Python" default>
+CocoIndex exposes process-level settings specified by `cocoindex.Settings` dataclass.
+Settings can be configured in three different ways.
+In the following sections, the later ones will override the earlier ones.
 
-    You need to explicitly call `cocoindex.init()` before doing anything with CocoIndex, and settings will be loaded during the call.
+### Environment Variables
 
-    *   If it's called without any argument, it will load settings from environment variables.
-        Only existing environment variables already set in your environment will be used.
-        If you want to load environment variables from a specific `.env` file, consider call `load_dotenv()` provided by the [`python-dotenv`](https://github.com/theskumar/python-dotenv) package.
+The simplest approach is to set corresponding environment variables.
+See [List of Environment Variables](#list-of-environment-variables) for specific environment variables.
 
-        ```py
-        from dotenv import load_dotenv
-        import cocoindex
+:::tip
 
-        load_dotenv()
-        cocoindex.init()
-        ```
+You can consider place a `.env` file in your directory.
+The [CLI](cli#environment-variables) will load environment variables from the `.env` file (see [CLI](cli#environment-variables) for more details).
+From your own main module, you can also load environment variables with a package like [`python-dotenv`](https://github.com/theskumar/python-dotenv).
 
-    *   It takes an optional `cocoindex.Settings` dataclass object as argument, so you can also construct settings explicitly and pass to it:
+:::
+
+### Setting Function
+
+A more flexible approach is to provide a setting function that returns a `cocoindex.Settings` dataclass object.
+The setting function can have any name, and needs to be decorated with the `@cocoindex.settings` decorator, for example:
+
+```py
+@cocoindex.settings
+def cocoindex_settings() -> cocoindex.Settings:
+    return cocoindex.Settings(
+        database=cocoindex.DatabaseConnectionSpec(
+            url="postgres://cocoindex:cocoindex@localhost/cocoindex"
+        )
+    )
+```
+
+This setting function will be called once when CocoIndex is initialized.
+Once the settings function is provided, environment variables will be ignored.
 
-        ```py
-        import cocoindex
+### `cocoindex.init()` function
 
-        cocoindex.init(
-            cocoindex.Settings(
-                database=cocoindex.DatabaseConnectionSpec(
-                    url="postgres://cocoindex:cocoindex@localhost/cocoindex"
-                )
-            )
+You can also call `cocoindex.init()` with a `cocoindex.Settings` dataclass object as argument, for example:
+
+```py
+cocoindex.init(
+    cocoindex.Settings(
+        database=cocoindex.DatabaseConnectionSpec(
+            url="postgres://cocoindex:cocoindex@localhost/cocoindex"
         )
-        ```
-    </TabItem>
-    </Tabs>
+    )
+)
+```
+
+For example, you can call it in the main function of your application.
+Once the `cocoindex.init()` is called with a `cocoindex.Settings` dataclass object as argument, the `@cocoindex.settings` function and environment variables will be ignored.
+
+This is more flexible, as you can more easily construct `cocoindex.Settings` based on other stuffs you loaded earlier.
+But be careful that if you call `cocoindex.init()` only under the path of main (e.g. within `if __name__ == "__main__":` guard), it won't be executed when you're using CocoIndex CLI, as it won't execute your main logic.
+
+:::info
+
+`cocoindex.init()` is optional:
+
+-   You can call `cocoindex.init()` with a `cocoindex.Settings` dataclass object as argument, or without any argument.
+    When without argument, the settings will be loaded from the `@cocoindex.settings` function or environment variables.
+
+-   You don't have to explicitly call `cocoindex.init()`.
+    CocoIndex will be automatically initialized when needed, e.g. when any method of any flow is called the first time.
+    But calling `cocoindex.init()` explicitly (usually at startup time, e.g. in the main function of your application) has the benefit of making sure CocoIndex library is initialized and any potential exceptions are raised earlier before proceeding with the application.
+    If you need this clarity, you can call it explicitly even if you don't want to provide settings by the `cocoindex.init()` call.
+
+:::
 
 ## List of Settings
 

{cocoindex-0.2.9 → cocoindex-0.2.10}/docs/docs/ops/functions.md

@@ -102,6 +102,7 @@ Currently, `SplitRecursively` supports the following languages:
 | Ruby | | `.rb` |
 | Rust | RS | `.rs` |
 | Scala | | `.scala` |
+| Solidity | | `.sol` |
 | SQL | | `.sql` |
 | Swift | | `.swift` |
 | TOML | | `.toml` |

{cocoindex-0.2.9 → cocoindex-0.2.10}/python/cocoindex/__init__.py

@@ -6,7 +6,12 @@ from . import functions, sources, targets, cli, utils
 
 from . import targets as storages  # Deprecated: Use targets instead
 
-from .auth_registry import AuthEntryReference, add_auth_entry, add_transient_auth_entry
+from .auth_registry import (
+    AuthEntryReference,
+    add_auth_entry,
+    add_transient_auth_entry,
+    ref_auth_entry,
+)
 from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
 from .flow import flow_def
 from .flow import EvaluateAndDumpOptions, GeneratedField
@@ -14,7 +19,7 @@ from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions, FlowUpdaterStatusUpda
 from .flow import open_flow
 from .flow import add_flow_def, remove_flow  # DEPRECATED
 from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
-from .lib import init, start_server, stop
+from .lib import settings, init, start_server, stop
 from .llm import LlmSpec, LlmApiType
 from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
 from .setting import DatabaseConnectionSpec, Settings, ServerSettings
@@ -65,6 +70,7 @@ __all__ = [
     "setup_all_flows",
     "drop_all_flows",
     # Lib
+    "settings",
     "init",
     "start_server",
     "stop",

{cocoindex-0.2.9 → cocoindex-0.2.10}/python/cocoindex/cli.py

@@ -91,8 +91,6 @@ def _load_user_app(app_target: str) -> None:
 
 
 def _initialize_cocoindex_in_process() -> None:
-    settings = setting.Settings.from_env()
-    lib.init(settings)
     atexit.register(lib.stop)
 
 

{cocoindex-0.2.9 → cocoindex-0.2.10}/python/cocoindex/flow.py

@@ -694,16 +694,11 @@ class Flow:
     """
 
     _name: str
-    _full_name: str
     _lazy_engine_flow: Callable[[], _engine.Flow] | None
 
-    def __init__(
-        self, name: str, full_name: str, engine_flow_creator: Callable[[], _engine.Flow]
-    ):
+    def __init__(self, name: str, engine_flow_creator: Callable[[], _engine.Flow]):
         validate_flow_name(name)
-        validate_full_flow_name(full_name)
         self._name = name
-        self._full_name = full_name
         engine_flow = None
         lock = Lock()
 
@@ -762,7 +757,7 @@ class Flow:
         """
         Get the full name of the flow.
         """
-        return self._full_name
+        return get_flow_full_name(self._name)
 
     def update(self, /, *, reexport_targets: bool = False) -> _engine.IndexUpdateInfo:
         """
@@ -861,9 +856,10 @@ def _create_lazy_flow(
     The flow will be built the first time when it's really needed.
     """
     flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
-    flow_full_name = get_flow_full_name(flow_name)
 
     def _create_engine_flow() -> _engine.Flow:
+        flow_full_name = get_flow_full_name(flow_name)
+        validate_full_flow_name(flow_full_name)
         flow_builder_state = _FlowBuilderState(flow_full_name)
         root_scope = DataScope(
             flow_builder_state, flow_builder_state.engine_flow_builder.root_scope()
@@ -873,7 +869,7 @@ def _create_lazy_flow(
             execution_context.event_loop
         )
 
-    return Flow(flow_name, flow_full_name, _create_engine_flow)
+    return Flow(flow_name, _create_engine_flow)
 
 
 _flows_lock = Lock()

cocoindex-0.2.10/python/cocoindex/lib.py

@@ -0,0 +1,75 @@
+"""
+Library level functions and states.
+"""
+
+import threading
+import warnings
+
+from . import _engine  # type: ignore
+from . import flow, setting
+from .convert import dump_engine_object
+from .validation import validate_app_namespace_name
+from typing import Any, Callable, overload
+
+
+def prepare_settings(settings: setting.Settings) -> Any:
+    """Prepare the settings for the engine."""
+    if settings.app_namespace:
+        validate_app_namespace_name(settings.app_namespace)
+    return dump_engine_object(settings)
+
+
+_engine.set_settings_fn(lambda: prepare_settings(setting.Settings.from_env()))
+
+
+_prev_settings_fn: Callable[[], setting.Settings] | None = None
+_prev_settings_fn_lock: threading.Lock = threading.Lock()
+
+
+@overload
+def settings(fn: Callable[[], setting.Settings]) -> Callable[[], setting.Settings]: ...
+@overload
+def settings(
+    fn: None,
+) -> Callable[[Callable[[], setting.Settings]], Callable[[], setting.Settings]]: ...
+def settings(fn: Callable[[], setting.Settings] | None = None) -> Any:
+    """
+    Decorate a function that returns a settings.Settings object.
+    It registers the function as a settings provider.
+    """
+
+    def _inner(fn: Callable[[], setting.Settings]) -> Callable[[], setting.Settings]:
+        global _prev_settings_fn  # pylint: disable=global-statement
+        with _prev_settings_fn_lock:
+            if _prev_settings_fn is not None:
+                warnings.warn(
+                    f"Setting a new settings function will override the previous one {_prev_settings_fn}."
+                )
+            _prev_settings_fn = fn
+        _engine.set_settings_fn(lambda: prepare_settings(fn()))
+        return fn
+
+    if fn is not None:
+        return _inner(fn)
+    else:
+        return _inner
+
+
+def init(settings: setting.Settings | None = None) -> None:
+    """
+    Initialize the cocoindex library.
+
+    If the settings are not provided, they are loaded from the environment variables.
+    """
+    _engine.init(prepare_settings(settings) if settings is not None else None)
+
+
+def start_server(settings: setting.ServerSettings) -> None:
+    """Start the cocoindex server."""
+    flow.ensure_all_flows_built()
+    _engine.start_server(settings.__dict__)
+
+
+def stop() -> None:
+    """Stop the cocoindex library."""
+    _engine.stop()

{cocoindex-0.2.9 → cocoindex-0.2.10}/python/cocoindex/setting.py

@@ -6,16 +6,15 @@ import os
 
 from typing import Callable, Self, Any, overload
 from dataclasses import dataclass
-from .validation import validate_app_namespace_name
-
-_app_namespace: str = ""
+from . import _engine  # type: ignore
 
 
 def get_app_namespace(*, trailing_delimiter: str | None = None) -> str:
     """Get the application namespace. Append the `trailing_delimiter` if not empty."""
-    if _app_namespace == "" or trailing_delimiter is None:
-        return _app_namespace
-    return f"{_app_namespace}{trailing_delimiter}"
+    app_namespace: str = _engine.get_app_namespace()
+    if app_namespace == "" or trailing_delimiter is None:
+        return app_namespace
+    return f"{app_namespace}{trailing_delimiter}"
 
 
 def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
@@ -26,14 +25,6 @@ def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
     return (parts[0], parts[1])
 
 
-def set_app_namespace(app_namespace: str) -> None:
-    """Set the application namespace."""
-    if app_namespace:
-        validate_app_namespace_name(app_namespace)
-    global _app_namespace  # pylint: disable=global-statement
-    _app_namespace = app_namespace
-
-
 @dataclass
 class DatabaseConnectionSpec:
     """

{cocoindex-0.2.9 → cocoindex-0.2.10}/src/builder/analyzer.rs

@@ -995,7 +995,7 @@ pub async fn analyze_flow(
     impl Future<Output = Result<ExecutionPlan>> + Send + use<>,
 )> {
     let analyzer_ctx = AnalyzerContext {
-        lib_ctx: get_lib_context()?,
+        lib_ctx: get_lib_context().await?,
         flow_ctx,
     };
     let root_data_scope = Arc::new(Mutex::new(DataScopeBuilder::new()));
@@ -1109,7 +1109,7 @@ pub async fn analyze_transient_flow<'a>(
 )> {
     let mut root_data_scope = DataScopeBuilder::new();
     let analyzer_ctx = AnalyzerContext {
-        lib_ctx: get_lib_context()?,
+        lib_ctx: get_lib_context().await?,
         flow_ctx,
     };
     let mut input_fields = vec![];

{cocoindex-0.2.9 → cocoindex-0.2.10}/src/builder/flow_builder.rs

@@ -18,7 +18,7 @@ use crate::{
     },
     lib_context::LibContext,
     ops::interface::FlowInstanceContext,
-    py::IntoPyResult,
+    py::{AnyhowIntoPyResult, IntoPyResult},
 };
 use crate::{lib_context::FlowContext, py};
 
@@ -247,8 +247,12 @@ pub struct FlowBuilder {
 #[pymethods]
 impl FlowBuilder {
     #[new]
-    pub fn new(name: &str) -> PyResult<Self> {
-        let lib_context = get_lib_context().into_py_result()?;
+    pub fn new(py: Python<'_>, name: &str) -> PyResult<Self> {
+        let lib_context = py
+            .allow_threads(|| -> anyhow::Result<Arc<LibContext>> {
+                get_runtime().block_on(get_lib_context())
+            })
+            .into_py_result()?;
         let root_op_scope = OpScope::new(
             spec::ROOT_SCOPE_NAME.to_string(),
             None,

{cocoindex-0.2.9 → cocoindex-0.2.10}/src/execution/db_tracking.rs

@@ -294,6 +294,8 @@ pub struct TrackedSourceKeyMetadata {
     pub processed_source_ordinal: Option<i64>,
     pub processed_source_fp: Option<Vec<u8>>,
     pub process_logic_fingerprint: Option<Vec<u8>>,
+    pub max_process_ordinal: Option<i64>,
+    pub process_ordinal: Option<i64>,
 }
 
 pub struct ListTrackedSourceKeyMetadataState {
@@ -314,7 +316,9 @@ impl ListTrackedSourceKeyMetadataState {
         pool: &'a PgPool,
     ) -> impl Stream<Item = Result<TrackedSourceKeyMetadata, sqlx::Error>> + 'a {
         self.query_str = format!(
-            "SELECT source_key, processed_source_ordinal, {}, process_logic_fingerprint FROM {} WHERE source_id = $1",
+            "SELECT \
+            source_key, processed_source_ordinal, {}, process_logic_fingerprint, max_process_ordinal, process_ordinal \
+            FROM {} WHERE source_id = $1",
             if db_setup.has_fast_fingerprint_column {
                 "processed_source_fp"
             } else {

{cocoindex-0.2.9 → cocoindex-0.2.10}/src/execution/db_tracking_setup.rs

@@ -252,7 +252,7 @@ impl ResourceSetupChange for TrackingTableSetupChange {
 
 impl TrackingTableSetupChange {
     pub async fn apply_change(&self) -> Result<()> {
-        let lib_context = get_lib_context()?;
+        let lib_context = get_lib_context().await?;
         let pool = lib_context.require_builtin_db_pool()?;
         if let Some(desired) = &self.desired_state {
             for lagacy_name in self.legacy_tracking_table_names.iter() {

{cocoindex-0.2.9 → cocoindex-0.2.10}/src/execution/evaluator.rs

@@ -5,7 +5,7 @@ use futures::future::try_join_all;
 
 use crate::base::value::EstimatedByteSize;
 use crate::builder::{AnalyzedTransientFlow, plan::*};
-use crate::py::IntoPyResult;
+use crate::py::AnyhowIntoPyResult;
 use crate::{
     base::{schema, value},
     utils::immutable::RefList,
@@ -73,6 +73,7 @@ impl ScopeValueBuilder {
             .zip(&mut builder.fields)
         {
             r.set(augmented_value(v, &t.value_type.typ)?)
+                .map_err(|_| anyhow!("Value of field `{}` is already set", t.name))
                 .into_py_result()?;
         }
         Ok(builder)