cocoindex 0.1.37__cp312-cp312-macosx_11_0_arm64.whl → 0.1.38__cp312-cp312-macosx_11_0_arm64.whl

cocoindex/cli.py

@@ -16,7 +16,7 @@ def cli():
 @cli.command()
 @click.option(
     "-a", "--all", "show_all", is_flag=True, show_default=True, default=False,
-    help="Also show all flows with persisted setup, even if not defined in the current process.")
+    help="Also show all flows with persisted setup under the current app namespace, even if not defined in the current process.")
 def ls(show_all: bool):
     """
     List all flows.
@@ -65,7 +65,7 @@ def show(flow_name: str | None, color: bool, verbose: bool):
 
     console.print()
     table = Table(
-        title=f"Schema for Flow: {flow.name}",
+        title=f"Schema for Flow: {flow.full_name}",
         show_header=True,
         header_style="bold magenta"
     )
@@ -108,7 +108,7 @@ def drop(flow_name: tuple[str, ...], drop_all: bool):
     if drop_all:
         flow_names = flow_names_with_setup()
     elif len(flow_name) == 0:
-        flow_names = [fl.name for fl in flow.flows()]
+        flow_names = flow.flow_names()
     else:
         flow_names = list(flow_name)
     setup_status = drop_setup(flow_names)
@@ -160,7 +160,7 @@ def evaluate(flow_name: str | None, output_dir: str | None, cache: bool = True):
     """
     fl = _flow_by_name(flow_name)
     if output_dir is None:
-        output_dir = f"eval_{fl.name}_{datetime.datetime.now().strftime('%y%m%d_%H%M%S')}"
+        output_dir = f"eval_{setting.get_app_namespace(trailing_delimiter='_')}{flow_name}_{datetime.datetime.now().strftime('%y%m%d_%H%M%S')}"
     options = flow.EvaluateAndDumpOptions(output_dir=output_dir, use_cache=cache)
     fl.evaluate_and_dump(options)
 

cocoindex/flow.py

@@ -19,6 +19,7 @@ from rich.tree import Tree
 from . import _engine
 from . import index
 from . import op
+from . import setting
 from .convert import dump_engine_object
 from .typing import encode_enriched_type
 from .runtime import execution_context
@@ -310,7 +311,7 @@ class _FlowBuilderState:
 
     def __init__(self, /, name: str | None = None):
         flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
-        self.engine_flow_builder = _engine.FlowBuilder(flow_name)
+        self.engine_flow_builder = _engine.FlowBuilder(get_full_flow_name(flow_name))
         self.field_name_builder = _NameBuilder()
 
     def get_data_slice(self, v: Any) -> _engine.DataSlice:
@@ -481,7 +482,7 @@ class Flow:
         Render the flow spec as a styled rich Tree with hierarchical structure.
         """
         spec = self._get_spec(verbose=verbose)
-        tree = Tree(f"Flow: {self.name}", style="cyan")
+        tree = Tree(f"Flow: {self.full_name}", style="cyan")
 
         def build_tree(label: str, lines: list):
             node = Tree(label, style="bold magenta" if lines else "cyan")
@@ -508,9 +509,9 @@ class Flow:
         return repr(self._lazy_engine_flow())
 
     @property
-    def name(self) -> str:
+    def full_name(self) -> str:
         """
-        Get the name of the flow.
+        Get the full name of the flow.
         """
         return self._lazy_engine_flow().name()
 
@@ -566,8 +567,16 @@ def _create_lazy_flow(name: str | None, fl_def: Callable[[FlowBuilder, DataScope
 _flows_lock = Lock()
 _flows: dict[str, Flow] = {}
 
+def get_full_flow_name(name: str) -> str:
+    """
+    Get the full name of a flow.
+    """
+    return f"{setting.get_app_namespace(trailing_delimiter='.')}{name}"
+
 def add_flow_def(name: str, fl_def: Callable[[FlowBuilder, DataScope], None]) -> Flow:
     """Add a flow definition to the cocoindex library."""
+    if not all(c.isalnum() or c == '_' for c in name):
+        raise ValueError(f"Flow name '{name}' contains invalid characters. Only alphanumeric characters and underscores are allowed.")
     with _flows_lock:
         if name in _flows:
             raise KeyError(f"Flow with name {name} already exists")
@@ -587,12 +596,12 @@ def flow_names() -> list[str]:
     with _flows_lock:
         return list(_flows.keys())
 
-def flows() -> list[Flow]:
+def flows() -> dict[str, Flow]:
     """
     Get all flows.
     """
     with _flows_lock:
-        return list(_flows.values())
+        return dict(_flows)
 
 def flow_by_name(name: str) -> Flow:
     """
@@ -605,14 +614,13 @@ def ensure_all_flows_built() -> None:
     """
     Ensure all flows are built.
     """
-    for fl in flows():
-        fl.internal_flow()
+    execution_context.run(ensure_all_flows_built_async())
 
 async def ensure_all_flows_built_async() -> None:
     """
     Ensure all flows are built.
     """
-    for fl in flows():
+    for fl in flows().values():
         await fl.internal_flow_async()
 
 def update_all_flows(options: FlowLiveUpdaterOptions) -> dict[str, _engine.IndexUpdateInfo]:
@@ -626,13 +634,13 @@ async def update_all_flows_async(options: FlowLiveUpdaterOptions) -> dict[str, _
     Update all flows.
     """
     await ensure_all_flows_built_async()
-    async def _update_flow(fl: Flow) -> _engine.IndexUpdateInfo:
+    async def _update_flow(name: str, fl: Flow) -> tuple[str, _engine.IndexUpdateInfo]:
         async with FlowLiveUpdater(fl, options) as updater:
             await updater.wait_async()
-            return updater.update_stats()
+            return (name, updater.update_stats())
     fls = flows()
-    all_stats = await asyncio.gather(*(_update_flow(fl) for fl in fls))
-    return {fl.name: stats for fl, stats in zip(fls, all_stats)}
+    all_stats = await asyncio.gather(*(_update_flow(name, fl) for (name, fl) in fls.items()))
+    return dict(all_stats)
 
 _transient_flow_name_builder = _NameBuilder()
 class TransientFlow:

cocoindex/lib.py

@@ -15,6 +15,7 @@ from .convert import dump_engine_object
 def init(settings: setting.Settings):
     """Initialize the cocoindex library."""
     _engine.init(dump_engine_object(settings))
+    setting.set_app_namespace(settings.app_namespace)
 
 
 def start_server(settings: setting.ServerSettings):

cocoindex/setting.py

@@ -6,6 +6,25 @@ import os
 from typing import Callable, Self, Any, overload
 from dataclasses import dataclass
 
+_app_namespace: str = ''
+
+def get_app_namespace(*, trailing_delimiter: str | None = None) -> str:
+    """Get the application namespace. Append the `trailing_delimiter` if not empty."""
+    if _app_namespace == '' or trailing_delimiter is None:
+        return _app_namespace
+    return f'{_app_namespace}{trailing_delimiter}'
+
+def split_app_namespace(full_name: str, delimiter: str) -> tuple[str, str]:
+    """Split the full name into the application namespace and the rest."""
+    parts = full_name.split(delimiter, 1)
+    if len(parts) == 1:
+        return '', parts[0]
+    return (parts[0], parts[1])
+
+def set_app_namespace(app_namespace: str):
+    """Set the application namespace."""
+    global _app_namespace  # pylint: disable=global-statement
+    _app_namespace = app_namespace
 
 @dataclass
 class DatabaseConnectionSpec:
@@ -30,6 +49,7 @@ def _load_field(target: dict[str, Any], name: str, env_name: str, required: bool
 class Settings:
     """Settings for the cocoindex library."""
     database: DatabaseConnectionSpec
+    app_namespace: str
 
     @classmethod
     def from_env(cls) -> Self:
@@ -40,7 +60,10 @@ class Settings:
         _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
         _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
         database = DatabaseConnectionSpec(**db_kwargs)
-        return cls(database=database)
+
+        app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", '')
+
+        return cls(database=database, app_namespace=app_namespace)
 
 @dataclass
 class ServerSettings:

cocoindex/setup.py

@@ -1,4 +1,5 @@
 from . import flow
+from . import setting
 from . import _engine
 
 def sync_setup() -> _engine.SetupStatus:
@@ -7,10 +8,15 @@ def sync_setup() -> _engine.SetupStatus:
 
 def drop_setup(flow_names: list[str]) -> _engine.SetupStatus:
     flow.ensure_all_flows_built()
-    return _engine.drop_setup(flow_names)
+    return _engine.drop_setup([flow.get_full_flow_name(name) for name in flow_names])
 
 def flow_names_with_setup() -> list[str]:
-    return _engine.flow_names_with_setup()
+    result = []
+    for name in _engine.flow_names_with_setup():
+        app_namespace, name = setting.split_app_namespace(name, '.')
+        if app_namespace == setting.get_app_namespace():
+            result.append(name)
+    return result
 
 def apply_setup_changes(setup_status: _engine.SetupStatus):
     _engine.apply_setup_changes(setup_status)

{cocoindex-0.1.37.dist-info → cocoindex-0.1.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.37
+Version: 0.1.38
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
@@ -148,6 +148,7 @@ It defines an index flow like this:
 | [Code Embedding](examples/code_embedding) | Index code embeddings for semantic search |
 | [PDF Embedding](examples/pdf_embedding) | Parse PDF and index text embeddings for semantic search |
 | [Manuals LLM Extraction](examples/manuals_llm_extraction) | Extract structured information from a manual using LLM |
+| [Amazon S3 Embedding](examples/amazon_s3_embedding) | Index text documents from Amazon S3 |
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |

{cocoindex-0.1.37.dist-info → cocoindex-0.1.38.dist-info}/RECORD

@@ -1,25 +1,25 @@
-cocoindex-0.1.37.dist-info/METADATA,sha256=q_5xT2W7gjD_IiCl_pBr__gMfn3ldcoFppGNTmROkMY,9699
-cocoindex-0.1.37.dist-info/WHEEL,sha256=ryp1uXCltaq3TPfjxhSWoam4EHMC40-Yabw-yvEW9gU,104
-cocoindex-0.1.37.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex-0.1.38.dist-info/METADATA,sha256=d1qjWo7V_MjeUiBe4zIOyqCyg8gM7-DaUDbKPSIf3_Q,9793
+cocoindex-0.1.38.dist-info/WHEEL,sha256=ryp1uXCltaq3TPfjxhSWoam4EHMC40-Yabw-yvEW9gU,104
+cocoindex-0.1.38.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 cocoindex/__init__.py,sha256=LpB0VjGvkD1beio8R9RCT6PI3eU0keV-3sBL45fHTQE,690
-cocoindex/_engine.cpython-312-darwin.so,sha256=ixvMKTQ1mhgoOpeiCWhmtTuxFhpjTKD0D1mrpP6FeoI,56739584
+cocoindex/_engine.cpython-312-darwin.so,sha256=yNba_1vfgJyswvHd7QM_s7XOfkYJBUP4o4h97J3WTEU,56739728
 cocoindex/auth_registry.py,sha256=NsALZ3SKsDG9cPdrlTlalIqUvgbgFOaFGAbWJNedtJE,692
-cocoindex/cli.py,sha256=1PJTdwX-X9r6Obw_W7VZDUgd5E7Xw6tLyvSDT8dFnY8,8924
+cocoindex/cli.py,sha256=Ac3ybnQW-HGVGJeUwIOHd1qhjs0KC5wCsemWuyouEfU,8999
 cocoindex/convert.py,sha256=tRY-QBeeFMFwCYiRk7a0_tuDqopw8iqBpg_Aswcq9JQ,6864
-cocoindex/flow.py,sha256=MZZ0Uf0ObAzR1yIjUecRgA-U0t__95eoLBK_DxwwLnk,23375
+cocoindex/flow.py,sha256=r1GnRIthmkniJEsAxGsYlDXbcf7ydMwZy1qJEKzwtqc,23814
 cocoindex/functions.py,sha256=F79dNmGE127LaU67kF5Oqtf_tIzebFQH7MkyceMX4-s,1830
 cocoindex/index.py,sha256=LssEOuZi6AqhwKtZM3QFeQpa9T-0ELi8G5DsrYKECvc,534
-cocoindex/lib.py,sha256=812GB8Z-2PyjG73Odvw5jtNBLnoeU9aOh9s2ZnETKa8,2329
+cocoindex/lib.py,sha256=OqTMuOHicdyX9PRA7fmTzznK8HZMrzxpUDbqxAEF--Q,2383
 cocoindex/llm.py,sha256=_3rtahuKcqcEHPkFSwhXOSrekZyGxVApPoYtlU_chcA,348
 cocoindex/op.py,sha256=OGYRYl7gPa7X7iSU30iTrCzvqRBu7jQqfvN4vjG__dA,10730
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/query.py,sha256=8_3Lb_EVjZtl2ZyJNZGX16LoKXEd-PL8OjY-zs9GQeA,3205
 cocoindex/runtime.py,sha256=jqRnWkkIlAhE04gi4y0Y5bzuq9FX4j0aVNU-nengLJk,980
-cocoindex/setting.py,sha256=pms1blwlXIOqZIpye-rfiwzqYUCAC8oEL7mQM5A160g,2356
-cocoindex/setup.py,sha256=AQLbtBLuJX066IANS7BGp20246mAGQ_4Z0W6MVJcQzY,481
+cocoindex/setting.py,sha256=AaIMclEktbBgK7Cks2D8LfS1cskf8UUcbSb6UBLdoSs,3260
+cocoindex/setup.py,sha256=ErNtX08NfFOFKehp5qGUvCx8Wiz9f3gmzvfBhAqrQyI,745
 cocoindex/sources.py,sha256=7lpwYLsFCRfbURKf79Vu0JZZoXjAYY0DxNHzUb-VHBY,1327
 cocoindex/storages.py,sha256=MFMsfyOCYMggTWeWrOi82miqOXQmiUuqq828x5htBr0,2207
 cocoindex/tests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 cocoindex/tests/test_convert.py,sha256=7jc--I3frrg7DB5MPr4JFzE7DSCznJuWyHdlDLQJ_fM,15516
 cocoindex/typing.py,sha256=369ABRtnpbaVSQVIBc2ZDutXW8jUmncvNJd9CHEWT3Q,8962
-cocoindex-0.1.37.dist-info/RECORD,,
+cocoindex-0.1.38.dist-info/RECORD,,