cocoindex 0.1.33__cp313-cp313-macosx_11_0_arm64.whl → 0.1.34__cp313-cp313-macosx_11_0_arm64.whl

cocoindex/cli.py

@@ -55,16 +55,17 @@ def ls(show_all: bool):
 
 @cli.command()
 @click.argument("flow_name", type=str, required=False)
-@click.option("--color/--no-color", default=True)
-def show(flow_name: str | None, color: bool):
+@click.option("--color/--no-color", default=True, help="Enable or disable colored output.")
+@click.option("--verbose", is_flag=True, help="Show verbose output with full details.")
+def show(flow_name: str | None, color: bool, verbose: bool):
     """
-    Show the flow spec in a readable format with colored output,
-    including the schema.
+    Show the flow spec and schema in a readable format with colored output.
     """
     flow = _flow_by_name(flow_name)
     console = Console(no_color=not color)
-    console.print(flow._render_text())
+    console.print(flow._render_spec(verbose=verbose))
 
+    console.print()
     table = Table(
         title=f"Schema for Flow: {flow.name}",
         show_header=True,
@@ -74,7 +75,7 @@ def show(flow_name: str | None, color: bool):
     table.add_column("Type", style="green")
     table.add_column("Attributes", style="yellow")
 
-    for field_name, field_type, attr_str in flow._render_schema():
+    for field_name, field_type, attr_str in flow._get_schema():
         table.add_row(field_name, field_type, attr_str)
 
     console.print(table)
@@ -85,15 +86,15 @@ def setup():
     Check and apply backend setup changes for flows, including the internal and target storage
     (to export).
     """
-    status_check = sync_setup()
-    click.echo(status_check)
-    if status_check.is_up_to_date():
+    setup_status = sync_setup()
+    click.echo(setup_status)
+    if setup_status.is_up_to_date():
         click.echo("No changes need to be pushed.")
         return
     if not click.confirm(
         "Changes need to be pushed. Continue? [yes/N]", default=False, show_default=False):
         return
-    apply_setup_changes(status_check)
+    apply_setup_changes(setup_status)
 
 @cli.command()
 @click.argument("flow_name", type=str, nargs=-1)
@@ -112,15 +113,15 @@ def drop(flow_name: tuple[str, ...], drop_all: bool):
         flow_names = [fl.name for fl in flow.flows()]
     else:
         flow_names = list(flow_name)
-    status_check = drop_setup(flow_names)
-    click.echo(status_check)
-    if status_check.is_up_to_date():
+    setup_status = drop_setup(flow_names)
+    click.echo(setup_status)
+    if setup_status.is_up_to_date():
         click.echo("No flows need to be dropped.")
         return
     if not click.confirm(
         "Changes need to be pushed. Continue? [yes/N]", default=False, show_default=False):
         return
-    apply_setup_changes(status_check)
+    apply_setup_changes(setup_status)
 
 @cli.command()
 @click.argument("flow_name", type=str, required=False)

cocoindex/flow.py

@@ -15,7 +15,7 @@ from threading import Lock
 from enum import Enum
 from dataclasses import dataclass
 from rich.text import Text
-from rich.console import Console
+from rich.tree import Tree
 
 from . import _engine
 from . import index
@@ -161,6 +161,9 @@ class DataSlice:
         """
         Apply a function to the data slice.
         """
+        if not isinstance(fn_spec, op.FunctionSpec):
+            raise ValueError("transform() can only be called on a CocoIndex function")
+
         transform_args: list[tuple[Any, str | None]]
         transform_args = [(self._state.engine_data_slice, None)]
         transform_args += [(self._state.flow_builder_state.get_data_slice(v), None) for v in args]
@@ -280,6 +283,9 @@ class DataCollector:
 
         `vector_index` is for backward compatibility only. Please use `vector_indexes` instead.
         """
+        if not isinstance(target_spec, op.StorageSpec):
+            raise ValueError("export() can only be called on a CocoIndex target storage")
+
         # For backward compatibility only.
         if len(vector_indexes) == 0 and len(vector_index) > 0:
             vector_indexes = [index.VectorIndexDef(field_name=field_name, metric=metric)
@@ -343,8 +349,10 @@ class FlowBuilder:
             refresh_interval: datetime.timedelta | None = None,
         ) -> DataSlice:
         """
-        Add a source to the flow.
+        Import a source to the flow.
         """
+        if not isinstance(spec, op.SourceSpec):
+            raise ValueError("add_source() can only be called on a CocoIndex source")
         return _create_data_slice(
             self._state,
             lambda target_scope, name: self._state.engine_flow_builder.add_source(
@@ -454,61 +462,33 @@ class Flow:
             return engine_flow
         self._lazy_engine_flow = _lazy_engine_flow
 
-    def _format_flow(self, flow_dict: dict) -> Text:
-        output = Text()
+    def _render_spec(self, verbose: bool = False) -> Tree:
+        """
+        Render the flow spec as a styled rich Tree with hierarchical structure.
+        """
+        spec = self._get_spec(verbose=verbose)
+        tree = Tree(f"Flow: {self.name}", style="cyan")
 
-        def add_line(content, indent=0, style=None, end="\n"):
-            output.append(" " * indent)
-            output.append(content, style=style)
-            output.append(end)
+        def build_tree(label: str, lines: list):
+            node = Tree(label, style="bold magenta" if lines else "cyan")
+            for line in lines:
+                child_node = node.add(Text(line.content, style="yellow"))
+                child_node.children = build_tree("", line.children).children
+            return node
 
-        def format_key_value(key, value, indent):
-            if isinstance(value, (dict, list)):
-                add_line(f"- {key}:", indent, style="green")
-                format_data(value, indent + 2)
-            else:
-                add_line(f"- {key}:", indent, style="green", end="")
-                add_line(f" {value}", style="yellow")
-
-        def format_data(data, indent=0):
-            if isinstance(data, dict):
-                for key, value in data.items():
-                    format_key_value(key, value, indent)
-            elif isinstance(data, list):
-                for i, item in enumerate(data):
-                    format_key_value(f"[{i}]", item, indent)
-            else:
-                add_line(str(data), indent, style="yellow")
-
-        # Header
-        flow_name = flow_dict.get("name", "Unnamed")
-        add_line(f"Flow: {flow_name}", style="bold cyan")
-
-        # Section
-        for section_title, section_key in [
-            ("Sources:", "import_ops"),
-            ("Processing:", "reactive_ops"),
-            ("Targets:", "export_ops"),
-        ]:
-            add_line("")
-            add_line(section_title, style="bold cyan")
-            format_data(flow_dict.get(section_key, []), indent=0)
-
-        return output
-
-    def _render_text(self) -> Text:
-        flow_spec_str = str(self._lazy_engine_flow())
-        try:
-            flow_dict = json.loads(flow_spec_str)
-            return self._format_flow(flow_dict)
-        except json.JSONDecodeError:
-            return Text(flow_spec_str)
+        for section, lines in spec.sections:
+            section_node = build_tree(f"{section}:", lines)
+            tree.children.append(section_node)
+        return tree
+
+    def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]:
+        return self._lazy_engine_flow().get_spec(output_mode="verbose" if verbose else "concise")
     
-    def _render_schema(self) -> list[tuple[str, str, str]]:
+    def _get_schema(self) -> list[tuple[str, str, str]]:
         return self._lazy_engine_flow().get_schema()
 
     def __str__(self):
-        return str(self._render_text())
+        return str(self._get_spec())
 
     def __repr__(self):
         return repr(self._lazy_engine_flow())

cocoindex/setup.py

@@ -1,16 +1,16 @@
 from . import flow
 from . import _engine
 
-def sync_setup() -> _engine.SetupStatusCheck:
+def sync_setup() -> _engine.SetupStatus:
     flow.ensure_all_flows_built()
     return _engine.sync_setup()
 
-def drop_setup(flow_names: list[str]) -> _engine.SetupStatusCheck:
+def drop_setup(flow_names: list[str]) -> _engine.SetupStatus:
     flow.ensure_all_flows_built()
     return _engine.drop_setup(flow_names)
 
 def flow_names_with_setup() -> list[str]:
     return _engine.flow_names_with_setup()
 
-def apply_setup_changes(status_check: _engine.SetupStatusCheck):
-    _engine.apply_setup_changes(status_check)
+def apply_setup_changes(setup_status: _engine.SetupStatus):
+    _engine.apply_setup_changes(setup_status)

{cocoindex-0.1.33.dist-info → cocoindex-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.33
+Version: 0.1.34
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
@@ -28,44 +28,74 @@ Project-URL: Homepage, https://cocoindex.io/
 [![PyPI version](https://img.shields.io/pypi/v/cocoindex?color=5B5BD6)](https://pypi.org/project/cocoindex/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cocoindex)](https://pypistats.org/packages/cocoindex)
 
-<!-- [![Python](https://img.shields.io/badge/python-3.11%20to%203.13-5B5BD6?logo=python&logoColor=white)](https://www.python.org/) -->
 [![CI](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
 [![release](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml/badge.svg?event=push&color=5B5BD6)](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
-<!--[![LinkedIn](https://img.shields.io/badge/LinkedIn-CocoIndex-5B5BD6?logo=linkedin&logoColor=white)](https://www.linkedin.com/company/cocoindex) -->
-<!--[![X (Twitter)](https://img.shields.io/twitter/follow/cocoindex_io)](https://twitter.com/intent/follow?screen_name=cocoindex_io) -->
-
 </div>
 
-CocoIndex is the world's first open-source engine that supports both custom transformation logic and incremental updates specialized for data indexing.
+**CocoIndex** is an ultra performant data transformation framework, with its core engine written in Rust. The problem it tries to solve is to make it easy to prepare fresh data for AI - either creating embedding, building knowledge graphs, or performing other data transformations - and take real-time data pipelines beyond traditional SQL.
+
+<p align="center">
+    <img src="https://cocoindex.io/images/cocoindex-features.png" alt="CocoIndex Features" width="500">
+</p>
+
+The philosophy is to have the framework handle the source updates, and having developers only worry about defining a series of data transformation, inspired by spreadsheet.
+
+## Dataflow programming
+Unlike a workflow orchestration framework where data is usually opaque, in CocoIndex, data and data operations are first class citizens. CocoIndex follows the idea of [Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) programming model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
+
+**Particularly**, users don't explicitly mutate data by creating, updating and deleting. Rather, they define something like - for a set of source data, this is the transformation or formula. The framework takes care of the data operations such as when to create, update, or delete.
+
+```python
+# import
+data['content'] = flow_builder.add_source(...) 
+
+# transform
+data['out'] = data['content'] 
+    .transform(...)
+    .transform(...)
+
+# collect data
+collector.collect(...)
+
+# export to db, vector db, graph db ...
+collector.export(...)
+```
+
+## Data Freshness
+As a data framework, CocoIndex takes it to the next level on data freshness. **Incremental processing** is one of the core values provided by CocoIndex.
+
 <p align="center">
-    <img src="https://cocoindex.io/images/venn.svg" alt="CocoIndex">
+    <img src="https://github.com/user-attachments/assets/f4eb29b3-84ee-4fa0-a1e2-80eedeeabde6" alt="Incremental Processing" width="700">
 </p>
-With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
+
+The frameworks takes care of
+- Change data capture.
+- Figure out what exactly needs to be updated, and only updating that without having to recompute everything.
+  
+This makes it fast to reflect any source updates to the target store. If you have concerns with surfacing stale data to AI agents and are spending lots of efforts working on infra piece to optimize the latency, the framework actually handles it for you.
 
 
 ## Quick Start:
-If you're new to CocoIndex 🤗, we recommend checking out the 📖 [Documentation](https://cocoindex.io/docs) and ⚡ [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart). We also have a ▶️ [quick start video tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT) for you to jump start.
+If you're new to CocoIndex, we recommend checking out 
+- 📖 [Documentation](https://cocoindex.io/docs)
+- ⚡  [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart)
+- 🎬 [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT) 
 
 ### Setup 
+
 1. Install CocoIndex Python library
 
 ```bash
 pip install -U cocoindex
 ```
 
-2. Setup Postgres with pgvector extension; or bring up a Postgres database using docker compose:
+2. [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one. CocoIndex uses it for incremental processing.
 
-    - Make sure Docker Compose is installed: [docs](https://docs.docker.com/compose/install/)
-    - Start a Postgres SQL database for cocoindex using our docker compose config:
 
-    ```bash
-    docker compose -f <(curl -L https://raw.githubusercontent.com/cocoindex-io/cocoindex/refs/heads/main/dev/postgres.yaml) up -d
-    ```
+### Define data flow
 
-### Start your first indexing flow!
-Follow [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart) to define your first indexing flow.
-A common indexing flow looks like:
+Follow [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart) to define your first indexing flow. An example flow looks like:
 
 ```python
 @cocoindex.flow_def(name="TextEmbedding")
@@ -106,10 +136,11 @@ def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
 ```
 
 It defines an index flow like this:
-![Flow diagram](docs/docs/core/flow_example.svg)
 
-### Play with existing example and demo
-Go to the [examples directory](examples) to try out with any of the examples, following instructions under specific example directory.
+<img width="363" alt="Data Flow" src="https://github.com/user-attachments/assets/2ea7be6d-3d94-42b1-b2bd-22515577e463" />
+
+
+## 🚀 Examples and demo
 
 | Example | Description |
 |---------|-------------|
@@ -121,8 +152,10 @@ Go to the [examples directory](examples) to try out with any of the examples, fo
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
 | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
+| [Product_Taxonomy_Knowledge_Graph](examples/product_taxonomy_knowledge_graph) | Build knowledge graph for product recommendations | 
+| [Image Search with Vision API](examples/image_search_example) | Generates detailed captions for images using a vision model, embeds them, enables semantic search via FastAPI and served on a React frontend.|
 
-More coming and stay tuned! If there's any specific examples you would like to see, please let us know in our [Discord community](https://discord.com/invite/zpA9S2DR7s) 🌱.
+More coming and stay tuned 👀!
 
 ## 📖 Documentation
 For detailed documentation, visit [CocoIndex Documentation](https://cocoindex.io/docs), including a [Quickstart guide](https://cocoindex.io/docs/getting_started/quickstart).
@@ -136,13 +169,13 @@ Welcome with a huge coconut hug 🥥⋆｡˚🤗. We are super excited for commu
 Join our community here:
 
 - 🌟 [Star us on GitHub](https://github.com/cocoindex-io/cocoindex)
-- 💬 [Start a GitHub Discussion](https://github.com/cocoindex-io/cocoindex/discussions)
 - 👋 [Join our Discord community](https://discord.com/invite/zpA9S2DR7s)
-- 𝕏 [Follow us on X](https://x.com/cocoindex_io)
-- 🐚 [Follow us on LinkedIn](https://www.linkedin.com/company/cocoindex/about/)
 - ▶️ [Subscribe to our YouTube channel](https://www.youtube.com/@cocoindex-io)
 - 📜 [Read our blog posts](https://cocoindex.io/blogs/)
 
+## Support us:
+We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star ⭐ at GitHub repo [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow. 
+
 ## License
 CocoIndex is Apache 2.0 licensed.
 

{cocoindex-0.1.33.dist-info → cocoindex-0.1.34.dist-info}/RECORD

@@ -1,25 +1,25 @@
-cocoindex-0.1.33.dist-info/METADATA,sha256=SNak6BWyWii_I2_xCTt5KPvqobfaGtBJuywUfhBFMuo,8236
-cocoindex-0.1.33.dist-info/WHEEL,sha256=_czbP61TsBkf9T201RekHMHlqESnWn7yJwXBJC9P-w0,104
-cocoindex-0.1.33.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex-0.1.34.dist-info/METADATA,sha256=kSB_ocT5BakKnBmCeN1LBCTywDZ9w2d1gEfq-XZvH4c,9686
+cocoindex-0.1.34.dist-info/WHEEL,sha256=E1n5fBV87GjwiE9CayevgONfw5z1JsKZqIr2VxzumWg,104
+cocoindex-0.1.34.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex/__init__.py,sha256=CMfiZ-CROvrcE6jjkmzEZBk4HjuN6s6nfRXtSd0c_z8,684
+cocoindex/_engine.cpython-313-darwin.so,sha256=3X91YMXEpjyxY5C1JxGGo6_zzkCOynIyYplJ98td1P0,49713808
+cocoindex/auth_registry.py,sha256=NsALZ3SKsDG9cPdrlTlalIqUvgbgFOaFGAbWJNedtJE,692
+cocoindex/cli.py,sha256=nJoSmQ44DxF0rAccwjCtLwu0AwhhIA1MtsfWQ2AORz8,9047
+cocoindex/convert.py,sha256=mBUTa_Ag39_ut-yE_jc1wqS3zLjtOm6QKet-bqJ-RWc,5947
+cocoindex/flow.py,sha256=baVNPwvJ1aWZlAgJmv1qnERLaBpejuooIdHxDteVM1c,22599
 cocoindex/functions.py,sha256=F79dNmGE127LaU67kF5Oqtf_tIzebFQH7MkyceMX4-s,1830
-cocoindex/query.py,sha256=8_3Lb_EVjZtl2ZyJNZGX16LoKXEd-PL8OjY-zs9GQeA,3205
 cocoindex/index.py,sha256=LssEOuZi6AqhwKtZM3QFeQpa9T-0ELi8G5DsrYKECvc,534
 cocoindex/lib.py,sha256=812GB8Z-2PyjG73Odvw5jtNBLnoeU9aOh9s2ZnETKa8,2329
-cocoindex/auth_registry.py,sha256=NsALZ3SKsDG9cPdrlTlalIqUvgbgFOaFGAbWJNedtJE,692
-cocoindex/convert.py,sha256=mBUTa_Ag39_ut-yE_jc1wqS3zLjtOm6QKet-bqJ-RWc,5947
-cocoindex/tests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-cocoindex/tests/test_convert.py,sha256=WPRKp0jv_uSEM81RGWEAmsax-J-FtXt90mZ0yEnvGLs,11236
-cocoindex/__init__.py,sha256=CMfiZ-CROvrcE6jjkmzEZBk4HjuN6s6nfRXtSd0c_z8,684
-cocoindex/flow.py,sha256=KVbB_Ebm0IpJgZxV4BLg30fjIPmsGFhrtmQOBqCZaIk,23037
 cocoindex/llm.py,sha256=_3rtahuKcqcEHPkFSwhXOSrekZyGxVApPoYtlU_chcA,348
-cocoindex/setting.py,sha256=pms1blwlXIOqZIpye-rfiwzqYUCAC8oEL7mQM5A160g,2356
-cocoindex/runtime.py,sha256=jqRnWkkIlAhE04gi4y0Y5bzuq9FX4j0aVNU-nengLJk,980
 cocoindex/op.py,sha256=OGYRYl7gPa7X7iSU30iTrCzvqRBu7jQqfvN4vjG__dA,10730
-cocoindex/sources.py,sha256=wZFU8lwSXjyofJR-syySH9fTyPnBlAPJ6-1hQNX8fGA,936
-cocoindex/setup.py,sha256=W1HshwYk_K2aeLOVn_e62ZOXBO9yWsoUboRiH4SjF48,496
-cocoindex/cli.py,sha256=Vh8bNZ41yLr1l_jJR1Z_b7mY-dOvN-EbiCRxDvtIsRk,8885
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cocoindex/typing.py,sha256=BI2vPw4Iu4S3aznNJQrfM2LZU_weGYASTXF1W3ZWh_Y,8568
+cocoindex/query.py,sha256=8_3Lb_EVjZtl2ZyJNZGX16LoKXEd-PL8OjY-zs9GQeA,3205
+cocoindex/runtime.py,sha256=jqRnWkkIlAhE04gi4y0Y5bzuq9FX4j0aVNU-nengLJk,980
+cocoindex/setting.py,sha256=pms1blwlXIOqZIpye-rfiwzqYUCAC8oEL7mQM5A160g,2356
+cocoindex/setup.py,sha256=AQLbtBLuJX066IANS7BGp20246mAGQ_4Z0W6MVJcQzY,481
+cocoindex/sources.py,sha256=wZFU8lwSXjyofJR-syySH9fTyPnBlAPJ6-1hQNX8fGA,936
 cocoindex/storages.py,sha256=MFMsfyOCYMggTWeWrOi82miqOXQmiUuqq828x5htBr0,2207
-cocoindex/_engine.cpython-313-darwin.so,sha256=1Zi8FznUsI6jQSwOd4kpEl4FkER6T0Zw4ern2jGR4es,49631504
-cocoindex-0.1.33.dist-info/RECORD,,
+cocoindex/tests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+cocoindex/tests/test_convert.py,sha256=WPRKp0jv_uSEM81RGWEAmsax-J-FtXt90mZ0yEnvGLs,11236
+cocoindex/typing.py,sha256=BI2vPw4Iu4S3aznNJQrfM2LZU_weGYASTXF1W3ZWh_Y,8568
+cocoindex-0.1.34.dist-info/RECORD,,

{cocoindex-0.1.33.dist-info → cocoindex-0.1.34.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.8.3)
+Generator: maturin (1.8.4)
 Root-Is-Purelib: false
 Tag: cp313-cp313-macosx_11_0_arm64