cocoindex 0.1.65__pp311-pypy311_pp73-manylinux_2_28_aarch64.whl → 0.1.66__pp311-pypy311_pp73-manylinux_2_28_aarch64.whl

cocoindex/__init__.py

@@ -33,6 +33,7 @@ __all__ = [
     # Submodules
     "_engine",
     "functions",
+    "llm",
     "sources",
     "targets",
     "storages",

cocoindex/convert.py

@@ -89,13 +89,26 @@ def make_engine_value_decoder(
     if dst_is_any:
         if src_type_kind == "Union":
             return lambda value: value[1]
-        if src_type_kind == "Struct" or src_type_kind in TABLE_TYPES:
+        if src_type_kind == "Struct":
+            return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
+        if src_type_kind in TABLE_TYPES:
             raise ValueError(
                 f"Missing type annotation for `{''.join(field_path)}`."
                 f"It's required for {src_type_kind} type."
             )
         return lambda value: value
 
+    # Handle struct -> dict binding for explicit dict annotations
+    is_dict_annotation = False
+    if dst_annotation is dict:
+        is_dict_annotation = True
+    elif getattr(dst_annotation, "__origin__", None) is dict:
+        args = getattr(dst_annotation, "__args__", ())
+        if args == (str, Any):
+            is_dict_annotation = True
+    if is_dict_annotation and src_type_kind == "Struct":
+        return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"])
+
     dst_type_info = analyze_type_info(dst_annotation)
 
     if src_type_kind == "Union":
@@ -294,6 +307,39 @@ def _make_engine_struct_value_decoder(
     )
 
 
+def _make_engine_struct_to_dict_decoder(
+    field_path: list[str],
+    src_fields: list[dict[str, Any]],
+) -> Callable[[list[Any] | None], dict[str, Any] | None]:
+    """Make a decoder from engine field values to a Python dict."""
+
+    field_decoders = []
+    for i, field_schema in enumerate(src_fields):
+        field_name = field_schema["name"]
+        field_path.append(f".{field_name}")
+        field_decoder = make_engine_value_decoder(
+            field_path,
+            field_schema["type"],
+            Any,  # Use Any for recursive decoding
+        )
+        field_path.pop()
+        field_decoders.append((field_name, field_decoder))
+
+    def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None:
+        if values is None:
+            return None
+        if len(field_decoders) != len(values):
+            raise ValueError(
+                f"Field count mismatch: expected {len(field_decoders)}, got {len(values)}"
+            )
+        return {
+            field_name: field_decoder(value)
+            for value, (field_name, field_decoder) in zip(values, field_decoders)
+        }
+
+    return decode_to_dict
+
+
 def dump_engine_object(v: Any) -> Any:
     """Recursively dump an object for engine. Engine side uses `Pythonized` to catch."""
     if v is None:

cocoindex/functions.py

@@ -45,6 +45,7 @@ class EmbedText(op.FunctionSpec):
     address: str | None = None
     output_dimension: int | None = None
     task_type: str | None = None
+    api_config: llm.VertexAiConfig | None = None
 
 
 class ExtractByLlm(op.FunctionSpec):

cocoindex/llm.py

@@ -8,6 +8,7 @@ class LlmApiType(Enum):
     OPENAI = "OpenAi"
     OLLAMA = "Ollama"
     GEMINI = "Gemini"
+    VERTEX_AI = "VertexAi"
     ANTHROPIC = "Anthropic"
     LITE_LLM = "LiteLlm"
     OPEN_ROUTER = "OpenRouter"
@@ -15,6 +16,16 @@ class LlmApiType(Enum):
     VLLM = "Vllm"
 
 
+@dataclass
+class VertexAiConfig:
+    """A specification for a Vertex AI LLM."""
+
+    kind = "VertexAi"
+
+    project: str
+    region: str | None = None
+
+
 @dataclass
 class LlmSpec:
     """A specification for a LLM."""
@@ -22,3 +33,4 @@ class LlmSpec:
     api_type: LlmApiType
     model: str
     address: str | None = None
+    api_config: VertexAiConfig | None = None

cocoindex/sources.py

@@ -43,3 +43,18 @@ class AmazonS3(op.SourceSpec):
     included_patterns: list[str] | None = None
     excluded_patterns: list[str] | None = None
     sqs_queue_url: str | None = None
+
+
+class AzureBlob(op.SourceSpec):
+    """
+    Import data from an Azure Blob Storage container. Supports optional prefix and file filtering by glob patterns.
+    """
+
+    _op_category = op.OpCategory.SOURCE
+
+    account_name: str
+    container_name: str
+    prefix: str | None = None
+    binary: bool = False
+    included_patterns: list[str] | None = None
+    excluded_patterns: list[str] | None = None

cocoindex/tests/test_convert.py

@@ -1229,3 +1229,115 @@ def test_full_roundtrip_scalar_with_python_types() -> None:
         annotated_float=2.0,
     )
     validate_full_roundtrip(instance, MixedStruct)
+
+
+def test_roundtrip_struct_to_dict_binding() -> None:
+    """Test struct -> dict binding with Any annotation."""
+
+    @dataclass
+    class SimpleStruct:
+        name: str
+        value: int
+        price: float
+
+    instance = SimpleStruct("test", 42, 3.14)
+    expected_dict = {"name": "test", "value": 42, "price": 3.14}
+
+    # Test Any annotation
+    validate_full_roundtrip(instance, SimpleStruct, (expected_dict, Any))
+
+
+def test_roundtrip_struct_to_dict_explicit() -> None:
+    """Test struct -> dict binding with explicit dict annotations."""
+
+    @dataclass
+    class Product:
+        id: str
+        name: str
+        price: float
+        active: bool
+
+    instance = Product("P1", "Widget", 29.99, True)
+    expected_dict = {"id": "P1", "name": "Widget", "price": 29.99, "active": True}
+
+    # Test explicit dict annotations
+    validate_full_roundtrip(
+        instance, Product, (expected_dict, dict), (expected_dict, dict[str, Any])
+    )
+
+
+def test_roundtrip_struct_to_dict_with_none_annotation() -> None:
+    """Test struct -> dict binding with None annotation."""
+
+    @dataclass
+    class Config:
+        host: str
+        port: int
+        debug: bool
+
+    instance = Config("localhost", 8080, True)
+    expected_dict = {"host": "localhost", "port": 8080, "debug": True}
+
+    # Test None annotation (should be treated as Any)
+    validate_full_roundtrip(instance, Config, (expected_dict, None))
+
+
+def test_roundtrip_struct_to_dict_nested() -> None:
+    """Test struct -> dict binding with nested structs."""
+
+    @dataclass
+    class Address:
+        street: str
+        city: str
+
+    @dataclass
+    class Person:
+        name: str
+        age: int
+        address: Address
+
+    address = Address("123 Main St", "Anytown")
+    person = Person("John", 30, address)
+    expected_dict = {
+        "name": "John",
+        "age": 30,
+        "address": {"street": "123 Main St", "city": "Anytown"},
+    }
+
+    # Test nested struct conversion
+    validate_full_roundtrip(person, Person, (expected_dict, dict[str, Any]))
+
+
+def test_roundtrip_struct_to_dict_with_list() -> None:
+    """Test struct -> dict binding with list fields."""
+
+    @dataclass
+    class Team:
+        name: str
+        members: list[str]
+        active: bool
+
+    instance = Team("Dev Team", ["Alice", "Bob", "Charlie"], True)
+    expected_dict = {
+        "name": "Dev Team",
+        "members": ["Alice", "Bob", "Charlie"],
+        "active": True,
+    }
+
+    validate_full_roundtrip(instance, Team, (expected_dict, dict))
+
+
+def test_roundtrip_namedtuple_to_dict_binding() -> None:
+    """Test NamedTuple -> dict binding."""
+
+    class Point(NamedTuple):
+        x: float
+        y: float
+        z: float
+
+    instance = Point(1.0, 2.0, 3.0)
+    expected_dict = {"x": 1.0, "y": 2.0, "z": 3.0}
+
+    validate_full_roundtrip(
+        instance, Point, (expected_dict, dict), (expected_dict, Any)
+    )

cocoindex/tests/test_typing.py

@@ -539,5 +539,8 @@ def test_invalid_list_kind() -> None:
 
 def test_unsupported_type() -> None:
     typ = set
-    with pytest.raises(ValueError, match="type unsupported yet: <class 'set'>"):
+    with pytest.raises(
+        ValueError,
+        match="Unsupported as a specific type annotation for CocoIndex data type.*: <class 'set'>",
+    ):
         analyze_type_info(typ)

cocoindex/typing.py

@@ -168,7 +168,8 @@ class AnalyzedTypeInfo:
 
 def analyze_type_info(t: Any) -> AnalyzedTypeInfo:
     """
-    Analyze a Python type and return the analyzed info.
+    Analyze a Python type annotation and extract CocoIndex-specific type information.
+    Type annotations for specific CocoIndex types are expected. Raises ValueError for Any, empty, or untyped dict types.
     """
     if isinstance(t, tuple) and len(t) == 2:
         kt, vt = t
@@ -239,9 +240,15 @@ def analyze_type_info(t: Any) -> AnalyzedTypeInfo:
         _ = DtypeRegistry.validate_dtype_and_get_kind(elem_type)
         vector_info = VectorInfo(dim=None) if vector_info is None else vector_info
 
-    elif base_type is collections.abc.Mapping or base_type is dict:
+    elif base_type is collections.abc.Mapping or base_type is dict or t is dict:
         args = typing.get_args(t)
-        elem_type = (args[0], args[1])
+        if len(args) == 0:  # Handle untyped dict
+            raise ValueError(
+                "Untyped dict is not accepted as a specific type annotation; please provide a concrete type, "
+                "e.g. a dataclass or namedtuple for Struct types, a dict[str, T] for KTable types."
+            )
+        else:
+            elem_type = (args[0], args[1])
         kind = "KTable"
     elif base_type in (types.UnionType, typing.Union):
         possible_types = typing.get_args(t)
@@ -283,7 +290,9 @@ def analyze_type_info(t: Any) -> AnalyzedTypeInfo:
         elif t is datetime.timedelta:
             kind = "TimeDelta"
         else:
-            raise ValueError(f"type unsupported yet: {t}")
+            raise ValueError(
+                f"Unsupported as a specific type annotation for CocoIndex data type (https://cocoindex.io/docs/core/data_types): {t}"
+            )
 
     return AnalyzedTypeInfo(
         kind=kind,

{cocoindex-0.1.65.dist-info → cocoindex-0.1.66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.65
+Version: 0.1.66
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
 Requires-Dist: python-dotenv>=1.1.0
@@ -25,7 +25,7 @@ Project-URL: Homepage, https://cocoindex.io/
     <img src="https://cocoindex.io/images/github.svg" alt="CocoIndex">
 </p>
 
-<h2 align="center">Extract, Transform, Index Data. Easy and Fresh. 🌴</h2>
+<h1 align="center">Data transformation for AI</h1>
 
 <div align="center">
 
@@ -40,18 +40,32 @@ Project-URL: Homepage, https://cocoindex.io/
 [![Discord](https://img.shields.io/discord/1314801574169673738?logo=discord&color=5B5BD6&logoColor=white)](https://discord.com/invite/zpA9S2DR7s)
 </div>
 
-**CocoIndex** is an ultra performant data transformation framework, with its core engine written in Rust. The problem it tries to solve is to make it easy to prepare fresh data for AI - either creating embedding, building knowledge graphs, or performing other data transformations - and take real-time data pipelines beyond traditional SQL.
+Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box.  Exceptional developer velocity. Production-ready at day 0.
+
+⭐ Drop a star to help us grow!
+
+</br>
+
+<p align="center">
+    <img src="https://cocoindex.io/images/transformation.svg" alt="CocoIndex Transformation">
+</p>
+
+</br>
+
+CocoIndex makes it super easy to transform data with AI workloads, and keep source data and target in sync effortlessly.
+
+</br>
 
 <p align="center">
-    <img src="https://cocoindex.io/images/cocoindex-features.png" alt="CocoIndex Features" width="500">
+    <img src="https://cocoindex.io/images/venn-features.png" alt="CocoIndex Features" width='480'>
 </p>
 
-The philosophy is to have the framework handle the source updates, and having developers only worry about defining a series of data transformation, inspired by spreadsheet.
+</br>
 
-## Dataflow programming
-Unlike a workflow orchestration framework where data is usually opaque, in CocoIndex, data and data operations are first class citizens. CocoIndex follows the idea of [Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) programming model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
+Either creating embedding, building knowledge graphs, or any data transformations - beyond traditional SQL.
 
-**Particularly**, users don't explicitly mutate data by creating, updating and deleting. Rather, they define something like - for a set of source data, this is the transformation or formula. The framework takes care of the data operations such as when to create, update, or delete.
+## Exceptional velocity
+Just declare transformation in dataflow with ~100 lines of python
 
 ```python
 # import
@@ -69,19 +83,27 @@ collector.collect(...)
 collector.export(...)
 ```
 
-## Data Freshness
-As a data framework, CocoIndex takes it to the next level on data freshness. **Incremental processing** is one of the core values provided by CocoIndex.
+CocoIndex follows the idea of [Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) programming model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
+
+**Particularly**, developers don't explicitly mutate data by creating, updating and deleting. They just need to define transformation/formula for a set of source data.
+
+## Build like LEGO
+Native builtins for different source, targets and transformations. Standardize interface, make it 1-line code switch between different components.
 
 <p align="center">
-    <img src="https://github.com/user-attachments/assets/f4eb29b3-84ee-4fa0-a1e2-80eedeeabde6" alt="Incremental Processing" width="700">
+    <img src="https://cocoindex.io/images/components.svg" alt="CocoIndex Features">
 </p>
 
-The frameworks takes care of
-- Change data capture.
-- Figure out what exactly needs to be updated, and only updating that without having to recompute everything.
+## Data Freshness
+CocoIndex keep source data and target in sync effortlessly.
 
-This makes it fast to reflect any source updates to the target store. If you have concerns with surfacing stale data to AI agents and are spending lots of efforts working on infra piece to optimize the latency, the framework actually handles it for you.
+<p align="center">
+    <img src="https://github.com/user-attachments/assets/f4eb29b3-84ee-4fa0-a1e2-80eedeeabde6" alt="Incremental Processing" width="700">
+</p>
 
+It has out-of-box support for incremental indexing:
+- minimal recomputation on source or logic change.
+- (re-)processing necessary portions; reuse cache when possible
 
 ## Quick Start:
 If you're new to CocoIndex, we recommend checking out
@@ -100,7 +122,7 @@ pip install -U cocoindex
 2. [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one. CocoIndex uses it for incremental processing.
 
 
-### Define data flow
+## Define data flow
 
 Follow [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart) to define your first indexing flow. An example flow looks like:
 
@@ -144,8 +166,9 @@ def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoind
 
 It defines an index flow like this:
 
-<img width="363" alt="Data Flow" src="https://github.com/user-attachments/assets/2ea7be6d-3d94-42b1-b2bd-22515577e463" />
-
+<p align="center">
+    <img width="400" alt="Data Flow" src="https://github.com/user-attachments/assets/2ea7be6d-3d94-42b1-b2bd-22515577e463" />
+</p>
 
 ## 🚀 Examples and demo
 
@@ -156,6 +179,7 @@ It defines an index flow like this:
 | [PDF Embedding](examples/pdf_embedding) | Parse PDF and index text embeddings for semantic search |
 | [Manuals LLM Extraction](examples/manuals_llm_extraction) | Extract structured information from a manual using LLM |
 | [Amazon S3 Embedding](examples/amazon_s3_embedding) | Index text documents from Amazon S3 |
+| [Azure Blob Storage Embedding](examples/azure_blob_embedding) | Index text documents from Azure Blob Storage |
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |

{cocoindex-0.1.65.dist-info → cocoindex-0.1.66.dist-info}/RECORD

@@ -1,28 +1,28 @@
-cocoindex-0.1.65.dist-info/METADATA,sha256=q_r8liwwqP1fcDFl0sNkTjty5ZZt6xbGEUFcpOOOR04,10136
-cocoindex-0.1.65.dist-info/WHEEL,sha256=7zxhuuBJUuil6R_MfVJvTZXJmNSPMntESIcLB40BBxA,116
-cocoindex-0.1.65.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
-cocoindex-0.1.65.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-cocoindex/__init__.py,sha256=TXK1FRqfm9DeDD7MycgDEfNutu8utM-dARQSurD98vc,1993
-cocoindex/_engine.pypy311-pp73-aarch64-linux-gnu.so,sha256=zyMcIKPC6h3SZcET-K5b2uMr8UOs7grRdOzh7uVFlBI,61389384
+cocoindex-0.1.66.dist-info/METADATA,sha256=ZwdUYvaVBt5qmZ7pQqJOB6FoszGiwbEerJyeI52aSBE,10098
+cocoindex-0.1.66.dist-info/WHEEL,sha256=7zxhuuBJUuil6R_MfVJvTZXJmNSPMntESIcLB40BBxA,116
+cocoindex-0.1.66.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
+cocoindex-0.1.66.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cocoindex/__init__.py,sha256=nr-W-LX0e9EX6ZVMbf0MRF0EhtntqADOdwc9F2nwfWw,2004
+cocoindex/_engine.pypy311-pp73-aarch64-linux-gnu.so,sha256=B6HiMN0h56MtAcVy_f6s3KOXjvVf1jURJcwSoEMxL6o,64916680
 cocoindex/auth_registry.py,sha256=1XqO7ibjmBBd8i11XSJTvTgdz8p1ptW-ZpuSgo_5zzk,716
 cocoindex/cli.py,sha256=-gp639JSyQN6YjnhGqCakIzYoSSqXxQMbxbkcYGP0QY,22359
-cocoindex/convert.py,sha256=qE1Ut_tAwX4wA4WqaWxpyj80-1t6WZ8Oi5_L9Mw5g4k,11393
+cocoindex/convert.py,sha256=RYfRUungabr-dHakG4k2kDvYambxHFljAmTuPQeQths,13117
 cocoindex/flow.py,sha256=MFPtfJBVTjQ56d7vUn2LvtY30Vg4q2rY6nqvjjJL1kQ,35085
-cocoindex/functions.py,sha256=IBwvdPpGR-S5mk53HvHpT2GVs15MI9wQznxgOdxA0ac,3202
+cocoindex/functions.py,sha256=EvRnd2Y3TIeEueYjOmproBG5RinlbqK6ym1Hnrmnm-0,3251
 cocoindex/index.py,sha256=j93B9jEvvLXHtpzKWL88SY6wCGEoPgpsQhEGHlyYGFg,540
 cocoindex/lib.py,sha256=f--9dAYd84CZosbDZqNW0oGbBLsY3dXiUTR1VrfQ_QY,817
-cocoindex/llm.py,sha256=0ri8ZRg9_Zf2gyC5xuQ1Kq6kdZUO8r-A5WLnxit5S_4,448
+cocoindex/llm.py,sha256=WxmWUbNcf9HOCM5xkbDeFs9lF67M3mr810B7deDDc-8,673
 cocoindex/op.py,sha256=r_Usx7Jqh49Cck3tsYLx2vLRNUZArkQP_g7bIID6LPU,11809
 cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cocoindex/runtime.py,sha256=bAdHYaXFWiiUWyAgzmKTeaAaRR0D_AmaqVCIdPO-v00,1056
 cocoindex/setting.py,sha256=ADuv7RaWd9k-m3V0Cfy2jmaCt6DupJCviWdOm0CTiVw,4734
 cocoindex/setup.py,sha256=7uIHKN4FOCuoidPXcKyGTrkqpkl9luL49-6UcnMxYzw,3068
-cocoindex/sources.py,sha256=JCnOhv1w4o28e03i7yvo4ESicWYAhckkBg5bQlxNH4U,1330
+cocoindex/sources.py,sha256=8MR_oyr7t0m-gUFq7FO6HHM-tDLmQSBAjheFXJzRd8g,1733
 cocoindex/targets.py,sha256=Nfh_tpFd1goTnS_cxBjIs4j9zl3Z4Z1JomAQ1dl3Sic,2796
 cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cocoindex/tests/test_convert.py,sha256=efwF-43SFJiu85sQ78Z9k9OaJphTz_es_1cm5BoPO2Y,42565
+cocoindex/tests/test_convert.py,sha256=48-fnWKv02gmFETV3b-8IC89SKMUZfJLEH-ucRtXGuI,45450
 cocoindex/tests/test_optional_database.py,sha256=snAmkNa6wtOSaxoZE1HgjvL5v_ylitt3Jt_9df4Cgdc,8506
-cocoindex/tests/test_typing.py,sha256=t6UCYShcfonTfjBlGRWPiFGMZ8DGFfABXo6idekPoJE,14757
-cocoindex/typing.py,sha256=qQ-nSdkHzu8pSxfuR5sGGfoE8nCKqCDb0D9jbmxVt4M,12635
+cocoindex/tests/test_typing.py,sha256=NB4nUzoumOF_wGFa4D2Xf6d0bUVtOiSXyb78M1pYSG4,14827
+cocoindex/typing.py,sha256=MO9HkrNpargvMPvpkd7jgSu2R-21KE_NaB9-WI4YOZA,13241
 cocoindex/utils.py,sha256=hUhX-XV6XGCtJSEIpBOuDv6VvqImwPlgBxztBTw7u0U,598
-cocoindex-0.1.65.dist-info/RECORD,,
+cocoindex-0.1.66.dist-info/RECORD,,