nemo-retriever 2026.8.12.dev181__tar.gz → 2026.8.13.dev182__tar.gz

{nemo_retriever-2026.8.12.dev181/src/nemo_retriever.egg-info → nemo_retriever-2026.8.13.dev182}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-retriever
-Version: 2026.8.12.dev181
+Version: 2026.8.13.dev182
 Summary: A modern RAG ingestion pipeline from Nvidia
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License-Expression: Apache-2.0

nemo_retriever-2026.8.13.dev182/src/nemo_retriever/_build_info.py

@@ -0,0 +1,4 @@
+"""Build metadata written by CI before packaging."""
+
+BUILD_GIT_SHA = "1b941066053710739c6bcadf8cf96fd2da382634"
+BUILD_DATE = "2026-08-13T23:47:43Z"

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/common/modality/content_transforms.py

@@ -49,13 +49,37 @@ def _deep_copy_row(row_dict: Dict[str, Any]) -> Dict[str, Any]:
 
     out: Dict[str, Any] = {}
     for key, value in row_dict.items():
-        if isinstance(value, (dict, list)) or _is_content_collection(value):
+        if isinstance(value, np.ndarray) and _is_content_collection(value):
+            out[key] = copy.deepcopy(value.tolist())
+        elif isinstance(value, (dict, list)):
             out[key] = copy.deepcopy(value)
         else:
             out[key] = value
     return out
 
 
+def _normalize_bbox(value: Any) -> Any:
+    """Return bbox coordinates in a stable non-tensor container."""
+    if hasattr(value, "to_numpy") and callable(value.to_numpy):
+        value = value.to_numpy()
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if isinstance(value, tuple):
+        return list(value)
+    return value
+
+
+def _normalize_bbox_column(frame: pd.DataFrame) -> pd.DataFrame:
+    """Force every batch to expose bbox values as ordinary Python objects."""
+    values = frame.get("_bbox_xyxy_norm", pd.Series([None] * len(frame.index), index=frame.index))
+    frame["_bbox_xyxy_norm"] = pd.Series(
+        [_normalize_bbox(value) for value in values],
+        index=frame.index,
+        dtype=object,
+    )
+    return frame
+
+
 def explode_content_to_rows(
     batch_df: Any,
     *,
@@ -69,8 +93,10 @@ def explode_content_to_rows(
     text_mod = text_elements_modality or modality
     struct_mod = structured_elements_modality or modality
 
-    if not isinstance(batch_df, pd.DataFrame) or batch_df.empty:
+    if not isinstance(batch_df, pd.DataFrame):
         return batch_df
+    if batch_df.empty:
+        return _normalize_bbox_column(batch_df)
 
     any_images = text_mod in IMAGE_MODALITIES or struct_mod in IMAGE_MODALITIES
 
@@ -85,7 +111,7 @@ def explode_content_to_rows(
                 lambda page_image: page_image.get("stored_image_uri") if isinstance(page_image, dict) else None
             )
         batch_df["_embed_modality"] = text_mod
-        return batch_df
+        return _normalize_bbox_column(batch_df)
 
     new_rows: List[Dict[str, Any]] = []
     for _, row in batch_df.iterrows():
@@ -133,8 +159,8 @@ def explode_content_to_rows(
                         if item_b64:
                             content_row["_image_b64"] = item_b64
                         elif page_image_b64:
-                            bbox = item.get("bbox_xyxy_norm")
-                            if bbox and len(bbox) == 4:
+                            bbox = _normalize_bbox(item.get("bbox_xyxy_norm"))
+                            if bbox is not None and len(bbox) == 4:
                                 cropped_b64, _ = _crop_b64_image_by_norm_bbox(page_image_b64, bbox_xyxy_norm=bbox)
                                 content_row["_image_b64"] = cropped_b64
                             else:
@@ -142,7 +168,7 @@ def explode_content_to_rows(
                         else:
                             content_row["_image_b64"] = None
                     content_row["_stored_image_uri"] = item.get("stored_image_uri") or page_stored_uri
-                    content_row["_bbox_xyxy_norm"] = item.get("bbox_xyxy_norm")
+                    content_row["_bbox_xyxy_norm"] = _normalize_bbox(item.get("bbox_xyxy_norm"))
                     new_rows.append(content_row)
                     exploded_any = True
 
@@ -156,7 +182,7 @@ def explode_content_to_rows(
             preserved["_bbox_xyxy_norm"] = None
             new_rows.append(preserved)
 
-    return pd.DataFrame(new_rows).reset_index(drop=True)
+    return _normalize_bbox_column(pd.DataFrame(new_rows).reset_index(drop=True))
 
 
 def collapse_content_to_page_rows(
@@ -170,24 +196,17 @@ def collapse_content_to_page_rows(
     if not isinstance(batch_df, pd.DataFrame) or batch_df.empty:
         return batch_df
 
-    batch_df = batch_df.copy()
-    batch_df[text_column] = batch_df.apply(
-        lambda row: _combine_text_with_content(row, text_column, content_columns),
-        axis=1,
-    )
-
-    if modality in IMAGE_MODALITIES:
-        if "page_image" in batch_df.columns:
-            batch_df["_image_b64"] = batch_df["page_image"].apply(
-                lambda page_image: inline_image_b64(page_image) if isinstance(page_image, dict) else None
-            )
-        else:
-            batch_df["_image_b64"] = None
-
-    if "page_image" in batch_df.columns:
-        batch_df["_stored_image_uri"] = batch_df["page_image"].apply(
-            lambda page_image: page_image.get("stored_image_uri") if isinstance(page_image, dict) else None
-        )
-
-    batch_df["_embed_modality"] = modality
-    return batch_df
+    has_page_image = "page_image" in batch_df.columns
+    new_rows: List[Dict[str, Any]] = []
+    for _, row in batch_df.iterrows():
+        row_dict = _deep_copy_row(row.to_dict())
+        row_dict[text_column] = _combine_text_with_content(row_dict, text_column, content_columns)
+        page_image = row_dict.get("page_image") if has_page_image else None
+        if modality in IMAGE_MODALITIES:
+            row_dict["_image_b64"] = inline_image_b64(page_image) if isinstance(page_image, dict) else None
+        if has_page_image:
+            row_dict["_stored_image_uri"] = page_image.get("stored_image_uri") if isinstance(page_image, dict) else None
+        row_dict["_embed_modality"] = modality
+        new_rows.append(row_dict)
+
+    return _normalize_bbox_column(pd.DataFrame(new_rows).reset_index(drop=True))

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/common/params/models.py

@@ -516,7 +516,13 @@ class ExtractParams(_ParamsModel):
     extract_page_as_image: Optional[bool] = True
 
     # Extraction options
-    method: str = "pdfium"
+    method: Literal["pdfium", "pdfium_hybrid", "ocr", "nemotron_parse", "audio"] = Field(
+        default="pdfium",
+        description=(
+            "Extraction method. PDF extraction supports 'pdfium', 'pdfium_hybrid', 'ocr', and "
+            "'nemotron_parse'; 'audio' is retained for the legacy params-driven audio path."
+        ),
+    )
     # Run PageElementDetection (layout/yolox). Required by TableStructure and
     # OCR. Safe to disable for text-only ingests.
     use_page_elements: bool = True

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/common/vdb/records.py

@@ -132,9 +132,21 @@ class RetrievalHit(TypedDict, total=False):
 
 def _embedding_from_graph_row(row: dict[str, Any], metadata: dict[str, Any]) -> Any:
     if metadata.get("embedding") is not None:
-        return metadata["embedding"]
-    payload = row.get("text_embeddings_1b_v2")
-    return payload.get("embedding") if isinstance(payload, dict) else None
+        embedding = metadata["embedding"]
+    else:
+        payload = row.get("text_embeddings_1b_v2")
+        embedding = payload.get("embedding") if isinstance(payload, dict) else None
+
+    if not isinstance(embedding, list):
+        to_numpy = getattr(embedding, "to_numpy", None)
+        if callable(to_numpy):
+            embedding = to_numpy()
+        tolist = getattr(embedding, "tolist", None)
+        if callable(tolist):
+            embedding = tolist()
+        elif isinstance(embedding, tuple):
+            embedding = list(embedding)
+    return embedding
 
 
 def _first_str(*values: Any) -> str:
@@ -317,6 +329,13 @@ def _client_record_from_graph_row(row: dict[str, Any], *, require_embedding: boo
     return {"document_type": str(document_type), "metadata": record_metadata}
 
 
+def _row_has_uploadable_content_without_embedding(row: dict[str, Any]) -> bool:
+    metadata = _dict_or_empty(row.get("metadata"))
+    if _embedding_from_graph_row(row, metadata) is not None:
+        return False
+    return bool(_text_from_graph_row(row, metadata)) or _is_image_backed_row(row)
+
+
 def to_client_vdb_records(rows: Any) -> list[list[dict[str, Any]]]:
     """Convert graph-ingest rows into the nested record shape expected by client VDBs.
 
@@ -325,6 +344,8 @@ def to_client_vdb_records(rows: Any) -> list[list[dict[str, Any]]]:
     ``if not records`` skips :meth:`~nemo_retriever.vdb.adt_vdb.VDB.run`.
     When at least one row converts, returns ``[batch]`` with a single non-empty inner list
     (never ``[[]]``, which would be truthy and could trip backends on an empty insert).
+    Uploadable graph content without an embedding raises ``ValueError`` when no
+    row survives conversion. Genuinely empty input continues to return ``[]``.
     """
     if isinstance(rows, list) and all(isinstance(batch, list) for batch in rows):
         return rows
@@ -332,14 +353,16 @@ def to_client_vdb_records(rows: Any) -> list[list[dict[str, Any]]]:
         rows = rows.to_pandas()
     if hasattr(rows, "to_dict"):
         rows = rows.to_dict("records")
+    graph_rows = [row for row in rows or [] if isinstance(row, dict)]
     # Walrus: bind conversion once per row — a plain ``if f(row)`` + ``f(row)`` list comp
     # would call _client_record_from_graph_row twice per row on large datasets.
     # isinstance(row, dict): plain lists are not normalized like DataFrame rows; skip None/Series/etc.
-    inner = [
-        record
-        for row in rows or []
-        if isinstance(row, dict) and (record := _client_record_from_graph_row(row)) is not None
-    ]
+    inner = [record for row in graph_rows if (record := _client_record_from_graph_row(row)) is not None]
+    if not inner and any(_row_has_uploadable_content_without_embedding(row) for row in graph_rows):
+        raise ValueError(
+            "vdb_upload requires embedded records, but no embeddings were found. "
+            "Add an embed stage or provide a supported embedding column."
+        )
     # Preserve legacy contract: no uploadable rows → [], not [[]].
     return [inner] if inner else []
 

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/graph/executor.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+import math
 from typing import Any, Dict, List, Optional, Set
 import pandas as pd
 
@@ -131,6 +132,18 @@ def _make_arrow_pandas_operator_adapter(operator_class: type) -> type[_ArrowPand
     return type(adapter_name, (_ArrowPandasOperatorAdapter,), {})
 
 
+def _preserves_pandas_output(operator_class: type, operator_kwargs: dict[str, Any]) -> bool:
+    """Return whether an operator's heterogeneous rows should stay in pandas."""
+    return bool(
+        getattr(operator_class, "PRESERVE_PANDAS_OUTPUT", False) or operator_kwargs.get("preserve_pandas_output", False)
+    )
+
+
+def _requires_stable_pandas_blocks(nodes: list[Node]) -> bool:
+    """Return whether repartitions must not promote object columns to tensors."""
+    return any(_preserves_pandas_output(node.operator_class, node.operator_kwargs) for node in nodes)
+
+
 def _concurrency_target(concurrency: Any) -> int:
     """Return the largest actor-pool size that resource planning can permit."""
     if isinstance(concurrency, tuple):
@@ -177,17 +190,20 @@ def preflight_executors(executors: list[Any], cluster_resources: ClusterResource
     fixed = [item for item in entries if not item[7]]
     auto = [item for item in entries if item[7]]
     fixed_cpu = sum(item[3] * item[5] for item in fixed)
+    source_cpu_reservation = sum(executor._source_cpu_reservation for executor in executors)
     fixed_gpu = sum(item[3] * item[6] for item in fixed)
     min_cpu = sum(item[4] * item[5] for item in auto)
     min_gpu = sum(item[4] * item[6] for item in auto)
-    if fixed_cpu + min_cpu > available_cpus or fixed_gpu + min_gpu > available_gpus:
+    requested_cpu = source_cpu_reservation + fixed_cpu + min_cpu
+    if requested_cpu > available_cpus or fixed_gpu + min_gpu > available_gpus:
         raise ValueError(
             "Infeasible Ray CPU/GPU plan: requested at least "
-            f"{fixed_cpu + min_cpu:g} CPUs and {fixed_gpu + min_gpu:g} GPUs, but Ray reports "
+            f"{requested_cpu:g} CPUs (including {source_cpu_reservation:g} for source reads) "
+            f"and {fixed_gpu + min_gpu:g} GPUs, but Ray reports "
             f"{available_cpus} CPUs and {available_gpus} GPUs available. "
             "Reduce explicit *_workers or node_overrides concurrency, or wait for cluster capacity."
         )
-    used_cpu, used_gpu = fixed_cpu + min_cpu, fixed_gpu + min_gpu
+    used_cpu, used_gpu = source_cpu_reservation + fixed_cpu + min_cpu, fixed_gpu + min_gpu
     planned = {(id(item[0]), item[1]): item[4] for item in auto}
     while True:
         candidates = sorted(
@@ -208,9 +224,12 @@ def preflight_executors(executors: list[Any], cluster_resources: ClusterResource
         used_cpu += selected[5]
         used_gpu += selected[6]
     for executor, name, concurrency, _target, _initial, _cpu, _gpu, _auto in auto:
-        executor._node_overrides[name]["concurrency"] = _planned_concurrency(concurrency, planned[(id(executor), name)])
+        executor._node_overrides.setdefault(name, {})["concurrency"] = _planned_concurrency(
+            concurrency, planned[(id(executor), name)]
+        )
     for executor in executors:
         executor._resources_preflight_complete = True
+        executor._preflight_source_cpu_reservation = executor._source_cpu_reservation
         executor._preflight_cluster_resources = cluster_resources
 
 
@@ -360,10 +379,19 @@ class RayDataExecutor(AbstractExecutor):
         num_gpus: float = 0,
         node_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
         auto_concurrency_nodes: Optional[Set[str]] = None,
+        source_cpu_reservation: float = 0,
     ) -> None:
         super().__init__(graph)
+        source_cpu_reservation = float(source_cpu_reservation)
+        if not math.isfinite(source_cpu_reservation) or source_cpu_reservation < 0:
+            raise ValueError("source_cpu_reservation must be a finite, non-negative CPU value.")
         self._preflight_cluster_resources: ClusterResources | None = None
         self._ray_address = ray_address
+        self._source_cpu_reservation = source_cpu_reservation
+        # ``preflight_executors`` records the source reservation it budgeted.
+        # A filesystem input supplied later must not silently increase that
+        # shared plan: re-planning this executor alone would ignore its peers.
+        self._preflight_source_cpu_reservation: float | None = None
         self._default_batch_size = batch_size
         self._default_batch_format = batch_format
         self._default_num_cpus = num_cpus
@@ -425,15 +453,17 @@ class RayDataExecutor(AbstractExecutor):
         fixed_cpu = sum(count * cpu for _name, _concurrency, count, _initial, cpu, _gpu in fixed)
         fixed_gpu = sum(count * gpu for _name, _concurrency, count, _initial, _cpu, gpu in fixed)
         minimum_cpu = sum(initial * cpu for _name, _concurrency, _count, initial, cpu, _gpu in auto)
+        requested_cpu = self._source_cpu_reservation + fixed_cpu + minimum_cpu
         minimum_gpu = sum(initial * gpu for _name, _concurrency, _count, initial, _cpu, gpu in auto)
-        if fixed_cpu + minimum_cpu > available_cpus or fixed_gpu + minimum_gpu > available_gpus:
+        if requested_cpu > available_cpus or fixed_gpu + minimum_gpu > available_gpus:
             raise ValueError(
                 "Infeasible Ray CPU/GPU plan: requested at least "
-                f"{fixed_cpu + minimum_cpu:g} CPUs and {fixed_gpu + minimum_gpu:g} GPUs, but Ray reports "
+                f"{requested_cpu:g} CPUs (including {self._source_cpu_reservation:g} for source reads) "
+                f"and {fixed_gpu + minimum_gpu:g} GPUs, but Ray reports "
                 f"{available_cpus} CPUs and {available_gpus} GPUs available. "
                 "Reduce explicit *_workers or node_overrides concurrency, or wait for cluster capacity."
             )
-        used_cpu, used_gpu = fixed_cpu + minimum_cpu, fixed_gpu + minimum_gpu
+        used_cpu, used_gpu = self._source_cpu_reservation + fixed_cpu + minimum_cpu, fixed_gpu + minimum_gpu
         planned = {name: initial for name, _concurrency, _count, initial, _cpu, _gpu in auto}
         while True:
             candidates = sorted(
@@ -455,7 +485,7 @@ class RayDataExecutor(AbstractExecutor):
             used_cpu += cpu
             used_gpu += gpu
         for name, concurrency, _count, _initial, _cpu, _gpu in auto:
-            self._node_overrides[name]["concurrency"] = _planned_concurrency(concurrency, planned[name])
+            self._node_overrides.setdefault(name, {})["concurrency"] = _planned_concurrency(concurrency, planned[name])
 
     @staticmethod
     def _linearize(graph: Graph) -> List[Node]:
@@ -510,21 +540,58 @@ class RayDataExecutor(AbstractExecutor):
         ctx = rd.DataContext.get_current()
         ctx.enable_rich_progress_bars = True
         ctx.use_ray_tqdm = False
+        is_filesystem_source = not isinstance(data, rd.Dataset)
+        if is_filesystem_source:
+            required_source_cpu_reservation = 1
+            if self._resources_preflight_complete:
+                planned_source_cpu_reservation = self._preflight_source_cpu_reservation
+                if (
+                    planned_source_cpu_reservation is None
+                    or planned_source_cpu_reservation < required_source_cpu_reservation
+                ):
+                    raise ValueError(
+                        "Filesystem inputs require 1 CPU for Ray Data source reads, but shared Ray resource "
+                        "preflight completed without that reservation. Construct RayDataExecutor with "
+                        "source_cpu_reservation=1 before calling preflight_executors."
+                    )
+            else:
+                self._source_cpu_reservation = required_source_cpu_reservation
 
         cluster = self._preflight_cluster_resources or gather_cluster_resources(ray)
         available_gpus = cluster.available_gpu_count()
         resolved_graph = resolve_graph(self.graph, cluster)
+        nodes = self._linearize(resolved_graph)
+        requires_stable_pandas_blocks = _requires_stable_pandas_blocks(nodes)
 
         if isinstance(data, rd.Dataset):
-            ds = data
+            ds = rd.Dataset.copy(data, _deep_copy=True) if requires_stable_pandas_blocks else data
+            if requires_stable_pandas_blocks:
+                # Ray copies this context into repartition workers. Disabling
+                # Arrow output and tensor promotion only on an operator actor is
+                # too late because Ray converts its result after the call.
+                ds.context.batch_to_block_arrow_format = False
+                ds.context.enable_tensor_extension_casting = False
         else:
             try:
-                ds = rd.read_binary_files(input_paths, include_paths=True)
+                if requires_stable_pandas_blocks:
+                    # read_binary_files snapshots the current context onto the
+                    # new Dataset; restore the process default immediately so
+                    # unrelated pipelines retain Ray's standard behavior.
+                    original_arrow_format = ctx.batch_to_block_arrow_format
+                    original_tensor_extension_casting = ctx.enable_tensor_extension_casting
+                    ctx.batch_to_block_arrow_format = False
+                    ctx.enable_tensor_extension_casting = False
+                try:
+                    ds = rd.read_binary_files(input_paths, include_paths=True)
+                finally:
+                    if requires_stable_pandas_blocks:
+                        ctx.batch_to_block_arrow_format = original_arrow_format
+                        ctx.enable_tensor_extension_casting = original_tensor_extension_casting
             except FileNotFoundError as exc:
                 raise_input_path_not_found(input_paths or [], exc)
-        nodes = self._linearize(resolved_graph)
         if nodes and not self._resources_preflight_complete:
             self._preflight_resources(nodes, cluster.available_cpu_count(), available_gpus)
+        preserve_pandas_output = False
         for node in nodes:
             overrides = dict(self._node_overrides.get(node.name, {}))
             target_num_rows_per_block = overrides.pop("target_num_rows_per_block", None)
@@ -580,7 +647,19 @@ class RayDataExecutor(AbstractExecutor):
                 # offsets for sliced structs with inferred null children.
                 # Compact the valid Arrow batch before that conversion.
                 map_operator_class = _make_arrow_pandas_operator_adapter(node.operator_class)
-                map_batch_format = "pyarrow"
+                stable_pandas_input = preserve_pandas_output
+                # Once an operator reshapes the dataset into heterogeneous
+                # object rows, keep those rows in pandas through every
+                # downstream map stage. Re-enabling Arrow at embedding would
+                # otherwise split optional bbox values into object and tensor
+                # schemas that Ray cannot concatenate.
+                preserve_pandas_output = preserve_pandas_output or _preserves_pandas_output(
+                    node.operator_class, node.operator_kwargs
+                )
+                # The opting-in stage still needs the compacting Arrow adapter
+                # on its input. Its downstream consumers must not ask Ray to
+                # turn the intentionally preserved pandas block back into Arrow.
+                map_batch_format = "pandas" if stable_pandas_input else "pyarrow"
                 constructor_kwargs = {
                     "operator_class": node.operator_class,
                     "operator_kwargs": node.operator_kwargs,

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/graph/ingestor_runtime.py

@@ -363,28 +363,6 @@ def batch_tuning_to_node_overrides(
             plan.pdf_extract_tasks if plan else None,
         )
 
-        # Cap PDF extract concurrency so persistent actors for page-elements,
-        # table structure, OCR, embed, and caption plus fixed pipeline tasks (DocToPdf,
-        # PDFSplit, UDFOperator(s), ReadBinary) cannot exhaust the cluster
-        # CPU budget.
-        if pdf_extract_tasks is not None and cluster_resources is not None:
-            # Conservative fixed overhead for the documented PDF flow:
-            # ReadBinary + DocToPdf + PDFSplit + TextChunk + DedupImages +
-            # the content-reshape UDF before embedding. Caption adds its actor
-            # and one additional UDF.
-            fixed_cpu_overhead = 6 + (2 if caption_params is not None else 0)
-            non_pdf_cpu_overhead = (
-                fixed_cpu_overhead
-                + page_elements_concurrency * page_elements_cpus
-                + ocr_concurrency * ocr_cpus
-                + embed_concurrency * embed_cpus
-                + ts_concurrency * ts_cpus
-            )
-            pdf_extract_tasks = min(
-                pdf_extract_tasks,
-                max(1, int((cluster_resources.total_cpu_count() - non_pdf_cpu_overhead) // pdf_extract_cpus)),
-            )
-
         _set(PDFExtractionActor.__name__, "batch_size", pdf_bs)
         _set(PDFExtractionActor.__name__, "concurrency", pdf_extract_tasks)
         _set(PDFExtractionActor.__name__, "num_cpus", pdf_extract_cpus if pdf_extract_cpus != 1.0 else None)
@@ -578,6 +556,7 @@ def _append_ordered_transform_stages(
                             content_columns=content_columns,
                         ),
                         name="CollapseContentToPageRows",
+                        preserve_pandas_output=True,
                     )
                 else:
                     graph = graph >> UDFOperator(
@@ -590,6 +569,7 @@ def _append_ordered_transform_stages(
                             content_columns=content_columns,
                         ),
                         name="ExplodeContentToRows",
+                        preserve_pandas_output=True,
                     )
             graph = graph >> _BatchEmbedActor(params=embed_params)
 

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/ingestor/branch_extraction.py

@@ -118,6 +118,7 @@ class ExtractionBranchExecutor:
                     graph,
                     derived_overrides,
                     default_concurrency_node_names(effective_extraction.extract_params, None, None, None),
+                    source_cpu_reservation=1 if isinstance(input_data, list) else 0,
                 )
                 branch_executors.append(executor)
                 branch_inputs.append((executor, input_data))
@@ -146,6 +147,7 @@ class ExtractionBranchExecutor:
             post_graph,
             post_overrides,
             default_concurrency_node_names(None, self.embed_params, self.store_params, self.caption_params),
+            source_cpu_reservation=0,
         )
         if hasattr(cluster_resources, "available_cpu_count"):
             preflight_executors([*branch_executors, post_executor], cluster_resources)
@@ -222,6 +224,7 @@ class ExtractionBranchExecutor:
         graph: Any,
         derived_overrides: dict[str, dict[str, Any]],
         auto_concurrency_nodes: set[str],
+        source_cpu_reservation: float,
     ) -> RayDataExecutor:
         return RayDataExecutor(
             graph,
@@ -231,6 +234,7 @@ class ExtractionBranchExecutor:
             num_gpus=self.num_gpus,
             node_overrides=merge_node_overrides(derived_overrides, self.node_overrides),
             auto_concurrency_nodes=auto_concurrency_nodes - set(self.node_overrides),
+            source_cpu_reservation=source_cpu_reservation,
         )
 
     def _inprocess_branch_input(self, branch: ExtractionBranchPlan) -> Any:

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/ingestor/graph_ingestor.py

@@ -65,6 +65,7 @@ from nemo_retriever.common.params import (
     ExtractParams,
     HtmlChunkParams,
     IngestExecuteParams,
+    NO_API_KEY,
     StoreParams,
     TextChunkParams,
     VideoFrameParams,
@@ -382,6 +383,9 @@ def _resolve_api_key(params: Any) -> Any:
     """Auto-resolve api_key from NVIDIA_API_KEY / NGC_API_KEY if not explicitly set."""
     if params is None:
         return params
+    uses_no_api_key = getattr(params, "_uses_no_api_key", None)
+    if callable(uses_no_api_key) and uses_no_api_key("api_key"):
+        return params
     if not getattr(params, "api_key", None) and hasattr(params, "model_copy"):
         key = resolve_remote_api_key()
         if key:
@@ -400,7 +404,20 @@ def _coerce(params: Any, kwargs: dict[str, Any], *, default_factory: Callable[[]
     if not kwargs:
         return params
     if hasattr(params, "model_copy"):
-        return params.model_copy(update=kwargs)
+        values = params.model_dump()
+        uses_no_api_key = getattr(params, "_uses_no_api_key", None)
+        api_key_env_reference = getattr(params, "_api_key_env_reference", None)
+        for field_name in type(params).model_fields:
+            if field_name in kwargs:
+                continue
+            if callable(uses_no_api_key) and uses_no_api_key(field_name):
+                values[field_name] = NO_API_KEY
+                continue
+            if callable(api_key_env_reference):
+                reference = api_key_env_reference(field_name)
+                if reference is not None:
+                    values[field_name] = reference
+        return type(params).model_validate(values | kwargs)
     return params
 
 
@@ -1199,7 +1216,11 @@ class GraphIngestor(ingestor):
                 if requested_columns is None
                 else [c for c in requested_columns if c in available_columns]
             )
-            for row_index, row in batch_df.iterrows():
+            # ``iterrows`` materializes the full frame through NumPy, which is
+            # unsafe for Ray pickled-object columns containing empty arrays.
+            row_values = batch_df.itertuples(index=False, name=None)
+            for row_index, values in zip(batch_df.index, row_values):
+                row = dict(zip(available_columns, values))
                 source_identifier = cls._source_identifier_from_row(row, row_index)
                 for column in target_columns:
                     for record in cls._iter_stage_errors_from_value(row[column]):

{nemo_retriever-2026.8.12.dev181 → nemo_retriever-2026.8.13.dev182}/src/nemo_retriever/operators/dedup.py

@@ -5,8 +5,9 @@
 from __future__ import annotations
 
 import hashlib
-from typing import Any, List, Tuple
+from typing import Any, List, Optional, Tuple
 
+import numpy as np
 import pandas as pd
 
 from nemo_retriever.common.api.internal.mutate.deduplicate import calculate_iou
@@ -15,19 +16,44 @@ from nemo_retriever.common.params import DedupParams
 _STRUCTURED_COLUMNS = ("table", "chart", "infographic")
 
 
+def _as_element_list(value: Any) -> Optional[List[Any]]:
+    """Return an extracted-element cell as a plain list, or ``None`` if unsupported.
+
+    Batch mode delivers structural-element cells as one-dimensional NumPy
+    object arrays (Arrow list columns converted to pandas), while inprocess
+    mode uses Python lists.
+    """
+    if isinstance(value, list):
+        return value
+    if isinstance(value, tuple):
+        return list(value)
+    if isinstance(value, np.ndarray) and value.ndim == 1:
+        return value.tolist()
+    return None
+
+
+def _as_bbox(value: Any) -> Optional[Tuple[float, ...]]:
+    """Return the first four bbox coordinates as a plain tuple."""
+    if isinstance(value, np.ndarray):
+        value = value.tolist()
+    if not isinstance(value, (list, tuple)) or len(value) < 4:
+        return None
+    return tuple(value[:4])
+
+
 def _collect_structured_bboxes(row: pd.Series) -> List[Tuple[float, ...]]:
     """Gather all bounding boxes from tables, charts, and infographics columns."""
     bboxes: List[Tuple[float, ...]] = []
     for col in _STRUCTURED_COLUMNS:
-        items = row.get(col)
-        if not isinstance(items, list):
+        items = _as_element_list(row.get(col))
+        if items is None:
             continue
         for item in items:
             if not isinstance(item, dict):
                 continue
-            bbox = item.get("bbox_xyxy_norm")
-            if bbox and len(bbox) >= 4:
-                bboxes.append(tuple(bbox[:4]))
+            bbox = _as_bbox(item.get("bbox_xyxy_norm"))
+            if bbox is not None:
+                bboxes.append(bbox)
     return bboxes
 
 
@@ -54,9 +80,13 @@ def dedup_images(
     if "images" not in batch_df.columns:
         return batch_df
 
+    # Arrow-backed columns reject assignment of a filtered Python list.
+    if isinstance(batch_df["images"].dtype, pd.ArrowDtype):
+        batch_df["images"] = batch_df["images"].astype(object)
+
     for row_idx, row in batch_df.iterrows():
-        images = row.get("images")
-        if not isinstance(images, list) or not images:
+        images = _as_element_list(row.get("images"))
+        if not images:
             continue
 
         filtered = list(images)
@@ -88,11 +118,10 @@ def dedup_images(
                     if not isinstance(item, dict):
                         surviving.append(item)
                         continue
-                    img_bbox = item.get("bbox_xyxy_norm")
-                    if not img_bbox or len(img_bbox) < 4:
+                    img_bbox_t = _as_bbox(item.get("bbox_xyxy_norm"))
+                    if img_bbox_t is None:
                         surviving.append(item)
                         continue
-                    img_bbox_t = tuple(img_bbox[:4])
                     overlaps = any(calculate_iou(img_bbox_t, sb) >= iou_threshold for sb in structured_bboxes)
                     if not overlaps:
                         surviving.append(item)