pixeltable 0.4.13__py3-none-any.whl → 0.4.15__py3-none-any.whl

pixeltable/exec/cache_prefetch_node.py

@@ -9,12 +9,12 @@ import urllib.request
 from collections import deque
 from concurrent import futures
 from pathlib import Path
-from typing import Any, AsyncIterator, Iterator, Optional
+from typing import AsyncIterator, Iterator, Optional
 from uuid import UUID
 
 from pixeltable import exceptions as excs, exprs
 from pixeltable.utils.filecache import FileCache
-from pixeltable.utils.media_store import TempStore
+from pixeltable.utils.object_stores import ObjectOps
 
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode
@@ -26,16 +26,17 @@ class CachePrefetchNode(ExecNode):
     """Brings files with external URLs into the cache
 
     TODO:
-    - adapting the number of download threads at runtime to maximize throughput
+    - Process a row at a time and limit the number of in-flight rows to control memory usage
+    - Create asyncio.Tasks to consume our input in order to increase concurrency.
     """
 
+    QUEUE_DEPTH_HIGH_WATER = 50  # target number of in-flight requests
+    QUEUE_DEPTH_LOW_WATER = 20  # target number of in-flight requests
     BATCH_SIZE = 16
-    NUM_EXECUTOR_THREADS = 16
+    MAX_WORKERS = 15
 
     retain_input_order: bool  # if True, return rows in the exact order they were received
     file_col_info: list[exprs.ColumnSlotIdx]
-    boto_client: Optional[Any]
-    boto_client_lock: threading.Lock
 
     # execution state
     num_returned_rows: int
@@ -64,10 +65,6 @@ class CachePrefetchNode(ExecNode):
         self.retain_input_order = retain_input_order
         self.file_col_info = file_col_info
 
-        # clients for specific services are constructed as needed, because it's time-consuming
-        self.boto_client = None
-        self.boto_client_lock = threading.Lock()
-
         self.num_returned_rows = 0
         self.ready_rows = deque()
         self.in_flight_rows = {}
@@ -75,24 +72,42 @@ class CachePrefetchNode(ExecNode):
         self.in_flight_urls = {}
         self.input_finished = False
         self.row_idx = itertools.count() if retain_input_order else itertools.repeat(None)
+        assert self.QUEUE_DEPTH_HIGH_WATER > self.QUEUE_DEPTH_LOW_WATER
 
-    async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
-        input_iter = self.input.__aiter__()
-        with futures.ThreadPoolExecutor(max_workers=self.NUM_EXECUTOR_THREADS) as executor:
-            # we create enough in-flight requests to fill the first batch
-            while not self.input_finished and self.__num_pending_rows() < self.BATCH_SIZE:
-                await self.__submit_input_batch(input_iter, executor)
+    @property
+    def queued_work(self) -> int:
+        return len(self.in_flight_requests)
 
-            while True:
-                # try to assemble a full batch of output rows
-                if not self.__has_ready_batch() and len(self.in_flight_requests) > 0:
-                    self.__wait_for_requests()
-
-                # try to create enough in-flight requests to fill the next batch
-                while not self.input_finished and self.__num_pending_rows() < self.BATCH_SIZE:
-                    await self.__submit_input_batch(input_iter, executor)
+    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> Optional[DataRowBatch]:
+        """Get the next batch of input rows, or None if there are no more rows"""
+        try:
+            input_batch = await anext(input_iter)
+            if input_batch is None:
+                self.input_finished = True
+            return input_batch
+        except StopAsyncIteration:
+            self.input_finished = True
+            return None
 
-                if len(self.ready_rows) > 0:
+    async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
+        input_iter = aiter(self.input)
+        with futures.ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            while True:
+                # Create work to fill the queue to the high water mark ... ?without overrunning the in-flight row limit.
+                while not self.input_finished and self.queued_work < self.QUEUE_DEPTH_HIGH_WATER:
+                    input_batch = await self.get_input_batch(input_iter)
+                    if input_batch is not None:
+                        self.__process_input_batch(input_batch, executor)
+
+                # Wait for enough completions to enable more queueing or if we're done
+                while self.queued_work > self.QUEUE_DEPTH_LOW_WATER or (self.input_finished and self.queued_work > 0):
+                    done, _ = futures.wait(self.in_flight_requests, return_when=futures.FIRST_COMPLETED)
+                    self.__process_completions(done, ignore_errors=self.ctx.ignore_errors)
+
+                # Emit results to meet batch size requirements or empty the in-flight row queue
+                if self.__has_ready_batch() or (
+                    len(self.ready_rows) > 0 and self.input_finished and self.queued_work == 0
+                ):
                     # create DataRowBatch from the first BATCH_SIZE ready rows
                     batch = DataRowBatch(self.row_builder)
                     rows = [self.ready_rows.popleft() for _ in range(min(self.BATCH_SIZE, len(self.ready_rows)))]
@@ -103,22 +118,15 @@ class CachePrefetchNode(ExecNode):
                     _logger.debug(f'returning {len(rows)} rows')
                     yield batch
 
-                if self.input_finished and self.__num_pending_rows() == 0:
+                if self.input_finished and self.queued_work == 0 and len(self.ready_rows) == 0:
                     return
 
-    def __num_pending_rows(self) -> int:
-        return len(self.in_flight_rows) + len(self.ready_rows)
-
     def __has_ready_batch(self) -> bool:
         """True if there are >= BATCH_SIZES entries in ready_rows and the first BATCH_SIZE ones are all non-None"""
         return (
             sum(int(row is not None) for row in itertools.islice(self.ready_rows, self.BATCH_SIZE)) == self.BATCH_SIZE
         )
 
-    def __ready_prefix_len(self) -> int:
-        """Length of the non-None prefix of ready_rows (= what we can return right now)"""
-        return sum(1 for _ in itertools.takewhile(lambda x: x is not None, self.ready_rows))
-
     def __add_ready_row(self, row: exprs.DataRow, row_idx: Optional[int]) -> None:
         if row_idx is None:
             self.ready_rows.append(row)
@@ -129,50 +137,36 @@ class CachePrefetchNode(ExecNode):
                 self.ready_rows.extend([None] * (idx - len(self.ready_rows) + 1))
             self.ready_rows[idx] = row
 
-    def __wait_for_requests(self) -> None:
-        """Wait for in-flight requests to complete until we have a full batch of rows"""
+    def __process_completions(self, done: set[futures.Future], ignore_errors: bool) -> None:
         file_cache = FileCache.get()
-        _logger.debug(f'waiting for requests; ready_batch_size={self.__ready_prefix_len()}')
-        while not self.__has_ready_batch() and len(self.in_flight_requests) > 0:
-            done, _ = futures.wait(self.in_flight_requests, return_when=futures.FIRST_COMPLETED)
-            for f in done:
-                url = self.in_flight_requests.pop(f)
-                tmp_path, exc = f.result()
-                local_path: Optional[Path] = None
-                if tmp_path is not None:
-                    # register the file with the cache for the first column in which it's missing
-                    assert url in self.in_flight_urls
-                    _, info = self.in_flight_urls[url][0]
-                    local_path = file_cache.add(info.col.tbl.id, info.col.id, url, tmp_path)
-                    _logger.debug(f'cached {url} as {local_path}')
-
-                # add the local path/exception to the slots that reference the url
-                for row, info in self.in_flight_urls.pop(url):
-                    if exc is not None:
-                        self.row_builder.set_exc(row, info.slot_idx, exc)
-                    else:
-                        assert local_path is not None
-                        row.set_file_path(info.slot_idx, str(local_path))
-                    state = self.in_flight_rows[id(row)]
-                    state.num_missing -= 1
-                    if state.num_missing == 0:
-                        del self.in_flight_rows[id(row)]
-                        self.__add_ready_row(row, state.idx)
-                        _logger.debug(f'row {state.idx} is ready (ready_batch_size={self.__ready_prefix_len()})')
-
-    async def __submit_input_batch(
-        self, input: AsyncIterator[DataRowBatch], executor: futures.ThreadPoolExecutor
-    ) -> None:
-        assert not self.input_finished
-        input_batch: Optional[DataRowBatch]
-        try:
-            input_batch = await anext(input)
-        except StopAsyncIteration:
-            input_batch = None
-        if input_batch is None:
-            self.input_finished = True
-            return
-
+        for f in done:
+            url = self.in_flight_requests.pop(f)
+            tmp_path, exc = f.result()
+            if exc is not None and not ignore_errors:
+                raise exc
+            local_path: Optional[Path] = None
+            if tmp_path is not None:
+                # register the file with the cache for the first column in which it's missing
+                assert url in self.in_flight_urls
+                _, info = self.in_flight_urls[url][0]
+                local_path = file_cache.add(info.col.tbl.id, info.col.id, url, tmp_path)
+                _logger.debug(f'cached {url} as {local_path}')
+
+            # add the local path/exception to the slots that reference the url
+            for row, info in self.in_flight_urls.pop(url):
+                if exc is not None:
+                    self.row_builder.set_exc(row, info.slot_idx, exc)
+                else:
+                    assert local_path is not None
+                    row.set_file_path(info.slot_idx, str(local_path))
+                state = self.in_flight_rows[id(row)]
+                state.num_missing -= 1
+                if state.num_missing == 0:
+                    del self.in_flight_rows[id(row)]
+                    self.__add_ready_row(row, state.idx)
+
+    def __process_input_batch(self, input_batch: DataRowBatch, executor: futures.ThreadPoolExecutor) -> None:
+        """Process a batch of input rows, submitting URLs for download and adding ready rows to ready_rows"""
         file_cache = FileCache.get()
 
         # URLs from this input batch that aren't already in the file cache;
@@ -180,7 +174,7 @@ class CachePrefetchNode(ExecNode):
         # the time it takes to get the next batch together
         cache_misses: list[str] = []
 
-        url_pos: dict[str, int] = {}  # url -> row_idx; used for logging
+        url_pos: dict[str, Optional[int]] = {}  # url -> row_idx; used for logging
         for row in input_batch:
             # identify missing local files in input batch, or fill in their paths if they're already cached
             num_missing = 0
@@ -221,6 +215,8 @@ class CachePrefetchNode(ExecNode):
 
     def __fetch_url(self, url: str) -> tuple[Optional[Path], Optional[Exception]]:
         """Fetches a remote URL into the TempStore and returns its path"""
+        from pixeltable.utils.local_store import TempStore
+
         _logger.debug(f'fetching url={url} thread_name={threading.current_thread().name}')
         parsed = urllib.parse.urlparse(url)
         # Use len(parsed.scheme) > 1 here to ensure we're not being passed
@@ -234,31 +230,11 @@ class CachePrefetchNode(ExecNode):
         tmp_path = TempStore.create_path(extension=extension)
         try:
             _logger.debug(f'Downloading {url} to {tmp_path}')
-            if parsed.scheme == 's3':
-                from pixeltable.utils.s3 import get_client
-
-                with self.boto_client_lock:
-                    if self.boto_client is None:
-                        config = {
-                            'max_pool_connections': self.NUM_EXECUTOR_THREADS + 4,  # +4: leave some headroom
-                            'connect_timeout': 5,
-                            'read_timeout': 30,
-                            'retries': {'max_attempts': 3, 'mode': 'adaptive'},
-                        }
-                        self.boto_client = get_client(**config)
-                self.boto_client.download_file(parsed.netloc, parsed.path.lstrip('/'), str(tmp_path))
-            elif parsed.scheme in ('http', 'https'):
-                with urllib.request.urlopen(url) as resp, open(tmp_path, 'wb') as f:
-                    data = resp.read()
-                    f.write(data)
-            else:
-                raise AssertionError(f'Unsupported URL scheme: {parsed.scheme}')
+            ObjectOps.copy_object_to_local_file(url, tmp_path)
             _logger.debug(f'Downloaded {url} to {tmp_path}')
             return tmp_path, None
         except Exception as e:
             # we want to add the file url to the exception message
             exc = excs.Error(f'Failed to download {url}: {e}')
             _logger.debug(f'Failed to download {url}: {e}', exc_info=e)
-            if not self.ctx.ignore_errors:
-                raise exc from None  # suppress original exception
             return None, exc

pixeltable/exec/data_row_batch.py

@@ -12,15 +12,14 @@ class DataRowBatch:
     """Set of DataRows, indexed by rowid.
 
     Contains the metadata needed to initialize DataRows.
+
+    Requires either num_rows or rows to be specified, but not both.
     """
 
     row_builder: exprs.RowBuilder
     rows: list[exprs.DataRow]
 
     def __init__(self, row_builder: exprs.RowBuilder, rows: Optional[list[exprs.DataRow]] = None):
-        """
-        Requires either num_rows or rows to be specified, but not both.
-        """
         self.row_builder = row_builder
         self.rows = [] if rows is None else rows
 
@@ -39,20 +38,5 @@ class DataRowBatch:
     def __getitem__(self, index: int) -> exprs.DataRow:
         return self.rows[index]
 
-    def flush_imgs(
-        self, idx_range: Optional[slice], stored_img_info: list[exprs.ColumnSlotIdx], flushed_img_slots: list[int]
-    ) -> None:
-        """Flushes images in the given range of rows."""
-        if len(stored_img_info) == 0 and len(flushed_img_slots) == 0:
-            return
-
-        if idx_range is None:
-            idx_range = slice(0, len(self.rows))
-        for row in self.rows[idx_range]:
-            for info in stored_img_info:
-                row.flush_img(info.slot_idx, info.col)
-            for slot_idx in flushed_img_slots:
-                row.flush_img(slot_idx)
-
     def __iter__(self) -> Iterator[exprs.DataRow]:
         return iter(self.rows)

pixeltable/exec/expr_eval/expr_eval_node.py

@@ -390,6 +390,17 @@ class ExprEvalNode(ExecNode):
         # end the main loop if we had an unhandled exception
         try:
             t.result()
+        except KeyboardInterrupt:
+            # ExprEvalNode instances are long-running and reused across multiple operations.
+            # When a user interrupts an operation (Ctrl+C), the main evaluation loop properly
+            # handles the KeyboardInterrupt and terminates the current operation. However,
+            # background tasks spawned by evaluators may complete asynchronously after the
+            # operation has ended, and their done callbacks will fire during subsequent
+            # operations. These "phantom" KeyboardInterrupt exceptions from previous
+            # operations' background tasks should not interfere with new operations, so we
+            # absorb them here rather than propagating them via self.error/self.exc_event.
+            _logger.debug('Task completed with KeyboardInterrupt (user cancellation)')
+            pass
         except asyncio.CancelledError:
             pass
         except Exception as exc:

pixeltable/exec/in_memory_data_node.py

@@ -2,7 +2,7 @@ import logging
 from typing import Any, AsyncIterator, Optional
 
 from pixeltable import catalog, exprs
-from pixeltable.utils.media_store import TempStore
+from pixeltable.utils.local_store import TempStore
 
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode

pixeltable/exec/object_store_save_node.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import dataclasses
+import itertools
+import logging
+from collections import defaultdict, deque
+from concurrent import futures
+from pathlib import Path
+from typing import AsyncIterator, Iterator, NamedTuple, Optional
+from uuid import UUID
+
+from pixeltable import exprs
+from pixeltable.utils.object_stores import ObjectOps, ObjectPath, StorageTarget
+
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
+
+_logger = logging.getLogger('pixeltable')
+
+
+class ObjectStoreSaveNode(ExecNode):
+    """Save files into designated object store(s).
+
+    Each row may have multiple files that need to be saved to a destination.
+    Each file may be referenced by more than one column in the row.
+    Each file may have multiple destinations, e.g., S3 bucket and local file system.
+    If there are multiple destinations, the file cannot be moved to any destination
+    until it has been copied to all of the other destinations.
+    Diagrammatically:
+        Row -> [src_path1, src_path2, ...]
+            src_path -> [dest1, dest2, ...]
+                dest1: [row_location1, row_location2, ...]
+    Paths with multiple destinations are removed from the TempStore only after all destination copies are complete.
+
+    TODO:
+    - Process a row at a time and limit the number of in-flight rows to control memory usage
+    """
+
+    QUEUE_DEPTH_HIGH_WATER = 50  # target number of in-flight requests
+    QUEUE_DEPTH_LOW_WATER = 20  # target number of in-flight requests
+    BATCH_SIZE = 16
+    MAX_WORKERS = 15
+
+    class WorkDesignator(NamedTuple):
+        """Specify the source and destination for a WorkItem"""
+
+        src_path: str  # source of the file to be processed
+        destination: Optional[str]  # destination URI for the file to be processed
+
+    class WorkItem(NamedTuple):
+        src_path: Path
+        destination: Optional[str]
+        info: exprs.ColumnSlotIdx  # column info for the file being processed
+        destination_count: int = 1  # number of unique destinations for this file
+
+    retain_input_order: bool  # if True, return rows in the exact order they were received
+    file_col_info: list[exprs.ColumnSlotIdx]
+
+    # execution state
+    num_returned_rows: int
+
+    # ready_rows: rows that are ready to be returned, ordered by row idx;
+    # the implied row idx of ready_rows[0] is num_returned_rows
+    ready_rows: deque[Optional[exprs.DataRow]]
+
+    in_flight_rows: dict[int, ObjectStoreSaveNode.RowState]  # rows with in-flight work; id(row) -> RowState
+    in_flight_requests: dict[
+        futures.Future, WorkDesignator
+    ]  # in-flight requests to save paths: Future -> WorkDesignator
+    in_flight_work: dict[
+        WorkDesignator, list[tuple[exprs.DataRow, exprs.ColumnSlotIdx]]
+    ]  # WorkDesignator -> [(row, info)]
+
+    input_finished: bool
+    row_idx: Iterator[Optional[int]]
+
+    @dataclasses.dataclass
+    class RowState:
+        row: exprs.DataRow
+        idx: Optional[int]  # position in input stream; None if we don't retain input order
+        num_missing: int  # number of references to media files in this row
+        delete_destinations: list[Path]  # paths to delete after all copies are complete
+
+    def __init__(
+        self, tbl_id: UUID, file_col_info: list[exprs.ColumnSlotIdx], input: ExecNode, retain_input_order: bool = True
+    ):
+        # input_/output_exprs=[]: we don't have anything to evaluate
+        super().__init__(input.row_builder, [], [], input)
+        self.retain_input_order = retain_input_order
+        self.file_col_info = file_col_info
+
+        self.num_returned_rows = 0
+        self.ready_rows = deque()
+        self.in_flight_rows = {}
+        self.in_flight_requests = {}
+        self.in_flight_work = {}
+        self.input_finished = False
+        self.row_idx = itertools.count() if retain_input_order else itertools.repeat(None)
+        assert self.QUEUE_DEPTH_HIGH_WATER > self.QUEUE_DEPTH_LOW_WATER
+
+    @property
+    def queued_work(self) -> int:
+        return len(self.in_flight_requests)
+
+    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> Optional[DataRowBatch]:
+        """Get the next batch of input rows, or None if there are no more rows"""
+        try:
+            input_batch = await anext(input_iter)
+            if input_batch is None:
+                self.input_finished = True
+            return input_batch
+        except StopAsyncIteration:
+            self.input_finished = True
+            return None
+
+    async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
+        input_iter = aiter(self.input)
+        with futures.ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            while True:
+                # Create work to fill the queue to the high water mark ... ?without overrunning the in-flight row limit.
+                while not self.input_finished and self.queued_work < self.QUEUE_DEPTH_HIGH_WATER:
+                    input_batch = await self.get_input_batch(input_iter)
+                    if input_batch is not None:
+                        self.__process_input_batch(input_batch, executor)
+
+                # Wait for enough completions to enable more queueing or if we're done
+                while self.queued_work > self.QUEUE_DEPTH_LOW_WATER or (self.input_finished and self.queued_work > 0):
+                    done, _ = futures.wait(self.in_flight_requests, return_when=futures.FIRST_COMPLETED)
+                    self.__process_completions(done, ignore_errors=self.ctx.ignore_errors)
+
+                # Emit results to meet batch size requirements or empty the in-flight row queue
+                if self.__has_ready_batch() or (
+                    len(self.ready_rows) > 0 and self.input_finished and self.queued_work == 0
+                ):
+                    # create DataRowBatch from the first BATCH_SIZE ready rows
+                    batch = DataRowBatch(self.row_builder)
+                    rows = [self.ready_rows.popleft() for _ in range(min(self.BATCH_SIZE, len(self.ready_rows)))]
+                    for row in rows:
+                        assert row is not None
+                        batch.add_row(row)
+                    self.num_returned_rows += len(rows)
+                    _logger.debug(f'returning {len(rows)} rows')
+                    yield batch
+
+                if self.input_finished and self.queued_work == 0 and len(self.ready_rows) == 0:
+                    return
+
+    def __has_ready_batch(self) -> bool:
+        """True if there are >= BATCH_SIZES entries in ready_rows and the first BATCH_SIZE ones are all non-None"""
+        return (
+            sum(int(row is not None) for row in itertools.islice(self.ready_rows, self.BATCH_SIZE)) == self.BATCH_SIZE
+        )
+
+    def __add_ready_row(self, row: exprs.DataRow, row_idx: Optional[int]) -> None:
+        if row_idx is None:
+            self.ready_rows.append(row)
+        else:
+            # extend ready_rows to accommodate row_idx
+            idx = row_idx - self.num_returned_rows
+            if idx >= len(self.ready_rows):
+                self.ready_rows.extend([None] * (idx - len(self.ready_rows) + 1))
+            self.ready_rows[idx] = row
+
+    def __process_completions(self, done: set[futures.Future], ignore_errors: bool) -> None:
+        from pixeltable.utils.local_store import TempStore
+
+        for f in done:
+            work_designator = self.in_flight_requests.pop(f)
+            new_file_url, exc = f.result()
+            if exc is not None and not ignore_errors:
+                raise exc
+            assert new_file_url is not None
+
+            # add the local path/exception to the slots that reference the url
+            for row, info in self.in_flight_work.pop(work_designator):
+                if exc is not None:
+                    self.row_builder.set_exc(row, info.slot_idx, exc)
+                else:
+                    row.file_urls[info.slot_idx] = new_file_url
+
+                state = self.in_flight_rows[id(row)]
+                state.num_missing -= 1
+                if state.num_missing == 0:
+                    # All operations for this row are complete. Delete all files which had multiple destinations
+                    for src_path in state.delete_destinations:
+                        TempStore.delete_media_file(src_path)
+                    del self.in_flight_rows[id(row)]
+                    self.__add_ready_row(row, state.idx)
+
+    def __process_input_row(self, row: exprs.DataRow) -> list[ObjectStoreSaveNode.WorkItem]:
+        """Process a batch of input rows, generating a list of work"""
+        from pixeltable.utils.local_store import LocalStore, TempStore
+
+        # Create a list of work to do for media storage in this row
+        row_idx = next(self.row_idx)
+        row_to_do: list[ObjectStoreSaveNode.WorkItem] = []
+        num_missing = 0
+        unique_destinations: dict[Path, int] = defaultdict(int)  # destination -> count of unique destinations
+
+        for info in self.file_col_info:
+            col, index = info
+            # we may need to store this imagehave yet to store this image
+            if row.prepare_col_val_for_save(index, col):
+                row.file_urls[index] = row.save_media_to_temp(index, col)
+
+            url = row.file_urls[index]
+            if url is None:
+                # nothing to do
+                continue
+
+            assert row.excs[index] is None
+            assert col.col_type.is_media_type()
+
+            destination = info.col.destination
+            soa = None if destination is None else ObjectPath.parse_object_storage_addr(destination, False)
+            if (
+                soa is not None
+                and soa.storage_target == StorageTarget.LOCAL_STORE
+                and LocalStore(soa).resolve_url(url) is not None
+            ):
+                # A local non-default destination was specified, and the url already points there
+                continue
+
+            src_path = LocalStore.file_url_to_path(url)
+            if src_path is None:
+                # The url does not point to a local file, do not attempt to copy/move it
+                continue
+
+            if destination is None and not TempStore.contains_path(src_path):
+                # Do not copy local file URLs to the LocalStore
+                continue
+
+            work_designator = ObjectStoreSaveNode.WorkDesignator(str(src_path), destination)
+            locations = self.in_flight_work.get(work_designator)
+            if locations is not None:
+                # we've already seen this
+                locations.append((row, info))
+                num_missing += 1
+                continue
+
+            work_item = ObjectStoreSaveNode.WorkItem(src_path, destination, info)
+            row_to_do.append(work_item)
+            self.in_flight_work[work_designator] = [(row, info)]
+            num_missing += 1
+            unique_destinations[src_path] += 1
+
+        # Update work items to reflect the number of unique destinations
+        new_to_do = []
+        for work_item in row_to_do:
+            if unique_destinations[work_item.src_path] == 1 and TempStore.contains_path(work_item.src_path):
+                new_to_do.append(work_item)
+            else:
+                new_to_do.append(
+                    ObjectStoreSaveNode.WorkItem(
+                        work_item.src_path,
+                        work_item.destination,
+                        work_item.info,
+                        destination_count=unique_destinations[work_item.src_path] + 1,
+                        # +1 for the TempStore destination
+                    )
+                )
+        delete_destinations = [k for k, v in unique_destinations.items() if v > 1 and TempStore.contains_path(k)]
+        row_to_do = new_to_do
+
+        if len(row_to_do) > 0:
+            self.in_flight_rows[id(row)] = self.RowState(
+                row, row_idx, num_missing, delete_destinations=delete_destinations
+            )
+        else:
+            self.__add_ready_row(row, row_idx)
+        return row_to_do
+
+    def __process_input_batch(self, input_batch: DataRowBatch, executor: futures.ThreadPoolExecutor) -> None:
+        """Process a batch of input rows, submitting temporary files for upload"""
+        work_to_do: list[ObjectStoreSaveNode.WorkItem] = []
+
+        for row in input_batch:
+            row_to_do = self.__process_input_row(row)
+            if len(row_to_do) > 0:
+                work_to_do.extend(row_to_do)
+
+        for work_item in work_to_do:
+            f = executor.submit(self.__persist_media_file, work_item)
+            self.in_flight_requests[f] = ObjectStoreSaveNode.WorkDesignator(
+                str(work_item.src_path), work_item.destination
+            )
+            _logger.debug(f'submitted {work_item}')
+
+    def __persist_media_file(self, work_item: WorkItem) -> tuple[Optional[str], Optional[Exception]]:
+        """Move data from the TempStore to another location"""
+        src_path = work_item.src_path
+        col = work_item.info.col
+        assert col.destination == work_item.destination
+        try:
+            new_file_url = ObjectOps.put_file(col, src_path, work_item.destination_count == 1)
+            return new_file_url, None
+        except Exception as e:
+            _logger.debug(f'Failed to move/copy {src_path}: {e}', exc_info=e)
+            return None, e