pixeltable 0.4.12__py3-none-any.whl → 0.4.14__py3-none-any.whl

pixeltable/exec/object_store_save_node.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import dataclasses
+import itertools
+import logging
+from collections import defaultdict, deque
+from concurrent import futures
+from pathlib import Path
+from typing import AsyncIterator, Iterator, NamedTuple, Optional
+from uuid import UUID
+
+from pixeltable import exprs
+from pixeltable.utils.object_stores import ObjectOps, ObjectPath, StorageTarget
+
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
+
+_logger = logging.getLogger('pixeltable')
+
+
+class ObjectStoreSaveNode(ExecNode):
+    """Save files into designated object store(s).
+
+    Each row may have multiple files that need to be saved to a destination.
+    Each file may be referenced by more than one column in the row.
+    Each file may have multiple destinations, e.g., S3 bucket and local file system.
+    If there are multiple destinations, the file cannot be moved to any destination
+    until it has been copied to all of the other destinations.
+    Diagrammatically:
+        Row -> [src_path1, src_path2, ...]
+            src_path -> [dest1, dest2, ...]
+                dest1: [row_location1, row_location2, ...]
+    Paths with multiple destinations are removed from the TempStore only after all destination copies are complete.
+
+    TODO:
+    - Process a row at a time and limit the number of in-flight rows to control memory usage
+    """
+
+    QUEUE_DEPTH_HIGH_WATER = 50  # target number of in-flight requests
+    QUEUE_DEPTH_LOW_WATER = 20  # target number of in-flight requests
+    BATCH_SIZE = 16
+    MAX_WORKERS = 15
+
+    class WorkDesignator(NamedTuple):
+        """Specify the source and destination for a WorkItem"""
+
+        src_path: str  # source of the file to be processed
+        destination: Optional[str]  # destination URI for the file to be processed
+
+    class WorkItem(NamedTuple):
+        src_path: Path
+        destination: Optional[str]
+        info: exprs.ColumnSlotIdx  # column info for the file being processed
+        destination_count: int = 1  # number of unique destinations for this file
+
+    retain_input_order: bool  # if True, return rows in the exact order they were received
+    file_col_info: list[exprs.ColumnSlotIdx]
+
+    # execution state
+    num_returned_rows: int
+
+    # ready_rows: rows that are ready to be returned, ordered by row idx;
+    # the implied row idx of ready_rows[0] is num_returned_rows
+    ready_rows: deque[Optional[exprs.DataRow]]
+
+    in_flight_rows: dict[int, ObjectStoreSaveNode.RowState]  # rows with in-flight work; id(row) -> RowState
+    in_flight_requests: dict[
+        futures.Future, WorkDesignator
+    ]  # in-flight requests to save paths: Future -> WorkDesignator
+    in_flight_work: dict[
+        WorkDesignator, list[tuple[exprs.DataRow, exprs.ColumnSlotIdx]]
+    ]  # WorkDesignator -> [(row, info)]
+
+    input_finished: bool
+    row_idx: Iterator[Optional[int]]
+
+    @dataclasses.dataclass
+    class RowState:
+        row: exprs.DataRow
+        idx: Optional[int]  # position in input stream; None if we don't retain input order
+        num_missing: int  # number of references to media files in this row
+        delete_destinations: list[Path]  # paths to delete after all copies are complete
+
+    def __init__(
+        self, tbl_id: UUID, file_col_info: list[exprs.ColumnSlotIdx], input: ExecNode, retain_input_order: bool = True
+    ):
+        # input_/output_exprs=[]: we don't have anything to evaluate
+        super().__init__(input.row_builder, [], [], input)
+        self.retain_input_order = retain_input_order
+        self.file_col_info = file_col_info
+
+        self.num_returned_rows = 0
+        self.ready_rows = deque()
+        self.in_flight_rows = {}
+        self.in_flight_requests = {}
+        self.in_flight_work = {}
+        self.input_finished = False
+        self.row_idx = itertools.count() if retain_input_order else itertools.repeat(None)
+        assert self.QUEUE_DEPTH_HIGH_WATER > self.QUEUE_DEPTH_LOW_WATER
+
+    @property
+    def queued_work(self) -> int:
+        return len(self.in_flight_requests)
+
+    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> Optional[DataRowBatch]:
+        """Get the next batch of input rows, or None if there are no more rows"""
+        try:
+            input_batch = await anext(input_iter)
+            if input_batch is None:
+                self.input_finished = True
+            return input_batch
+        except StopAsyncIteration:
+            self.input_finished = True
+            return None
+
+    async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
+        input_iter = aiter(self.input)
+        with futures.ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            while True:
+                # Create work to fill the queue to the high water mark ... ?without overrunning the in-flight row limit.
+                while not self.input_finished and self.queued_work < self.QUEUE_DEPTH_HIGH_WATER:
+                    input_batch = await self.get_input_batch(input_iter)
+                    if input_batch is not None:
+                        self.__process_input_batch(input_batch, executor)
+
+                # Wait for enough completions to enable more queueing or if we're done
+                while self.queued_work > self.QUEUE_DEPTH_LOW_WATER or (self.input_finished and self.queued_work > 0):
+                    done, _ = futures.wait(self.in_flight_requests, return_when=futures.FIRST_COMPLETED)
+                    self.__process_completions(done, ignore_errors=self.ctx.ignore_errors)
+
+                # Emit results to meet batch size requirements or empty the in-flight row queue
+                if self.__has_ready_batch() or (
+                    len(self.ready_rows) > 0 and self.input_finished and self.queued_work == 0
+                ):
+                    # create DataRowBatch from the first BATCH_SIZE ready rows
+                    batch = DataRowBatch(self.row_builder)
+                    rows = [self.ready_rows.popleft() for _ in range(min(self.BATCH_SIZE, len(self.ready_rows)))]
+                    for row in rows:
+                        assert row is not None
+                        batch.add_row(row)
+                    self.num_returned_rows += len(rows)
+                    _logger.debug(f'returning {len(rows)} rows')
+                    yield batch
+
+                if self.input_finished and self.queued_work == 0 and len(self.ready_rows) == 0:
+                    return
+
+    def __has_ready_batch(self) -> bool:
+        """True if there are >= BATCH_SIZES entries in ready_rows and the first BATCH_SIZE ones are all non-None"""
+        return (
+            sum(int(row is not None) for row in itertools.islice(self.ready_rows, self.BATCH_SIZE)) == self.BATCH_SIZE
+        )
+
+    def __add_ready_row(self, row: exprs.DataRow, row_idx: Optional[int]) -> None:
+        if row_idx is None:
+            self.ready_rows.append(row)
+        else:
+            # extend ready_rows to accommodate row_idx
+            idx = row_idx - self.num_returned_rows
+            if idx >= len(self.ready_rows):
+                self.ready_rows.extend([None] * (idx - len(self.ready_rows) + 1))
+            self.ready_rows[idx] = row
+
+    def __process_completions(self, done: set[futures.Future], ignore_errors: bool) -> None:
+        from pixeltable.utils.local_store import TempStore
+
+        for f in done:
+            work_designator = self.in_flight_requests.pop(f)
+            new_file_url, exc = f.result()
+            if exc is not None and not ignore_errors:
+                raise exc
+            assert new_file_url is not None
+
+            # add the local path/exception to the slots that reference the url
+            for row, info in self.in_flight_work.pop(work_designator):
+                if exc is not None:
+                    self.row_builder.set_exc(row, info.slot_idx, exc)
+                else:
+                    row.file_urls[info.slot_idx] = new_file_url
+
+                state = self.in_flight_rows[id(row)]
+                state.num_missing -= 1
+                if state.num_missing == 0:
+                    # All operations for this row are complete. Delete all files which had multiple destinations
+                    for src_path in state.delete_destinations:
+                        TempStore.delete_media_file(src_path)
+                    del self.in_flight_rows[id(row)]
+                    self.__add_ready_row(row, state.idx)
+
+    def __process_input_row(self, row: exprs.DataRow) -> list[ObjectStoreSaveNode.WorkItem]:
+        """Process a batch of input rows, generating a list of work"""
+        from pixeltable.utils.local_store import LocalStore, TempStore
+
+        # Create a list of work to do for media storage in this row
+        row_idx = next(self.row_idx)
+        row_to_do: list[ObjectStoreSaveNode.WorkItem] = []
+        num_missing = 0
+        unique_destinations: dict[Path, int] = defaultdict(int)  # destination -> count of unique destinations
+
+        for info in self.file_col_info:
+            col, index = info
+            # we may need to store this imagehave yet to store this image
+            if row.prepare_col_val_for_save(index, col):
+                row.file_urls[index] = row.save_media_to_temp(index, col)
+
+            url = row.file_urls[index]
+            if url is None:
+                # nothing to do
+                continue
+
+            assert row.excs[index] is None
+            assert col.col_type.is_media_type()
+
+            destination = info.col.destination
+            soa = None if destination is None else ObjectPath.parse_object_storage_addr(destination, False)
+            if (
+                soa is not None
+                and soa.storage_target == StorageTarget.LOCAL_STORE
+                and LocalStore(soa).resolve_url(url) is not None
+            ):
+                # A local non-default destination was specified, and the url already points there
+                continue
+
+            src_path = LocalStore.file_url_to_path(url)
+            if src_path is None:
+                # The url does not point to a local file, do not attempt to copy/move it
+                continue
+
+            if destination is None and not TempStore.contains_path(src_path):
+                # Do not copy local file URLs to the LocalStore
+                continue
+
+            work_designator = ObjectStoreSaveNode.WorkDesignator(str(src_path), destination)
+            locations = self.in_flight_work.get(work_designator)
+            if locations is not None:
+                # we've already seen this
+                locations.append((row, info))
+                num_missing += 1
+                continue
+
+            work_item = ObjectStoreSaveNode.WorkItem(src_path, destination, info)
+            row_to_do.append(work_item)
+            self.in_flight_work[work_designator] = [(row, info)]
+            num_missing += 1
+            unique_destinations[src_path] += 1
+
+        # Update work items to reflect the number of unique destinations
+        new_to_do = []
+        for work_item in row_to_do:
+            if unique_destinations[work_item.src_path] == 1 and TempStore.contains_path(work_item.src_path):
+                new_to_do.append(work_item)
+            else:
+                new_to_do.append(
+                    ObjectStoreSaveNode.WorkItem(
+                        work_item.src_path,
+                        work_item.destination,
+                        work_item.info,
+                        destination_count=unique_destinations[work_item.src_path] + 1,
+                        # +1 for the TempStore destination
+                    )
+                )
+        delete_destinations = [k for k, v in unique_destinations.items() if v > 1 and TempStore.contains_path(k)]
+        row_to_do = new_to_do
+
+        if len(row_to_do) > 0:
+            self.in_flight_rows[id(row)] = self.RowState(
+                row, row_idx, num_missing, delete_destinations=delete_destinations
+            )
+        else:
+            self.__add_ready_row(row, row_idx)
+        return row_to_do
+
+    def __process_input_batch(self, input_batch: DataRowBatch, executor: futures.ThreadPoolExecutor) -> None:
+        """Process a batch of input rows, submitting temporary files for upload"""
+        work_to_do: list[ObjectStoreSaveNode.WorkItem] = []
+
+        for row in input_batch:
+            row_to_do = self.__process_input_row(row)
+            if len(row_to_do) > 0:
+                work_to_do.extend(row_to_do)
+
+        for work_item in work_to_do:
+            f = executor.submit(self.__persist_media_file, work_item)
+            self.in_flight_requests[f] = ObjectStoreSaveNode.WorkDesignator(
+                str(work_item.src_path), work_item.destination
+            )
+            _logger.debug(f'submitted {work_item}')
+
+    def __persist_media_file(self, work_item: WorkItem) -> tuple[Optional[str], Optional[Exception]]:
+        """Move data from the TempStore to another location"""
+        src_path = work_item.src_path
+        col = work_item.info.col
+        assert col.destination == work_item.destination
+        try:
+            new_file_url = ObjectOps.put_file(col, src_path, work_item.destination_count == 1)
+            return new_file_url, None
+        except Exception as e:
+            _logger.debug(f'Failed to move/copy {src_path}: {e}', exc_info=e)
+            return None, e

pixeltable/exec/sql_node.py

@@ -71,6 +71,13 @@ class SqlNode(ExecNode):
     If set_pk is True, they are added to the end of the result set when creating the SQL statement
     so they can always be referenced as cols[-num_pk_cols:] in the result set.
     The pk_columns consist of the rowid columns of the target table followed by the version number.
+
+    If row_builder contains references to unstored iter columns, expands the select list to include their
+    SQL-materializable subexpressions.
+
+    Args:
+        select_list: output of the query
+        set_pk: if True, sets the primary for each DataRow
     """
 
     tbl: Optional[catalog.TableVersionPath]
@@ -97,14 +104,6 @@ class SqlNode(ExecNode):
         sql_elements: exprs.SqlElementCache,
         set_pk: bool = False,
     ):
-        """
-        If row_builder contains references to unstored iter columns, expands the select list to include their
-        SQL-materializable subexpressions.
-
-        Args:
-            select_list: output of the query
-            set_pk: if True, sets the primary for each DataRow
-        """
         # create Select stmt
         self.sql_elements = sql_elements
         self.tbl = tbl
@@ -374,6 +373,11 @@ class SqlScanNode(SqlNode):
     Materializes data from the store via a Select stmt.
 
     Supports filtering and ordering.
+
+    Args:
+        select_list: output of the query
+        set_pk: if True, sets the primary for each DataRow
+        exact_version_only: tables for which we only want to see rows created at the current version
     """
 
     exact_version_only: list[catalog.TableVersionHandle]
@@ -386,12 +390,6 @@ class SqlScanNode(SqlNode):
         set_pk: bool = False,
         exact_version_only: Optional[list[catalog.TableVersionHandle]] = None,
     ):
-        """
-        Args:
-            select_list: output of the query
-            set_pk: if True, sets the primary for each DataRow
-            exact_version_only: tables for which we only want to see rows created at the current version
-        """
         sql_elements = exprs.SqlElementCache()
         super().__init__(tbl, row_builder, select_list, sql_elements, set_pk=set_pk)
         # create Select stmt
@@ -413,6 +411,11 @@ class SqlScanNode(SqlNode):
 class SqlLookupNode(SqlNode):
     """
     Materializes data from the store via a Select stmt with a WHERE clause that matches a list of key values
+
+    Args:
+        select_list: output of the query
+        sa_key_cols: list of key columns in the store table
+        key_vals: list of key values to look up
     """
 
     def __init__(
@@ -423,12 +426,6 @@ class SqlLookupNode(SqlNode):
         sa_key_cols: list[sql.Column],
         key_vals: list[tuple],
     ):
-        """
-        Args:
-            select_list: output of the query
-            sa_key_cols: list of key columns in the store table
-            key_vals: list of key values to look up
-        """
         sql_elements = exprs.SqlElementCache()
         super().__init__(tbl, row_builder, select_list, sql_elements, set_pk=True)
         # Where clause: (key-col-1, key-col-2, ...) IN ((val-1, val-2, ...), ...)
@@ -444,6 +441,11 @@ class SqlLookupNode(SqlNode):
 class SqlAggregationNode(SqlNode):
     """
     Materializes data from the store via a Select stmt with a WHERE clause that matches a list of key values
+
+    Args:
+        select_list: can contain calls to AggregateFunctions
+        group_by_items: list of expressions to group by
+        limit: max number of rows to return: None = no limit
     """
 
     group_by_items: Optional[list[exprs.Expr]]
@@ -458,12 +460,6 @@ class SqlAggregationNode(SqlNode):
         limit: Optional[int] = None,
         exact_version_only: Optional[list[catalog.TableVersion]] = None,
     ):
-        """
-        Args:
-            select_list: can contain calls to AggregateFunctions
-            group_by_items: list of expressions to group by
-            limit: max number of rows to return: None = no limit
-        """
         self.input_cte, input_col_map = input.to_cte()
         sql_elements = exprs.SqlElementCache(input_col_map)
         super().__init__(None, row_builder, select_list, sql_elements)
@@ -529,6 +525,12 @@ class SqlJoinNode(SqlNode):
 class SqlSampleNode(SqlNode):
     """
     Returns rows sampled from the input node.
+
+    Args:
+        input: SqlNode to sample from
+        select_list: can contain calls to AggregateFunctions
+        sample_clause: specifies the sampling method
+        stratify_exprs: Analyzer processed list of expressions to stratify by.
     """
 
     input_cte: Optional[sql.CTE]
@@ -544,13 +546,6 @@ class SqlSampleNode(SqlNode):
         sample_clause: 'SampleClause',
         stratify_exprs: list[exprs.Expr],
     ):
-        """
-        Args:
-            input: SqlNode to sample from
-            select_list: can contain calls to AggregateFunctions
-            sample_clause: specifies the sampling method
-            stratify_exprs: Analyzer processed list of expressions to stratify by.
-        """
         assert isinstance(input, SqlNode)
         self.input_cte, input_col_map = input.to_cte(keep_pk=True)
         self.pk_count = input.num_pk_cols

pixeltable/exprs/data_row.py

@@ -14,7 +14,7 @@ import PIL.Image
 import sqlalchemy as sql
 
 from pixeltable import catalog, env
-from pixeltable.utils.media_store import MediaStore, TempStore
+from pixeltable.utils.local_store import TempStore
 
 
 class DataRow:
@@ -257,42 +257,48 @@ class DataRow:
             self.vals[idx] = val
         self.has_val[idx] = True
 
-    def flush_img(self, index: int, col: Optional[catalog.Column] = None) -> None:
-        """Save or discard the in-memory value (required to be a PIL.Image.Image)"""
+    def prepare_col_val_for_save(self, index: int, col: Optional[catalog.Column] = None) -> bool:
+        """
+        Prepare to save a column's value into the appropriate store. Discard unneeded values.
+
+        Return:
+            True if the media object in the column needs to be saved.
+        """
         if self.vals[index] is None:
-            return
+            return False
+
+        if self.file_urls[index] is not None:
+            return False
+
         assert self.excs[index] is None
         if self.file_paths[index] is None:
             if col is not None:
-                image = self.vals[index]
-                format = None
-                if isinstance(image, PIL.Image.Image):
-                    # Default to JPEG unless the image has a transparency layer (which isn't supported by JPEG).
-                    # In that case, use WebP instead.
-                    format = 'webp' if image.has_transparency_data else 'jpeg'
-                filepath, url = MediaStore.get().save_media_object(image, col, format=format)
-                self.file_paths[index] = str(filepath)
-                self.file_urls[index] = url
+                # This is a media object that needs to be saved
+                return True
             else:
-                # we discard the content of this cell
+                # This is a media object that we don't care about, so we discard it
                 self.has_val[index] = False
         else:
             # we already have a file for this image, nothing left to do
             pass
+
         self.vals[index] = None
+        return False
 
-    def move_tmp_media_file(self, index: int, col: catalog.Column) -> None:
-        """If a media url refers to data in a temporary file, move the data to a MediaStore"""
-        if self.file_urls[index] is None:
-            return
-        assert self.excs[index] is None
+    def save_media_to_temp(self, index: int, col: catalog.Column) -> str:
+        """Save the media object in the column to the TempStore.
+        Objects cannot be saved directly to general destinations."""
         assert col.col_type.is_media_type()
-        src_path = TempStore.resolve_url(self.file_urls[index])
-        if src_path is None:
-            # The media url does not point to a temporary file, leave it as is
-            return
-        new_file_url = MediaStore.get().relocate_local_media_file(src_path, col)
-        self.file_urls[index] = new_file_url
+        val = self.vals[index]
+        format = None
+        if isinstance(val, PIL.Image.Image):
+            # Default to JPEG unless the image has a transparency layer (which isn't supported by JPEG).
+            # In that case, use WebP instead.
+            format = 'webp' if val.has_transparency_data else 'jpeg'
+        filepath, url = TempStore.save_media_object(val, col, format=format)
+        self.file_paths[index] = str(filepath) if filepath is not None else None
+        self.vals[index] = None
+        return url
 
     @property
     def rowid(self) -> tuple[int, ...]:

pixeltable/exprs/json_path.py

@@ -17,14 +17,15 @@ from .sql_element_cache import SqlElementCache
 
 
 class JsonPath(Expr):
+    """
+    anchor can be None, in which case this is a relative JsonPath and the anchor is set later via set_anchor().
+    scope_idx: for relative paths, index of referenced JsonMapper
+    (0: indicates the immediately preceding JsonMapper, -1: the parent of the immediately preceding mapper, ...)
+    """
+
     def __init__(
         self, anchor: Optional[Expr], path_elements: Optional[list[str | int | slice]] = None, scope_idx: int = 0
     ) -> None:
-        """
-        anchor can be None, in which case this is a relative JsonPath and the anchor is set later via set_anchor().
-        scope_idx: for relative paths, index of referenced JsonMapper
-        (0: indicates the immediately preceding JsonMapper, -1: the parent of the immediately preceding mapper, ...)
-        """
         if path_elements is None:
             path_elements = []
         super().__init__(ts.JsonType(nullable=True))  # JsonPath expressions are always nullable

pixeltable/exprs/row_builder.py

@@ -48,6 +48,12 @@ class RowBuilder:
 
     For ColumnRefs to unstored iterator columns:
     - in order for them to be executable, we also record the iterator args and pass them to the ColumnRef
+
+    Args:
+        output_exprs: list of Exprs to be evaluated
+        columns: list of columns to be materialized
+        input_exprs: list of Exprs that are excluded from evaluation (because they're already materialized)
+    TODO: enforce that output_exprs doesn't overlap with input_exprs?
     """
 
     unique_exprs: ExprSet
@@ -105,13 +111,6 @@ class RowBuilder:
         input_exprs: Iterable[Expr],
         tbl: Optional[catalog.TableVersion] = None,
     ):
-        """
-        Args:
-            output_exprs: list of Exprs to be evaluated
-            columns: list of columns to be materialized
-            input_exprs: list of Exprs that are excluded from evaluation (because they're already materialized)
-        TODO: enforce that output_exprs doesn't overlap with input_exprs?
-        """
         self.unique_exprs: ExprSet[Expr] = ExprSet()  # dependencies precede their dependents
         self.next_slot_idx = 0
         self.stored_img_cols = []
@@ -474,11 +473,6 @@ class RowBuilder:
                     # exceptions get stored in the errortype/-msg properties of the cellmd column
                     table_row.append(ColumnPropertyRef.create_cellmd_exc(exc))
             else:
-                if col.col_type.is_media_type():
-                    if col.col_type.is_image_type() and data_row.file_urls[slot_idx] is None:
-                        # we have yet to store this image
-                        data_row.flush_img(slot_idx, col)
-                    data_row.move_tmp_media_file(slot_idx, col)
                 val = data_row.get_stored_val(slot_idx, col.get_sa_col_type())
                 table_row.append(val)
                 if col.stores_cellmd:

pixeltable/functions/gemini.py

@@ -15,7 +15,7 @@ import PIL.Image
 import pixeltable as pxt
 from pixeltable import env, exceptions as excs, exprs
 from pixeltable.utils.code import local_public_names
-from pixeltable.utils.media_store import TempStore
+from pixeltable.utils.local_store import TempStore
 
 if TYPE_CHECKING:
     from google import genai

pixeltable/functions/openai.py

@@ -23,7 +23,7 @@ import pixeltable as pxt
 from pixeltable import env, exprs, type_system as ts
 from pixeltable.func import Batch, Tools
 from pixeltable.utils.code import local_public_names
-from pixeltable.utils.media_store import TempStore
+from pixeltable.utils.local_store import TempStore
 
 if TYPE_CHECKING:
     import openai

pixeltable/functions/video.py

@@ -17,7 +17,7 @@ import pixeltable as pxt
 import pixeltable.utils.av as av_utils
 from pixeltable.env import Env
 from pixeltable.utils.code import local_public_names
-from pixeltable.utils.media_store import TempStore
+from pixeltable.utils.local_store import TempStore
 
 _logger = logging.getLogger('pixeltable')
 _format_defaults: dict[str, tuple[str, str]] = {  # format -> (codec, ext)
@@ -49,6 +49,10 @@ class make_video(pxt.Aggregator):
     """
     Aggregator that creates a video from a sequence of images, using the default video encoder and yuv420p pixel format.
 
+    Follows https://pyav.org/docs/develop/cookbook/numpy.html#generating-video
+
+    TODO: provide parameters for video_encoder and pix_fmt
+
     Args:
         fps: Frames per second for the output video.
 
@@ -98,11 +102,6 @@ class make_video(pxt.Aggregator):
     fps: int
 
     def __init__(self, fps: int = 25):
-        """
-        Follows https://pyav.org/docs/develop/cookbook/numpy.html#generating-video
-
-        TODO: provide parameters for video_encoder and pix_fmt
-        """
         self.container = None
         self.stream = None
         self.fps = fps

pixeltable/globals.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, NamedTuple, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Any, Iterable, Literal, NamedTuple, Optional, Union
 
 import pandas as pd
 import pydantic
@@ -24,9 +24,8 @@ if TYPE_CHECKING:
         str,
         os.PathLike,
         Path,  # OS paths, filenames, URLs
-        Iterator[dict[str, Any]],  # iterator producing dictionaries of values
-        RowData,  # list of dictionaries
-        Sequence[pydantic.BaseModel],  # list of Pydantic models
+        Iterable[dict[str, Any]],  # dictionaries of values
+        Iterable[pydantic.BaseModel],  # Pydantic model instances
         DataFrame,  # Pixeltable DataFrame
         pd.DataFrame,  # pandas DataFrame
         datasets.Dataset,
@@ -764,15 +763,15 @@ def ls(path: str = '') -> pd.DataFrame:
                 base = md['base'] or ''
                 if base.startswith('_'):
                     base = '<anonymous base table>'
-                if md['is_snapshot']:
+                if md['is_replica']:
+                    kind = 'replica'
+                elif md['is_snapshot']:
                     kind = 'snapshot'
                 elif md['is_view']:
                     kind = 'view'
                 else:
                     kind = 'table'
                 version = '' if kind == 'snapshot' else str(md['version'])
-                if md['is_replica']:
-                    kind = f'{kind}-replica'
             rows.append([name, kind, version, base])
         return rows