pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/type_system.py

@@ -5,10 +5,14 @@ import datetime
 import enum
 import io
 import json
+import types
 import typing
 import urllib.parse
 import urllib.request
-from typing import Any, ClassVar, Iterable, Literal, Mapping, Optional, Sequence, Union
+from pathlib import Path
+from typing import Any, ClassVar, Iterable, Literal, Mapping, Sequence, Union
+
+from typing import _GenericAlias  # type: ignore[attr-defined]  # isort: skip
 
 import av
 import jsonschema
@@ -21,10 +25,9 @@ import sqlalchemy as sql
 from typing_extensions import _AnnotatedAlias
 
 import pixeltable.exceptions as excs
+from pixeltable.env import Env
 from pixeltable.utils import parse_local_file_path
 
-from typing import _GenericAlias  # type: ignore[attr-defined]  # isort: skip
-
 
 class ColumnType:
     @enum.unique
@@ -48,11 +51,11 @@ class ColumnType:
         @classmethod
         def supertype(
             cls,
-            type1: Optional['ColumnType.Type'],
-            type2: Optional['ColumnType.Type'],
+            type1: 'ColumnType.Type' | None,
+            type2: 'ColumnType.Type' | None,
             # we need to pass this in because we can't easily append it as a class member
             common_supertypes: dict[tuple['ColumnType.Type', 'ColumnType.Type'], 'ColumnType.Type'],
-        ) -> Optional['ColumnType.Type']:
+        ) -> 'ColumnType.Type' | None:
             if type1 == type2:
                 return type1
             t = common_supertypes.get((type1, type2))
@@ -185,7 +188,7 @@ class ColumnType:
         if as_schema:
             return base_str if self.nullable else f'Required[{base_str}]'
         else:
-            return f'Optional[{base_str}]' if self.nullable else base_str
+            return f'{base_str} | None' if self.nullable else base_str
 
     def _to_base_str(self) -> str:
         """
@@ -214,7 +217,7 @@ class ColumnType:
         # Default: just compare base types (this works for all types whose only parameter is nullable)
         return self._type == other._type
 
-    def supertype(self, other: ColumnType) -> Optional[ColumnType]:
+    def supertype(self, other: ColumnType) -> ColumnType | None:
         if self == other:
             return self
         if self.matches(other):
@@ -234,7 +237,7 @@ class ColumnType:
         return None
 
     @classmethod
-    def infer_literal_type(cls, val: Any, nullable: bool = False) -> Optional[ColumnType]:
+    def infer_literal_type(cls, val: Any, nullable: bool = False) -> ColumnType | None:
         if val is None:
             return InvalidType(nullable=True)
         if isinstance(val, str):
@@ -268,7 +271,7 @@ class ColumnType:
         return None
 
     @classmethod
-    def infer_common_literal_type(cls, vals: Iterable[Any]) -> Optional[ColumnType]:
+    def infer_common_literal_type(cls, vals: Iterable[Any]) -> ColumnType | None:
         """
         Returns the most specific type that is a supertype of all literals in `vals`. If no such type
         exists, returns None.
@@ -276,7 +279,7 @@ class ColumnType:
         Args:
             vals: A collection of literals.
         """
-        inferred_type: Optional[ColumnType] = None
+        inferred_type: ColumnType | None = None
         for val in vals:
             val_type = cls.infer_literal_type(val)
             if inferred_type is None:
@@ -291,8 +294,12 @@ class ColumnType:
 
     @classmethod
     def from_python_type(
-        cls, t: Union[type, _GenericAlias], nullable_default: bool = False, allow_builtin_types: bool = True
-    ) -> Optional[ColumnType]:
+        cls,
+        t: type | _GenericAlias,
+        nullable_default: bool = False,
+        allow_builtin_types: bool = True,
+        infer_pydantic_json: bool = False,
+    ) -> ColumnType | None:
         """
         Convert a Python type into a Pixeltable `ColumnType` instance.
 
@@ -304,16 +311,20 @@ class ColumnType:
                 allowed (as in UDF definitions). If False, then only Pixeltable types such as `pxt.String`,
                 `pxt.Int`, etc., will be allowed (as in schema definitions). `Optional` and `Required`
                 designations will be allowed regardless.
+            infer_pydantic_json: If True, accepts an extended set of built-ins (eg, Enum, Path) and returns the type to
+                which pydantic.BaseModel.model_dump(mode='json') serializes it.
         """
         origin = typing.get_origin(t)
         type_args = typing.get_args(t)
-        if origin is typing.Union:
-            # Check if `t` has the form Optional[T].
+        if origin in (typing.Union, types.UnionType):
+            # Check if `t` has the form T | None.
             if len(type_args) == 2 and type(None) in type_args:
-                # `t` is a type of the form Optional[T] (equivalently, Union[T, None] or Union[None, T]).
+                # `t` is a type of the form T | None (equivalently, T | None or None | T).
                 # We treat it as the underlying type but with nullable=True.
                 underlying_py_type = type_args[0] if type_args[1] is type(None) else type_args[1]
-                underlying = cls.from_python_type(underlying_py_type, allow_builtin_types=allow_builtin_types)
+                underlying = cls.from_python_type(
+                    underlying_py_type, allow_builtin_types=allow_builtin_types, infer_pydantic_json=infer_pydantic_json
+                )
                 if underlying is not None:
                     return underlying.copy(nullable=True)
         elif origin is Required:
@@ -327,7 +338,7 @@ class ColumnType:
             if isinstance(parameters, ColumnType):
                 return parameters.copy(nullable=nullable_default)
         else:
-            # It's something other than Optional[T], Required[T], or an explicitly annotated type.
+            # It's something other than T | None, Required[T], or an explicitly annotated type.
             if origin is not None:
                 # Discard type parameters to ensure that parameterized types such as `list[T]`
                 # are correctly mapped to Pixeltable types.
@@ -340,6 +351,13 @@ class ColumnType:
                     if literal_type is None:
                         return None
                     return literal_type.copy(nullable=(literal_type.nullable or nullable_default))
+                if infer_pydantic_json and isinstance(t, type) and issubclass(t, enum.Enum):
+                    literal_type = cls.infer_common_literal_type(member.value for member in t)
+                    if literal_type is None:
+                        return None
+                    return literal_type.copy(nullable=(literal_type.nullable or nullable_default))
+                if infer_pydantic_json and t is Path:
+                    return StringType(nullable=nullable_default)
                 if t is str:
                     return StringType(nullable=nullable_default)
                 if t is int:
@@ -360,10 +378,7 @@ class ColumnType:
 
     @classmethod
     def normalize_type(
-        cls,
-        t: Union[ColumnType, type, _AnnotatedAlias],
-        nullable_default: bool = False,
-        allow_builtin_types: bool = True,
+        cls, t: ColumnType | type | _AnnotatedAlias, nullable_default: bool = False, allow_builtin_types: bool = True
     ) -> ColumnType:
         """
         Convert any type recognizable by Pixeltable to its corresponding ColumnType.
@@ -388,13 +403,43 @@ class ColumnType:
     ]
 
     @classmethod
-    def __raise_exc_for_invalid_type(cls, t: Union[type, _AnnotatedAlias]) -> None:
+    def __raise_exc_for_invalid_type(cls, t: type | _AnnotatedAlias) -> None:
         for builtin_type, suggestion in cls.__TYPE_SUGGESTIONS:
             if t is builtin_type or (isinstance(t, type) and issubclass(t, builtin_type)):
                 name = t.__name__ if t.__module__ == 'builtins' else f'{t.__module__}.{t.__name__}'
                 raise excs.Error(f'Standard Python type `{name}` cannot be used here; use `{suggestion}` instead')
         raise excs.Error(f'Unknown type: {t}')
 
+    @classmethod
+    def from_json_schema(cls, schema: dict[str, Any]) -> ColumnType | None:
+        # We first express the JSON schema as a Python type, and then convert it to a Pixeltable type.
+        # TODO: Is there a meaningful fallback if one of these operations fails? (Maybe another use case for a pxt Any
+        #     type?)
+        py_type = cls.__json_schema_to_py_type(schema)
+        return cls.from_python_type(py_type) if py_type is not None else None
+
+    @classmethod
+    def __json_schema_to_py_type(cls, schema: dict[str, Any]) -> type | _GenericAlias | None:
+        if 'type' in schema:
+            if schema['type'] == 'null':
+                return type(None)
+            if schema['type'] == 'string':
+                return str
+            if schema['type'] == 'integer':
+                return int
+            if schema['type'] == 'number':
+                return float
+            if schema['type'] == 'boolean':
+                return bool
+            if schema['type'] in ('array', 'object'):
+                return list
+        elif 'anyOf' in schema:
+            subscripts = tuple(cls.__json_schema_to_py_type(subschema) for subschema in schema['anyOf'])
+            if all(subscript is not None for subscript in subscripts):
+                return Union[subscripts]
+
+        return None
+
     def validate_literal(self, val: Any) -> None:
         """Raise TypeError if val is not a valid literal for this type"""
         if val is None:
@@ -629,8 +674,9 @@ class TimestampType(ColumnType):
     def _create_literal(self, val: Any) -> Any:
         if isinstance(val, str):
             return datetime.datetime.fromisoformat(val)
-        if isinstance(val, datetime.datetime):
-            return val
+        # Place naive timestamps in the default time zone
+        if isinstance(val, datetime.datetime) and val.tzinfo is None:
+            return val.replace(tzinfo=Env.get().default_time_zone)
         return val
 
 
@@ -658,10 +704,10 @@ class DateType(ColumnType):
 
 
 class JsonType(ColumnType):
-    json_schema: Optional[dict[str, Any]]
-    __validator: Optional[jsonschema.protocols.Validator]
+    json_schema: dict[str, Any] | None
+    __validator: jsonschema.protocols.Validator | None
 
-    def __init__(self, json_schema: Optional[dict[str, Any]] = None, nullable: bool = False):
+    def __init__(self, json_schema: dict[str, Any] | None = None, nullable: bool = False):
         super().__init__(self.Type.JSON, nullable=nullable)
         self.json_schema = json_schema
         if json_schema is None:
@@ -716,7 +762,7 @@ class JsonType(ColumnType):
 
     @classmethod
     def __is_valid_json(cls, val: Any) -> bool:
-        if val is None or isinstance(val, (str, int, float, bool)):
+        if val is None or isinstance(val, (str, int, float, bool, np.ndarray, PIL.Image.Image)):
             return True
         if isinstance(val, (list, tuple)):
             return all(cls.__is_valid_json(v) for v in val)
@@ -731,7 +777,7 @@ class JsonType(ColumnType):
             return val.model_dump()
         return val
 
-    def supertype(self, other: ColumnType) -> Optional[JsonType]:
+    def supertype(self, other: ColumnType) -> JsonType | None:
         # Try using the (much faster) supertype logic in ColumnType first. That will work if, for example, the types
         # are identical except for nullability. If that doesn't work and both types are JsonType, then we will need to
         # merge their schemas.
@@ -753,7 +799,7 @@ class JsonType(ColumnType):
         )
 
     @classmethod
-    def __superschema(cls, a: dict[str, Any], b: dict[str, Any]) -> Optional[dict[str, Any]]:
+    def __superschema(cls, a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any] | None:
         # Defining a general type hierarchy over all JSON schemas would be a challenging problem. In order to keep
         # things manageable, we only define a hierarchy among "conforming" schemas, which provides enough generality
         # for the most important use cases (unions for type inference, validation of inline exprs). A schema is
@@ -813,7 +859,7 @@ class JsonType(ColumnType):
         return {}  # Unresolvable type conflict; the supertype is an unrestricted JsonType.
 
     @classmethod
-    def __superschema_with_nulls(cls, a: dict[str, Any], b: dict[str, Any]) -> Optional[dict[str, Any]]:
+    def __superschema_with_nulls(cls, a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any] | None:
         a, a_nullable = cls.__unpack_null_from_schema(a)
         b, b_nullable = cls.__unpack_null_from_schema(b)
 
@@ -842,15 +888,12 @@ class JsonType(ColumnType):
 
 
 class ArrayType(ColumnType):
-    shape: Optional[tuple[Optional[int], ...]]
-    pxt_dtype: Optional[ColumnType]
-    dtype: Optional[ColumnType.Type]
+    shape: tuple[int | None, ...] | None
+    pxt_dtype: ColumnType | None
+    dtype: ColumnType.Type | None
 
     def __init__(
-        self,
-        shape: Optional[tuple[Optional[int], ...]] = None,
-        dtype: Optional[ColumnType] = None,
-        nullable: bool = False,
+        self, shape: tuple[int | None, ...] | None = None, dtype: ColumnType | None = None, nullable: bool = False
     ):
         super().__init__(self.Type.ARRAY, nullable=nullable)
         assert shape is None or dtype is not None, (shape, dtype)  # cannot specify a shape without a dtype
@@ -875,7 +918,7 @@ class ArrayType(ColumnType):
     def __hash__(self) -> int:
         return hash((self._type, self.nullable, self.shape, self.dtype))
 
-    def supertype(self, other: ColumnType) -> Optional[ArrayType]:
+    def supertype(self, other: ColumnType) -> ArrayType | None:
         basic_supertype = super().supertype(other)
         if basic_supertype is not None:
             assert isinstance(basic_supertype, ArrayType)
@@ -888,7 +931,7 @@ class ArrayType(ColumnType):
         if super_dtype is None:
             # if the dtypes are incompatible, then the supertype is a fully general array
             return ArrayType(nullable=(self.nullable or other.nullable))
-        super_shape: Optional[tuple[Optional[int], ...]]
+        super_shape: tuple[int | None, ...] | None
         if self.shape is None or other.shape is None or len(self.shape) != len(other.shape):
             super_shape = None
         else:
@@ -919,7 +962,7 @@ class ArrayType(ColumnType):
         return cls(shape, dtype, nullable=d['nullable'])
 
     @classmethod
-    def from_np_dtype(cls, dtype: np.dtype, nullable: bool) -> Optional[ColumnType]:
+    def from_np_dtype(cls, dtype: np.dtype, nullable: bool) -> ColumnType | None:
         """
         Return pixeltable type corresponding to a given simple numpy dtype
         """
@@ -948,10 +991,10 @@ class ArrayType(ColumnType):
         return None
 
     @classmethod
-    def from_literal(cls, val: np.ndarray, nullable: bool = False) -> Optional[ArrayType]:
+    def from_literal(cls, val: np.ndarray, nullable: bool = False) -> ArrayType | None:
         # determine our dtype
         assert isinstance(val, np.ndarray)
-        pxttype: Optional[ColumnType] = cls.from_np_dtype(val.dtype, nullable)
+        pxttype: ColumnType | None = cls.from_np_dtype(val.dtype, nullable)
         if pxttype is None:
             return None
         return cls(val.shape, dtype=pxttype, nullable=nullable)
@@ -1014,7 +1057,7 @@ class ArrayType(ColumnType):
     def to_sa_type(cls) -> sql.types.TypeEngine:
         return sql.LargeBinary()
 
-    def numpy_dtype(self) -> Optional[np.dtype]:
+    def numpy_dtype(self) -> np.dtype | None:
         if self.dtype is None:
             return None
         if self.dtype == self.Type.INT:
@@ -1031,15 +1074,13 @@ class ArrayType(ColumnType):
 class ImageType(ColumnType):
     def __init__(
         self,
-        width: Optional[int] = None,
-        height: Optional[int] = None,
-        size: Optional[tuple[int, int]] = None,
-        mode: Optional[str] = None,
+        width: int | None = None,
+        height: int | None = None,
+        size: tuple[int, int] | None = None,
+        mode: str | None = None,
         nullable: bool = False,
     ):
-        """
-        TODO: does it make sense to specify only width or height?
-        """
+        # TODO: does it make sense to specify only width or height?
         super().__init__(self.Type.IMAGE, nullable=nullable)
         assert not (width is not None and size is not None)
         assert not (height is not None and size is not None)
@@ -1077,7 +1118,7 @@ class ImageType(ColumnType):
     def __hash__(self) -> int:
         return hash((self._type, self.nullable, self.size, self.mode))
 
-    def supertype(self, other: ColumnType) -> Optional[ImageType]:
+    def supertype(self, other: ColumnType) -> ImageType | None:
         basic_supertype = super().supertype(other)
         if basic_supertype is not None:
             assert isinstance(basic_supertype, ImageType)
@@ -1092,7 +1133,7 @@ class ImageType(ColumnType):
         return ImageType(width=width, height=height, mode=mode, nullable=(self.nullable or other.nullable))
 
     @property
-    def size(self) -> Optional[tuple[int, int]]:
+    def size(self) -> tuple[int, int] | None:
         if self.width is None or self.height is None:
             return None
         return (self.width, self.height)
@@ -1123,8 +1164,8 @@ class ImageType(ColumnType):
                 img.load()
                 return img
             except Exception as exc:
-                errormsg_val = val if len(val) < 50 else val[:50] + '...'
-                raise excs.Error(f'data URL could not be decoded into a valid image: {errormsg_val}') from exc
+                error_msg_val = val if len(val) < 50 else val[:50] + '...'
+                raise excs.Error(f'data URL could not be decoded into a valid image: {error_msg_val}') from exc
         return val
 
     def _validate_literal(self, val: Any) -> None:
@@ -1211,7 +1252,7 @@ class DocumentType(ColumnType):
         TXT = 4
 
         @classmethod
-        def from_extension(cls, ext: str) -> Optional['DocumentType.DocumentFormat']:
+        def from_extension(cls, ext: str) -> 'DocumentType.DocumentFormat' | None:
             if ext in ('.htm', '.html'):
                 return cls.HTML
             if ext == '.md':
@@ -1224,7 +1265,7 @@ class DocumentType(ColumnType):
                 return cls.TXT
             return None
 
-    def __init__(self, nullable: bool = False, doc_formats: Optional[str] = None):
+    def __init__(self, nullable: bool = False, doc_formats: str | None = None):
         super().__init__(self.Type.DOCUMENT, nullable=nullable)
         self.doc_formats = doc_formats
         if doc_formats is not None:
@@ -1321,13 +1362,13 @@ class Array(np.ndarray, _PxtType):
     def __class_getitem__(cls, item: Any) -> _AnnotatedAlias:
         """
         `item` (the type subscript) must be a tuple with exactly two elements (in any order):
-        - A tuple of `Optional[int]`s, specifying the shape of the array
+        - A tuple of `int | None`s, specifying the shape of the array
         - A type, specifying the dtype of the array
         Example: Array[(3, None, 2), pxt.Float]
         """
         params = item if isinstance(item, tuple) else (item,)
-        shape: Optional[tuple] = None
-        dtype: Optional[ColumnType] = None
+        shape: tuple | None = None
+        dtype: ColumnType | None = None
         if not any(isinstance(param, (type, _AnnotatedAlias)) for param in params):
             raise TypeError('Array type parameter must include a dtype.')
         for param in params:
@@ -1367,8 +1408,8 @@ class Image(PIL.Image.Image, _PxtType):
         else:
             # Not a tuple (single arg)
             params = (item,)
-        size: Optional[tuple] = None
-        mode: Optional[str] = None
+        size: tuple | None = None
+        mode: str | None = None
         for param in params:
             if isinstance(param, tuple):
                 if (

pixeltable/utils/__init__.py

@@ -2,7 +2,6 @@ import hashlib
 import urllib.parse
 import urllib.request
 from pathlib import Path
-from typing import Optional, Union
 
 
 def print_perf_counter_delta(delta: float) -> str:
@@ -24,7 +23,7 @@ def print_perf_counter_delta(delta: float) -> str:
         return f'{delta:.2f} s'
 
 
-def sha256sum(path: Union[Path, str]) -> str:
+def sha256sum(path: Path | str) -> str:
     """
     Compute the SHA256 hash of a file.
     """
@@ -39,7 +38,7 @@ def sha256sum(path: Union[Path, str]) -> str:
     return h.hexdigest()
 
 
-def parse_local_file_path(file_or_url: str) -> Optional[Path]:
+def parse_local_file_path(file_or_url: str) -> Path | None:
     """
     Parses a string that may be either a URL or a local file path.
 

pixeltable/utils/arrow.py

@@ -1,15 +1,22 @@
 import datetime
-from typing import Any, Iterator, Optional, Union
+import io
+import json
+from typing import TYPE_CHECKING, Any, Iterator, cast
 
 import numpy as np
+import PIL.Image
 import pyarrow as pa
 
+import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 
+if TYPE_CHECKING:
+    import pixeltable as pxt
+
 PA_TO_PXT_TYPES: dict[pa.DataType, ts.ColumnType] = {
     pa.string(): ts.StringType(nullable=True),
     pa.large_string(): ts.StringType(nullable=True),
-    pa.timestamp('us', tz=datetime.timezone.utc): ts.TimestampType(nullable=True),
+    pa.timestamp('us', tz='UTC'): ts.TimestampType(nullable=True),
     pa.bool_(): ts.BoolType(nullable=True),
     pa.int8(): ts.IntType(nullable=True),
     pa.int16(): ts.IntType(nullable=True),
@@ -28,7 +35,7 @@ PA_TO_PXT_TYPES: dict[pa.DataType, ts.ColumnType] = {
 
 PXT_TO_PA_TYPES: dict[type[ts.ColumnType], pa.DataType] = {
     ts.StringType: pa.string(),
-    ts.TimestampType: pa.timestamp('us', tz=datetime.timezone.utc),  # postgres timestamp is microseconds
+    ts.TimestampType: pa.timestamp('us', tz='UTC'),  # postgres timestamp is microseconds
     ts.DateType: pa.date32(),  # This could be date64
     ts.BoolType: pa.bool_(),
     ts.IntType: pa.int64(),
@@ -41,7 +48,7 @@ PXT_TO_PA_TYPES: dict[type[ts.ColumnType], pa.DataType] = {
 }
 
 
-def to_pixeltable_type(arrow_type: pa.DataType, nullable: bool) -> Optional[ts.ColumnType]:
+def to_pixeltable_type(arrow_type: pa.DataType, nullable: bool) -> ts.ColumnType | None:
     """Convert a pyarrow DataType to a pixeltable ColumnType if one is defined.
     Returns None if no conversion is currently implemented.
     """
@@ -54,12 +61,12 @@ def to_pixeltable_type(arrow_type: pa.DataType, nullable: bool) -> Optional[ts.C
         dtype = to_pixeltable_type(arrow_type.value_type, nullable)
         if dtype is None:
             return None
-        return ts.ArrayType(shape=arrow_type.shape, dtype=dtype, nullable=nullable)
+        return ts.ArrayType(shape=tuple(arrow_type.shape), dtype=dtype, nullable=nullable)
     else:
         return None
 
 
-def to_arrow_type(pixeltable_type: ts.ColumnType) -> Optional[pa.DataType]:
+def to_arrow_type(pixeltable_type: ts.ColumnType) -> pa.DataType | None:
     """Convert a pixeltable DataType to a pyarrow datatype if one is defined.
     Returns None if no conversion is currently implemented.
     """
@@ -71,7 +78,7 @@ def to_arrow_type(pixeltable_type: ts.ColumnType) -> Optional[pa.DataType]:
         return None
 
 
-def ar_infer_schema(
+def to_pxt_schema(
     arrow_schema: pa.Schema, schema_overrides: dict[str, Any], primary_key: list[str]
 ) -> dict[str, ts.ColumnType]:
     """Convert a pyarrow Schema to a schema using pyarrow names and pixeltable types."""
@@ -85,19 +92,107 @@ def ar_infer_schema(
 
 
 def to_arrow_schema(pixeltable_schema: dict[str, Any]) -> pa.Schema:
-    return pa.schema((name, to_arrow_type(typ)) for name, typ in pixeltable_schema.items())  # type: ignore[misc]
+    return pa.schema((name, to_arrow_type(typ)) for name, typ in pixeltable_schema.items())
+
+
+def _to_record_batch(column_vals: dict[str, list[Any]], schema: pa.Schema) -> pa.RecordBatch:
+    import pyarrow as pa
+
+    pa_arrays: list[pa.Array] = []
+    for field in schema:
+        if isinstance(field.type, pa.FixedShapeTensorType):
+            stacked_arr = np.stack(column_vals[field.name])
+            pa_arrays.append(pa.FixedShapeTensorArray.from_numpy_ndarray(stacked_arr))
+        else:
+            pa_array = cast(pa.Array, pa.array(column_vals[field.name]))
+            pa_arrays.append(pa_array)
+    return pa.RecordBatch.from_arrays(pa_arrays, schema=schema)
+
+
+def to_record_batches(df: 'pxt.DataFrame', batch_size_bytes: int) -> Iterator[pa.RecordBatch]:
+    arrow_schema = to_arrow_schema(df.schema)
+    batch_columns: dict[str, list[Any]] = {k: [] for k in df.schema}
+    current_byte_estimate = 0
+    num_batch_rows = 0
+
+    # TODO: in order to avoid having to deal with ExprEvalError here, DataFrameResultSet should be an iterator
+    # over _exec()
+    try:
+        for data_row in df._exec():
+            num_batch_rows += 1
+            for (col_name, col_type), e in zip(df.schema.items(), df._select_list_exprs):
+                val = data_row[e.slot_idx]
+                val_size_bytes: int
+                if val is None:
+                    batch_columns[col_name].append(val)
+                    continue
+
+                assert val is not None
+                if col_type.is_image_type():
+                    # images get inlined into the parquet file
+                    if data_row.file_paths[e.slot_idx] is not None:
+                        # if there is a file, read directly to preserve information
+                        with open(data_row.file_paths[e.slot_idx], 'rb') as f:
+                            val = f.read()
+                    elif isinstance(val, PIL.Image.Image):
+                        # no file available: save as png
+                        buf = io.BytesIO()
+                        val.save(buf, format='png')
+                        val = buf.getvalue()
+                    else:
+                        raise excs.Error(f'unknown image type {type(val)}')
+                    val_size_bytes = len(val)
+                elif col_type.is_string_type():
+                    val_size_bytes = len(val)
+                elif col_type.is_media_type():
+                    assert data_row.file_paths[e.slot_idx] is not None
+                    val = data_row.file_paths[e.slot_idx]
+                    val_size_bytes = len(val)
+                elif col_type.is_json_type():
+                    val = json.dumps(val)
+                    val_size_bytes = len(val)
+                elif col_type.is_array_type():
+                    val_size_bytes = val.nbytes
+                elif col_type.is_int_type() or col_type.is_float_type():
+                    val_size_bytes = 8
+                elif col_type.is_bool_type():
+                    val_size_bytes = 1
+                elif col_type.is_date_type():
+                    val_size_bytes = 4
+                elif col_type.is_timestamp_type():
+                    val = val.astimezone(datetime.timezone.utc)
+                    val_size_bytes = 8
+                else:
+                    raise excs.Error(f'unknown type {col_type} for {col_name}')
+
+                batch_columns[col_name].append(val)
+                current_byte_estimate += val_size_bytes
+
+            if current_byte_estimate > batch_size_bytes and num_batch_rows > 0:
+                record_batch = _to_record_batch(batch_columns, arrow_schema)
+                yield record_batch
+                batch_columns = {k: [] for k in df.schema}
+                current_byte_estimate = 0
+                num_batch_rows = 0
+
+    except excs.ExprEvalError as e:
+        df._raise_expr_eval_err(e)
+
+    if num_batch_rows > 0:
+        record_batch = _to_record_batch(batch_columns, arrow_schema)
+        yield record_batch
 
 
-def to_pydict(batch: Union[pa.Table, pa.RecordBatch]) -> dict[str, Union[list, np.ndarray]]:
+def to_pydict(batch: pa.Table | pa.RecordBatch) -> dict[str, list | np.ndarray]:
     """Convert a RecordBatch to a dictionary of lists, unlike pa.lib.RecordBatch.to_pydict,
     this function will not convert numpy arrays to lists, and will preserve the original numpy dtype.
     """
-    out: dict[str, Union[list, np.ndarray]] = {}
+    out: dict[str, list | np.ndarray] = {}
     for k, name in enumerate(batch.schema.names):
         col = batch.column(k)
         if isinstance(col.type, pa.FixedShapeTensorType):
             # treat array columns as numpy arrays to easily preserve numpy type
-            out[name] = col.to_numpy(zero_copy_only=False)  # type: ignore[call-arg]
+            out[name] = col.to_numpy(zero_copy_only=False)
         else:
             # for the rest, use pydict to preserve python types
             out[name] = col.to_pylist()
@@ -105,7 +200,7 @@ def to_pydict(batch: Union[pa.Table, pa.RecordBatch]) -> dict[str, Union[list, n
     return out
 
 
-def iter_tuples(batch: Union[pa.Table, pa.RecordBatch]) -> Iterator[dict[str, Any]]:
+def iter_tuples(batch: pa.Table | pa.RecordBatch) -> Iterator[dict[str, Any]]:
     """Convert a RecordBatch to an iterator of dictionaries. also works with pa.Table and pa.RowGroup"""
     pydict = to_pydict(batch)
     assert len(pydict) > 0, 'empty record batch'
@@ -145,7 +240,7 @@ def _ar_val_to_pxt_val(val: Any, pxt_type: ts.ColumnType) -> Any:
 
 
 def iter_tuples2(
-    batch: Union[pa.Table, pa.RecordBatch], col_mapping: Optional[dict[str, str]], schema: dict[str, ts.ColumnType]
+    batch: pa.Table | pa.RecordBatch, col_mapping: dict[str, str] | None, schema: dict[str, ts.ColumnType]
 ) -> Iterator[dict[str, Any]]:
     """Convert a RecordBatch to an iterator of dictionaries. also works with pa.Table and pa.RowGroup"""
     pydict = to_pydict(batch)