pixeltable 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

pixeltable/metadata/converters/util.py

@@ -0,0 +1,63 @@
+import copy
+import logging
+from typing import Any, Callable, Optional
+
+import sqlalchemy as sql
+
+from pixeltable.metadata.schema import Table
+
+__logger = logging.getLogger('pixeltable')
+
+
+def convert_table_md(
+    engine: sql.engine.Engine,
+    column_md_updater: Optional[Callable[[dict], None]] = None,
+    remote_md_updater: Optional[Callable[[dict], None]] = None,
+    substitution_fn: Optional[Callable[[Any, Any], Optional[tuple[Any, Any]]]] = None
+) -> None:
+    with engine.begin() as conn:
+        for row in conn.execute(sql.select(Table)):
+            id = row[0]
+            table_md = row[2]
+            assert isinstance(table_md, dict)
+            updated_table_md = copy.deepcopy(table_md)
+            if column_md_updater is not None:
+                __update_column_md(updated_table_md, column_md_updater)
+            if remote_md_updater is not None:
+                __update_remote_md(updated_table_md, remote_md_updater)
+            if substitution_fn is not None:
+                updated_table_md = __substitute_md_rec(updated_table_md, substitution_fn)
+            if updated_table_md != table_md:
+                __logger.info(f'Updating schema for table: {id}')
+                conn.execute(sql.update(Table).where(Table.id == id).values(md=updated_table_md))
+
+
+def __update_column_md(table_md: dict, column_md_updater: Callable[[dict], None]) -> None:
+    columns_md = table_md['column_md']
+    assert isinstance(columns_md, dict)
+    for column_md in columns_md.values():
+        column_md_updater(column_md)
+
+
+def __update_remote_md(table_md: dict, remote_md_updater: Callable[[dict], None]) -> None:
+    remotes_md = table_md['remotes']
+    assert isinstance(remotes_md, list)
+    for remote_md in remotes_md:
+        remote_md_updater(remote_md)
+
+
+def __substitute_md_rec(md: Any, substitution_fn: Callable[[Any, Any], Optional[tuple[Any, Any]]]) -> Any:
+    if isinstance(md, dict):
+        updated_md = {}
+        for k, v in md.items():
+            substitute = substitution_fn(k, v)
+            if substitute is not None:
+                updated_k, updated_v = substitute
+                updated_md[updated_k] = updated_v
+            else:
+                updated_md[k] = __substitute_md_rec(v, substitution_fn)
+        return updated_md
+    elif isinstance(md, list):
+        return [__substitute_md_rec(v, substitution_fn) for v in md]
+    else:
+        return md

pixeltable/metadata/schema.py

@@ -1,12 +1,11 @@
-from typing import Optional, List, get_type_hints, Type, Any, TypeVar, Tuple, Union
-import platform
-import uuid
 import dataclasses
+import uuid
+from typing import Optional, List, get_type_hints, Type, Any, TypeVar, Tuple, Union
 
 import sqlalchemy as sql
-from sqlalchemy import Integer, String, Boolean, BigInteger, LargeBinary
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer, BigInteger, LargeBinary
 from sqlalchemy.dialects.postgresql import UUID, JSONB
-from sqlalchemy import ForeignKey, UniqueConstraint, ForeignKeyConstraint
 from sqlalchemy.orm import declarative_base
 
 Base = declarative_base()
@@ -93,6 +92,9 @@ class ColumnMd:
     # if True, the column is present in the stored table
     stored: Optional[bool]
 
+    # if specified, the column is a stored proxy of another column
+    proxy_base: Optional[int]
+
 
 @dataclasses.dataclass
 class IndexMd:
@@ -143,6 +145,10 @@ class TableMd:
     # - every row is assigned a unique and immutable rowid on insertion
     next_row_id: int
 
+    # Metadata format for remotes:
+    # {'class': 'pixeltable.datatransfer.LabelStudioProject', 'md': {'project_id': 3}}
+    remotes: list[dict[str, Any]]
+
     column_md: dict[int, ColumnMd]  # col_id -> ColumnMd
     index_md: dict[int, IndexMd]  # index_id -> IndexMd
     view_md: Optional[ViewMd]
@@ -160,7 +166,7 @@ class Table(Base):
 
     MAX_VERSION = 9223372036854775807  # 2^63 - 1
 
-    id = sql.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, nullable=False)
+    id = sql.Column(UUID(as_uuid=True), primary_key=True, nullable=False)
     dir_id = sql.Column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=False)
     md = sql.Column(JSONB, nullable=False)  # TableMd
 

pixeltable/plan.py

@@ -60,25 +60,10 @@ class Analyzer:
         # filter predicate applied to output rows of the SQL scan
         self.filter: Optional[exprs.Predicate] = None
         # not executable
-        self.similarity_clause: Optional[exprs.ImageSimilarityPredicate] = None
+        #self.similarity_clause: Optional[exprs.ImageSimilarityPredicate] = None
         if where_clause is not None:
             where_clause_conjuncts, self.filter = where_clause.split_conjuncts(lambda e: e.sql_expr() is not None)
             self.sql_where_clause = exprs.CompoundPredicate.make_conjunction(where_clause_conjuncts)
-            if self.filter is not None:
-                similarity_clauses, self.filter = self.filter.split_conjuncts(
-                    lambda e: isinstance(e, exprs.ImageSimilarityPredicate))
-                if len(similarity_clauses) > 1:
-                    raise excs.Error(f'More than one nearest() not supported')
-                if len(similarity_clauses) == 1:
-                    if len(self.order_by_clause) > 0:
-                        raise excs.Error((
-                            f'nearest() returns results in order of proximity and cannot be used in conjunction with '
-                            f'order_by()'))
-                    self.similarity_clause = similarity_clauses[0]
-                    img_col = self.similarity_clause.img_col_ref.col
-                    indexed_col_ids = {info.col.id for info in tbl.tbl_version.idxs_by_name.values()}
-                    if img_col.id not in indexed_col_ids:
-                        raise excs.Error(f'nearest() not available for unindexed column {img_col.name}')
 
         # all exprs that are evaluated in Python; not executable
         self.all_exprs = self.select_list.copy()
@@ -204,8 +189,6 @@ class Planner:
         refd_tbl_ids: Set[UUID] = set()
         if where_clause is not None:
             analyzer = cls.analyze(tbl, where_clause)
-            if analyzer.similarity_clause is not None:
-                raise excs.Error('nearest() cannot be used with count()')
             if analyzer.filter is not None:
                 raise excs.Error(f'Filter {analyzer.filter} not expressible in SQL')
             clause_element = analyzer.sql_where_clause.sql_expr()
@@ -268,7 +251,7 @@ class Planner:
         Returns:
             - root node of the plan
             - list of qualified column names that are getting updated
-            - list of columns that are being recomputed
+            - list of user-visible columns that are being recomputed
         """
         # retrieve all stored cols and all target exprs
         assert isinstance(tbl, catalog.TableVersionPath)
@@ -277,7 +260,10 @@ class Planner:
         if len(recompute_targets) > 0:
             recomputed_cols = recompute_targets.copy()
         else:
-            recomputed_cols = target.get_dependent_columns(updated_cols) if cascade else {}
+            recomputed_cols = target.get_dependent_columns(updated_cols) if cascade else set()
+            # regardless of cascade, we need to update all indices on any updated column
+            idx_val_cols = target.get_idx_val_columns(updated_cols)
+            recomputed_cols.update(idx_val_cols)
             # we only need to recompute stored columns (unstored ones are substituted away)
             recomputed_cols = {c for c in recomputed_cols if c.is_stored}
         recomputed_base_cols = {col for col in recomputed_cols if col.tbl == target}
@@ -290,8 +276,8 @@ class Planner:
         recomputed_exprs = \
             [c.value_expr.copy().resolve_computed_cols(resolve_cols=recomputed_base_cols) for c in recomputed_base_cols]
         # recomputed cols reference the new values of the updated cols
-        for col, e in update_targets.items():
-            exprs.Expr.list_substitute(recomputed_exprs, exprs.ColumnRef(col), e)
+        spec = {exprs.ColumnRef(col): e for col, e in update_targets.items()}
+        exprs.Expr.list_substitute(recomputed_exprs, spec)
         select_list.extend(recomputed_exprs)
 
         # we need to retrieve the PK columns of the existing rows
@@ -299,7 +285,8 @@ class Planner:
         all_base_cols = copied_cols + updated_cols + list(recomputed_base_cols)  # same order as select_list
         # update row builder with column information
         [plan.row_builder.add_table_column(col, select_list[i].slot_idx) for i, col in enumerate(all_base_cols)]
-        return plan, [f'{c.tbl.name}.{c.name}' for c in updated_cols + list(recomputed_cols)], list(recomputed_cols)
+        recomputed_user_cols = [c for c in recomputed_cols if c.name is not None]
+        return plan, [f'{c.tbl.name}.{c.name}' for c in updated_cols + recomputed_user_cols], recomputed_user_cols
 
     @classmethod
     def create_view_update_plan(
@@ -570,7 +557,7 @@ class Planner:
         sql_select_list = analyzer.sql_exprs.copy()
         plan = exec.SqlScanNode(
             tbl, row_builder, select_list=sql_select_list, where_clause=analyzer.sql_where_clause,
-            filter=analyzer.filter, similarity_clause=analyzer.similarity_clause, order_by_items=order_by_items,
+            filter=analyzer.filter, order_by_items=order_by_items,
             limit=sql_limit, set_pk=with_pk, exact_version_only=exact_version_only)
         plan = cls._insert_prefetch_node(tbl.tbl_version.id, analyzer.select_list, row_builder, plan)
 

pixeltable/store.py

@@ -66,7 +66,6 @@ class StoreBase:
         """Create self.sa_tbl from self.tbl_version."""
         system_cols = self._create_system_columns()
         all_cols = system_cols.copy()
-        idxs: List[sql.Index] = []
         for col in [c for c in self.tbl_version.cols if c.is_stored]:
             # re-create sql.Column for each column, regardless of whether it already has sa_col set: it was bound
             # to the last sql.Table version we created and cannot be reused
@@ -76,26 +75,18 @@ class StoreBase:
                 all_cols.append(col.sa_errormsg_col)
                 all_cols.append(col.sa_errortype_col)
 
-            # we create an index for:
-            # - scalar columns (except for strings, because long strings can't be used for B-tree indices)
-            # - non-computed video and image columns (they will contain external paths/urls that users might want to
-            #   filter on)
-            if (col.col_type.is_scalar_type() and not col.col_type.is_string_type()) \
-                    or (col.col_type.is_media_type() and not col.is_computed):
-                # index names need to be unique within the Postgres instance
-                idx_name = f'idx_{col.id}_{self.tbl_version.id.hex}'
-                idxs.append(sql.Index(idx_name, col.sa_col))
-
         if self.sa_tbl is not None:
             # if we're called in response to a schema change, we need to remove the old table first
             self.sa_md.remove(self.sa_tbl)
 
+        idxs: List[sql.Index] = []
         # index for all system columns:
         # - base x view joins can be executed as merge joins
         # - speeds up ORDER BY rowid DESC
         # - allows filtering for a particular table version in index scan
         idx_name = f'sys_cols_idx_{self.tbl_version.id.hex}'
         idxs.append(sql.Index(idx_name, *system_cols))
+
         # v_min/v_max indices: speeds up base table scans needed to propagate a base table insert or delete
         idx_name = f'vmin_idx_{self.tbl_version.id.hex}'
         idxs.append(sql.Index(idx_name, self.v_min_col, postgresql_using='brin'))
@@ -201,10 +192,10 @@ class StoreBase:
         if col.records_errors:
             # we also need to create the errormsg and errortype storage cols
             stmt = (f'ALTER TABLE {self._storage_name()} '
-                    f'ADD COLUMN {col.errormsg_store_name()} {StringType().to_sql()} DEFAULT NULL')
+                    f'ADD COLUMN {col.errormsg_store_name()} VARCHAR DEFAULT NULL')
             conn.execute(sql.text(stmt))
             stmt = (f'ALTER TABLE {self._storage_name()} '
-                    f'ADD COLUMN {col.errortype_store_name()} {StringType().to_sql()} DEFAULT NULL')
+                    f'ADD COLUMN {col.errortype_store_name()} VARCHAR DEFAULT NULL')
             conn.execute(sql.text(stmt))
             added_storage_cols.extend([col.errormsg_store_name(), col.errortype_store_name()])
         self.create_sa_tbl()
@@ -264,7 +255,8 @@ class StoreBase:
         return num_excs
 
     def insert_rows(
-            self, exec_plan: ExecNode, conn: sql.engine.Connection, v_min: Optional[int] = None
+            self, exec_plan: ExecNode, conn: sql.engine.Connection, v_min: Optional[int] = None,
+            show_progress: bool = True
     ) -> Tuple[int, int, Set[int]]:
         """Insert rows into the store table and update the catalog table's md
         Returns:
@@ -293,15 +285,16 @@ class StoreBase:
                             self._create_table_row(row, row_builder, media_cols, cols_with_excs, v_min=v_min)
                         num_excs += num_row_exc
                         table_rows.append(table_row)
-                        if progress_bar is None:
-                            warnings.simplefilter("ignore", category=TqdmWarning)
-                            progress_bar = tqdm(
-                                desc=f'Inserting rows into `{self.tbl_version.name}`',
-                                unit=' rows',
-                                ncols=100,
-                                file=sys.stdout
-                            )
-                        progress_bar.update(1)
+                        if show_progress:
+                            if progress_bar is None:
+                                warnings.simplefilter("ignore", category=TqdmWarning)
+                                progress_bar = tqdm(
+                                    desc=f'Inserting rows into `{self.tbl_version.name}`',
+                                    unit=' rows',
+                                    ncols=100,
+                                    file=sys.stdout
+                                )
+                            progress_bar.update(1)
                     self._move_tmp_media_files(table_rows, media_cols, v_min)
                     conn.execute(sql.insert(self.sa_tbl), table_rows)
             if progress_bar is not None:

pixeltable/tool/create_test_db_dump.py

@@ -11,6 +11,7 @@ import toml
 import pixeltable as pxt
 import pixeltable.metadata as metadata
 from pixeltable.env import Env
+from pixeltable.func import Batch
 from pixeltable.type_system import \
     StringType, IntType, FloatType, BoolType, TimestampType, JsonType
 
@@ -29,9 +30,9 @@ class Dumper:
         os.environ['PIXELTABLE_DB'] = db_name
         os.environ['PIXELTABLE_PGDATA'] = str(shared_home / 'pgdata')
 
-        Env.get().set_up(reinit_db=True)
-        self.cl = pxt.Client()
-        self.cl.logging(level=logging.DEBUG, to_stdout=True)
+        Env._init_env(reinit_db=True)
+
+        Env.get().configure_logging(level=logging.DEBUG, to_stdout=True)
 
     def dump_db(self) -> None:
         md_version = metadata.VERSION
@@ -76,8 +77,18 @@ class Dumper:
             'c6': JsonType(nullable=False),
             'c7': JsonType(nullable=False),
         }
-        t = self.cl.create_table('sample_table', schema, primary_key='c2')
+        t = pxt.create_table('sample_table', schema, primary_key='c2')
+
+        # Add columns for InlineArray and InlineDict
         t.add_column(c8=[[1, 2, 3], [4, 5, 6]])
+        t.add_column(c9=[['a', 'b', 'c'], ['d', 'e', 'f']])
+        t.add_column(c10=[t.c1, [t.c1n, t.c2]])
+        t.add_column(c11={'int': 22, 'dict': {'key': 'val'}, 'expr': t.c1})
+
+        # InPredicate
+        t.add_column(isin_1=t.c1.isin(['test string 1', 'test string 2', 'test string 3']))
+        t.add_column(isin_2=t.c2.isin([1, 2, 3, 4, 5]))
+        t.add_column(isin_3=t.c2.isin(t.c6.f5))
 
         # Add columns for .astype converters to ensure they're persisted properly
         t.add_column(c2_as_float=t.c2.astype(FloatType()))
@@ -136,24 +147,48 @@ class Dumper:
             for i in range(num_rows)
         ]
         t.insert(rows)
-        self.cl.create_dir('views')
-        v = self.cl.create_view('views.sample_view', t, filter=(t.c2 < 50))
-        _ = self.cl.create_view('views.sample_snapshot', t, filter=(t.c2 >= 75), is_snapshot=True)
+        pxt.create_dir('views')
+        v = pxt.create_view('views.sample_view', t, filter=(t.c2 < 50))
+        _ = pxt.create_view('views.sample_snapshot', t, filter=(t.c2 >= 75), is_snapshot=True)
+        e = pxt.create_view('views.empty_view', t, filter=t.c2 == 4171780)
+        assert e.count() == 0
         # Computed column using a library function
         v['str_format'] = pxt.functions.string.str_format('{0} {key}', t.c1, key=t.c1)
-        # Computed column using a bespoke udf
-        v['test_udf'] = test_udf(t.c2)
+        # Computed column using a bespoke stored udf
+        v['test_udf'] = test_udf_stored(t.c2)
+        # Computed column using a batched function
+        # (apply this to the empty view, since it's a "heavyweight" function)
+        e['batched'] = pxt.functions.huggingface.clip_text(t.c1, model_id='openai/clip-vit-base-patch32')
+        # computed column using a stored batched function
+        v['test_udf_batched'] = test_udf_stored_batched(t.c1, upper=False)
         # astype
         v['astype'] = t.c1.astype(pxt.FloatType())
-        # computed column using a stored function
-        v['stored'] = t.c1.apply(lambda x: f'Hello, {x}', col_type=pxt.StringType())
 
-
-@pxt.udf
-def test_udf(n: int) -> int:
+        # Add remotes
+        from pixeltable.datatransfer.remote import MockRemote
+        v.link(
+            MockRemote('remote', {'int_field': pxt.IntType()}, {'str_field': pxt.StringType()}),
+            col_mapping={'test_udf': 'int_field', 'c1': 'str_field'}
+        )
+        # We're just trying to test metadata here, so reach "under the covers" and link a fake
+        # Label Studio project without validation (so we don't need a real Label Studio server)
+        from pixeltable.datatransfer.label_studio import LabelStudioProject
+        v.tbl_version_path.tbl_version.link(
+            LabelStudioProject(4171780, media_import_method='file'),
+            col_mapping={'str_format': 'str_format'}
+        )
+
+
+@pxt.udf(_force_stored=True)
+def test_udf_stored(n: int) -> int:
     return n + 1
 
 
+@pxt.udf(batch_size=4, _force_stored=True)
+def test_udf_stored_batched(strings: Batch[str], *, upper: bool = True) -> Batch[str]:
+    return [string.upper() if upper else string.lower() for string in strings]
+
+
 def main() -> None:
     _logger.info("Creating pixeltable test artifact.")
     dumper = Dumper()

pixeltable/type_system.py

@@ -7,7 +7,7 @@ import json
 import typing
 import urllib.parse
 import urllib.request
-from copy import copy
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Optional, Tuple, Dict, Callable, List, Union, Sequence, Mapping
 
@@ -82,7 +82,11 @@ class ColumnType:
 
     def __init__(self, t: Type, nullable: bool = False):
         self._type = t
-        self.nullable = nullable
+        self._nullable = nullable
+
+    @property
+    def nullable(self) -> bool:
+        return self._nullable
 
     @property
     def type_enum(self) -> Type:
@@ -91,6 +95,12 @@ class ColumnType:
     def serialize(self) -> str:
         return json.dumps(self.as_dict())
 
+    def copy(self, nullable: Optional[bool] = None) -> ColumnType:
+        result = deepcopy(self)
+        if nullable is not None:
+            result._nullable = nullable
+        return result
+
     @classmethod
     def serialize_list(cls, type_list: List[ColumnType]) -> str:
         return json.dumps([t.as_dict() for t in type_list])
@@ -177,7 +187,7 @@ class ColumnType:
         if type(self) != type(other):
             return False
         for member_var in vars(self).keys():
-            if member_var == 'nullable':
+            if member_var == '_nullable':
                 continue
             if getattr(self, member_var) != getattr(other, member_var):
                 return False
@@ -225,6 +235,8 @@ class ColumnType:
             return BoolType()
         if isinstance(val, datetime.datetime) or isinstance(val, datetime.date):
             return TimestampType()
+        if isinstance(val, PIL.Image.Image):
+            return ImageType(width=val.width, height=val.height)
         if isinstance(val, np.ndarray):
             col_type = ArrayType.from_literal(val)
             if col_type is not None:
@@ -248,7 +260,7 @@ class ColumnType:
                 # We treat it as the underlying type but with nullable=True.
                 underlying = cls.from_python_type(union_args[0])
                 if underlying is not None:
-                    underlying.nullable = True
+                    underlying._nullable = True
                     return underlying
         else:
             # Discard type parameters to ensure that parameterized types such as `list[T]`
@@ -370,13 +382,6 @@ class ColumnType:
         # types that refer to external media files
         return self.is_image_type() or self.is_video_type() or self.is_audio_type() or self.is_document_type()
 
-    @abc.abstractmethod
-    def to_sql(self) -> str:
-        """
-        Return corresponding Postgres type.
-        """
-        pass
-
     @abc.abstractmethod
     def to_sa_type(self) -> sql.types.TypeEngine:
         """
@@ -404,9 +409,6 @@ class InvalidType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.INVALID, nullable=nullable)
 
-    def to_sql(self) -> str:
-        assert False
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         assert False
 
@@ -432,9 +434,6 @@ class StringType(ColumnType):
                 return None
         return convert
 
-    def to_sql(self) -> str:
-        return 'VARCHAR'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.String()
 
@@ -458,9 +457,6 @@ class IntType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.INT, nullable=nullable)
 
-    def to_sql(self) -> str:
-        return 'BIGINT'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.BigInteger()
 
@@ -473,9 +469,6 @@ class FloatType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.FLOAT, nullable=nullable)
 
-    def to_sql(self) -> str:
-        return 'FLOAT'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.Float()
 
@@ -493,9 +486,6 @@ class BoolType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.BOOL, nullable=nullable)
 
-    def to_sql(self) -> str:
-        return 'BOOLEAN'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.Boolean()
 
@@ -513,9 +503,6 @@ class TimestampType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.TIMESTAMP, nullable=nullable)
 
-    def to_sql(self) -> str:
-        return 'INTEGER'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.TIMESTAMP()
 
@@ -551,14 +538,13 @@ class JsonType(ColumnType):
             }
         return cls(type_spec, nullable=d['nullable'])
 
-    def to_sql(self) -> str:
-        return 'JSONB'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.dialects.postgresql.JSONB()
 
     def print_value(self, val: Any) -> str:
         val_type = self.infer_literal_type(val)
+        if val_type is None:
+            return super().print_value(val)
         if val_type == self:
             return str(val)
         return val_type.print_value(val)
@@ -657,9 +643,6 @@ class ArrayType(ColumnType):
             return np.array(val, dtype=self.numpy_dtype())
         return val
 
-    def to_sql(self) -> str:
-        return 'BYTEA'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.LargeBinary()
 
@@ -762,9 +745,6 @@ class ImageType(ColumnType):
             return img
         return convert
 
-    def to_sql(self) -> str:
-        return 'VARCHAR'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
         return sql.String()
 
@@ -785,11 +765,8 @@ class VideoType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.VIDEO, nullable=nullable)
 
-    def to_sql(self) -> str:
-        # stored as a file path
-        return 'VARCHAR'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
+        # stored as a file path
         return sql.String()
 
     def _validate_literal(self, val: Any) -> None:
@@ -820,11 +797,8 @@ class AudioType(ColumnType):
     def __init__(self, nullable: bool = False):
         super().__init__(self.Type.AUDIO, nullable=nullable)
 
-    def to_sql(self) -> str:
-        # stored as a file path
-        return 'VARCHAR'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
+        # stored as a file path
         return sql.String()
 
     def _validate_literal(self, val: Any) -> None:
@@ -864,11 +838,8 @@ class DocumentType(ColumnType):
         else:
             self._doc_formats = [t for t in self.DocumentFormat]
 
-    def to_sql(self) -> str:
-        # stored as a file path
-        return 'VARCHAR'
-
     def to_sa_type(self) -> sql.types.TypeEngine:
+        # stored as a file path
         return sql.String()
 
     def _validate_literal(self, val: Any) -> None:
@@ -877,11 +848,9 @@ class DocumentType(ColumnType):
     def validate_media(self, val: Any) -> None:
         assert isinstance(val, str)
         from pixeltable.utils.documents import get_document_handle
-        with open(val, 'r', encoding='utf8') as fh:
-            try:
-                s = fh.read()
-                dh = get_document_handle(s)
-                if dh is None:
-                    raise excs.Error(f'Not a recognized document format: {val}')
-            except Exception as e:
-                raise excs.Error(f'Not a recognized document format: {val}') from None
+        try:
+            dh = get_document_handle(val)
+            if dh is None:
+                raise excs.Error(f'Not a recognized document format: {val}')
+        except Exception as e:
+            raise excs.Error(f'Not a recognized document format: {val}') from None