pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/iterators/video.py

@@ -1,15 +1,21 @@
+import glob
 import logging
 import math
+import subprocess
 from fractions import Fraction
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Iterator, Literal
 
 import av
 import pandas as pd
 import PIL.Image
 
+import pixeltable as pxt
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
+import pixeltable.utils.av as av_utils
+from pixeltable.env import Env
+from pixeltable.utils.local_store import TempStore
 
 from .base import ComponentIterator
 
@@ -29,12 +35,29 @@ class FrameIterator(ComponentIterator):
             extracted). If `fps` is greater than the frame rate of the video, an error will be raised.
         num_frames: Exact number of frames to extract. The frames will be spaced as evenly as possible. If
             `num_frames` is greater than the number of frames in the video, all frames will be extracted.
+        all_frame_attrs:
+            If True, outputs a `pxt.Json` column `frame_attrs` with the following `pyav`-provided attributes
+            (for more information, see `pyav`'s documentation on
+            [VideoFrame](https://pyav.org/docs/develop/api/video.html#module-av.video.frame) and
+            [Frame](https://pyav.org/docs/develop/api/frame.html)):
+
+            * `index` (`int`)
+            * `pts` (`int | None`)
+            * `dts` (`int | None`)
+            * `time` (`float | None`)
+            * `is_corrupt` (`bool`)
+            * `key_frame` (`bool`)
+            * `pict_type` (`int`)
+            * `interlaced_frame` (`bool`)
+
+            If False, only outputs frame attributes `frame_idx`, `pos_msec`, and `pos_frame` as separate columns.
     """
 
     # Input parameters
     video_path: Path
-    fps: Optional[float]
-    num_frames: Optional[int]
+    fps: float | None
+    num_frames: int | None
+    all_frame_attrs: bool
 
     # Video info
     container: av.container.input.InputContainer
@@ -44,13 +67,15 @@ class FrameIterator(ComponentIterator):
     video_start_time: int
 
     # List of frame indices to be extracted, or None to extract all frames
-    frames_to_extract: Optional[list[int]]
+    frames_to_extract: list[int] | None
 
     # Next frame to extract, as an iterator `pos` index. If `frames_to_extract` is None, this is the same as the
     # frame index in the video. Otherwise, the corresponding video index is `frames_to_extract[next_pos]`.
     next_pos: int
 
-    def __init__(self, video: str, *, fps: Optional[float] = None, num_frames: Optional[int] = None):
+    def __init__(
+        self, video: str, *, fps: float | None = None, num_frames: int | None = None, all_frame_attrs: bool = False
+    ):
         if fps is not None and num_frames is not None:
             raise excs.Error('At most one of `fps` or `num_frames` may be specified')
 
@@ -60,6 +85,7 @@ class FrameIterator(ComponentIterator):
         self.container = av.open(str(video_path))
         self.fps = fps
         self.num_frames = num_frames
+        self.all_frame_attrs = all_frame_attrs
 
         self.video_framerate = self.container.streams.video[0].average_rate
         self.video_time_base = self.container.streams.video[0].time_base
@@ -115,16 +141,17 @@ class FrameIterator(ComponentIterator):
             'video': ts.VideoType(nullable=False),
             'fps': ts.FloatType(nullable=True),
             'num_frames': ts.IntType(nullable=True),
+            'all_frame_attrs': ts.BoolType(nullable=False),
         }
 
     @classmethod
     def output_schema(cls, *args: Any, **kwargs: Any) -> tuple[dict[str, ts.ColumnType], list[str]]:
-        return {
-            'frame_idx': ts.IntType(),
-            'pos_msec': ts.FloatType(),
-            'pos_frame': ts.IntType(),
-            'frame': ts.ImageType(),
-        }, ['frame']
+        attrs: dict[str, ts.ColumnType]
+        if kwargs.get('all_frame_attrs'):
+            attrs = {'frame_attrs': ts.JsonType()}
+        else:
+            attrs = {'frame_idx': ts.IntType(), 'pos_msec': ts.FloatType(), 'pos_frame': ts.IntType()}
+        return {**attrs, 'frame': ts.ImageType()}, ['frame']
 
     def __next__(self) -> dict[str, Any]:
         # Determine the frame index in the video corresponding to the iterator index `next_pos`;
@@ -164,8 +191,22 @@ class FrameIterator(ComponentIterator):
                 raise excs.Error(f'Frame {next_video_idx} is missing from the video (video file is corrupt)')
             img = frame.to_image()
             assert isinstance(img, PIL.Image.Image)
-            pos_msec = float(pts * self.video_time_base * 1000)
-            result = {'frame_idx': self.next_pos, 'pos_msec': pos_msec, 'pos_frame': video_idx, 'frame': img}
+            pts_msec = float(pts * self.video_time_base * 1000)
+            result: dict[str, Any] = {'frame': img}
+            if self.all_frame_attrs:
+                attrs = {
+                    'index': video_idx,
+                    'pts': frame.pts,
+                    'dts': frame.dts,
+                    'time': frame.time,
+                    'is_corrupt': frame.is_corrupt,
+                    'key_frame': frame.key_frame,
+                    'pict_type': frame.pict_type,
+                    'interlaced_frame': frame.interlaced_frame,
+                }
+                result['frame_attrs'] = attrs
+            else:
+                result.update({'frame_idx': self.next_pos, 'pos_msec': pts_msec, 'pos_frame': video_idx})
             self.next_pos += 1
             return result
 
@@ -184,3 +225,242 @@ class FrameIterator(ComponentIterator):
         # then the iterator will step forward to the desired frame on the subsequent call to next().
         self.container.seek(seek_pos, backward=True, stream=self.container.streams.video[0])
         self.next_pos = pos
+
+
+class VideoSplitter(ComponentIterator):
+    """
+    Iterator over segments of a video file, which is split into fixed-size segments of length `segment_duration`
+    seconds.
+
+    Args:
+        duration: Video segment duration in seconds
+        overlap: Overlap between consecutive segments in seconds. Only available for `mode='fast'`.
+        min_segment_duration: Drop the last segment if it is smaller than min_segment_duration.
+        mode: Segmentation mode:
+            - `'fast'`: Quick segmentation using stream copy (splits only at keyframes, approximate durations)
+            - `'accurate'`: Precise segmentation with re-encoding (exact durations, slower)
+        video_encoder: Video encoder to use. If not specified, uses the default encoder for the current platform.
+            Only available for `mode='accurate'`.
+        video_encoder_args: Additional arguments to pass to the video encoder. Only available for `mode='accurate'`.
+    """
+
+    # Input parameters
+    video_path: Path
+    segment_duration: float | None
+    segment_times: list[float] | None
+    overlap: float
+    min_segment_duration: float
+    video_encoder: str | None
+    video_encoder_args: dict[str, Any] | None
+
+    # Video metadata
+    video_duration: float
+    video_time_base: Fraction
+    video_start_time: int
+
+    output_iter: Iterator[dict[str, Any]]
+
+    def __init__(
+        self,
+        video: str,
+        *,
+        duration: float | None = None,
+        overlap: float | None = None,
+        min_segment_duration: float | None = None,
+        segment_times: list[float] | None = None,
+        mode: Literal['fast', 'accurate'] = 'accurate',
+        video_encoder: str | None = None,
+        video_encoder_args: dict[str, Any] | None = None,
+    ):
+        Env.get().require_binary('ffmpeg')
+        assert (duration is not None) != (segment_times is not None)
+        if segment_times is not None:
+            assert len(segment_times) > 0
+        if duration is not None:
+            assert duration > 0.0
+            assert duration >= min_segment_duration
+            assert overlap is None or overlap < duration
+
+        video_path = Path(video)
+        assert video_path.exists() and video_path.is_file()
+
+        self.video_path = video_path
+        self.segment_duration = duration
+        self.overlap = overlap if overlap is not None else 0.0
+        self.min_segment_duration = min_segment_duration if min_segment_duration is not None else 0.0
+        self.segment_times = segment_times
+        self.video_encoder = video_encoder
+        self.video_encoder_args = video_encoder_args
+
+        with av.open(str(video_path)) as container:
+            video_stream = container.streams.video[0]
+            self.video_time_base = video_stream.time_base
+            self.video_start_time = video_stream.start_time or 0
+
+        self.output_iter = self.fast_iter() if mode == 'fast' else self.accurate_iter()
+
+    @classmethod
+    def input_schema(cls) -> dict[str, ts.ColumnType]:
+        return {
+            'video': ts.VideoType(nullable=False),
+            'duration': ts.FloatType(nullable=True),
+            'overlap': ts.FloatType(nullable=True),
+            'min_segment_duration': ts.FloatType(nullable=True),
+            'segment_times': ts.JsonType(nullable=True),
+            'mode': ts.StringType(nullable=False),
+            'video_encoder': ts.StringType(nullable=True),
+            'video_encoder_args': ts.JsonType(nullable=True),
+        }
+
+    @classmethod
+    def output_schema(cls, *args: Any, **kwargs: Any) -> tuple[dict[str, ts.ColumnType], list[str]]:
+        param_names = ['duration', 'overlap', 'min_segment_duration', 'segment_times']
+        params = dict(zip(param_names, args))
+        params.update(kwargs)
+
+        segment_duration = params.get('duration')
+        segment_times = params.get('segment_times')
+        overlap = params.get('overlap')
+        min_segment_duration = params.get('min_segment_duration')
+        mode = params.get('mode', 'fast')
+
+        if segment_duration is None and segment_times is None:
+            raise excs.Error('Must specify either duration or segment_times')
+        if segment_duration is not None and segment_times is not None:
+            raise excs.Error('duration and segment_times cannot both be specified')
+        if segment_times is not None:
+            if len(segment_times) == 0:
+                raise excs.Error('segment_times cannot be empty')
+            if overlap is not None:
+                raise excs.Error('overlap cannot be specified with segment_times')
+        if segment_duration is not None:
+            if segment_duration <= 0.0:
+                raise excs.Error('duration must be a positive number')
+            if min_segment_duration is not None and segment_duration < min_segment_duration:
+                raise excs.Error('duration must be at least min_segment_duration')
+            if overlap is not None and overlap >= segment_duration:
+                raise excs.Error('overlap must be less than duration')
+        if mode == 'accurate' and overlap is not None:
+            raise excs.Error("Cannot specify overlap for mode='accurate'")
+        if mode == 'fast':
+            if params.get('video_encoder') is not None:
+                raise excs.Error("Cannot specify video_encoder for mode='fast'")
+            if params.get('video_encoder_args') is not None:
+                raise excs.Error("Cannot specify video_encoder_args for mode='fast'")
+
+        return {
+            'segment_start': ts.FloatType(nullable=False),
+            'segment_start_pts': ts.IntType(nullable=False),
+            'segment_end': ts.FloatType(nullable=False),
+            'segment_end_pts': ts.IntType(nullable=False),
+            'video_segment': ts.VideoType(nullable=False),
+        }, []
+
+    def fast_iter(self) -> Iterator[dict[str, Any]]:
+        segment_path: str = ''
+        try:
+            start_time = 0.0
+            start_pts = 0
+            segment_idx = 0
+            while True:
+                target_duration: float | None
+                if self.segment_duration is not None:
+                    target_duration = self.segment_duration
+                elif self.segment_times is not None and segment_idx < len(self.segment_times):
+                    target_duration = self.segment_times[segment_idx] - start_time
+                else:
+                    target_duration = None  # the rest of the video
+
+                segment_path = str(TempStore.create_path(extension='.mp4'))
+                cmd = av_utils.ffmpeg_clip_cmd(str(self.video_path), segment_path, start_time, target_duration)
+                _ = subprocess.run(cmd, capture_output=True, text=True, check=True)
+
+                # use the actual duration
+                segment_duration = av_utils.get_video_duration(segment_path)
+                if segment_duration - self.overlap == 0.0 or segment_duration < self.min_segment_duration:
+                    # we're done
+                    Path(segment_path).unlink()
+                    return
+
+                segment_end = start_time + segment_duration
+                segment_end_pts = start_pts + round(segment_duration / self.video_time_base)
+                result = {
+                    'segment_start': start_time,
+                    'segment_start_pts': start_pts,
+                    'segment_end': segment_end,
+                    'segment_end_pts': segment_end_pts,
+                    'video_segment': segment_path,
+                }
+                yield result
+
+                start_time = segment_end - self.overlap
+                start_pts = segment_end_pts - round(self.overlap / self.video_time_base)
+
+                segment_idx += 1
+                if self.segment_times is not None and segment_idx > len(self.segment_times):
+                    # We've created all segments including the final segment after the last segment_time
+                    break
+
+        except subprocess.CalledProcessError as e:
+            if segment_path and Path(segment_path).exists():
+                Path(segment_path).unlink()
+            error_msg = f'ffmpeg failed with return code {e.returncode}'
+            if e.stderr:
+                error_msg += f': {e.stderr.strip()}'
+            raise pxt.Error(error_msg) from e
+
+    def accurate_iter(self) -> Iterator[dict[str, Any]]:
+        base_path = TempStore.create_path(extension='')
+        # Use ffmpeg -f segment for accurate segmentation with re-encoding
+        output_pattern = f'{base_path}_segment_%04d.mp4'
+        cmd = av_utils.ffmpeg_segment_cmd(
+            str(self.video_path),
+            output_pattern,
+            segment_duration=self.segment_duration,
+            segment_times=self.segment_times,
+            video_encoder=self.video_encoder,
+            video_encoder_args=self.video_encoder_args,
+        )
+
+        try:
+            _ = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            output_paths = sorted(glob.glob(f'{base_path}_segment_*.mp4'))
+            # TODO: is this actually an error?
+            # if len(output_paths) == 0:
+            #     stderr_output = result.stderr.strip() if result.stderr is not None else ''
+            #     raise pxt.Error(
+            #         f'ffmpeg failed to create output files for commandline: {" ".join(cmd)}\n{stderr_output}'
+            #     )
+            start_time = 0.0
+            start_pts = 0
+            for segment_path in output_paths:
+                segment_duration = av_utils.get_video_duration(segment_path)
+                if segment_duration < self.min_segment_duration:
+                    Path(segment_path).unlink()
+                    return
+
+                result = {
+                    'segment_start': start_time,
+                    'segment_start_pts': start_pts,
+                    'segment_end': start_time + segment_duration,
+                    'segment_end_pts': start_pts + round(segment_duration / self.video_time_base),
+                    'video_segment': segment_path,
+                }
+                yield result
+                start_time += segment_duration
+                start_pts += round(segment_duration / self.video_time_base)
+
+        except subprocess.CalledProcessError as e:
+            error_msg = f'ffmpeg failed with return code {e.returncode}'
+            if e.stderr:
+                error_msg += f': {e.stderr.strip()}'
+            raise pxt.Error(error_msg) from e
+
+    def __next__(self) -> dict[str, Any]:
+        return next(self.output_iter)
+
+    def close(self) -> None:
+        pass
+
+    def set_pos(self, pos: int) -> None:
+        pass

pixeltable/metadata/__init__.py

@@ -18,13 +18,14 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 _logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 37
+VERSION = 41
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:
     """Create the system metadata record"""
     system_md = SystemInfoMd(schema_version=VERSION)
     record = SystemInfo(md=dataclasses.asdict(system_md))
+    _logger.debug(f'Creating pixeltable system info record {record}')
     with orm.Session(engine, future=True) as session:
         # Write system metadata only once for idempotency
         if session.query(SystemInfo).count() == 0:
@@ -54,7 +55,8 @@ for _, modname, _ in pkgutil.iter_modules([os.path.dirname(__file__) + '/convert
 def upgrade_md(engine: sql.engine.Engine) -> None:
     """Upgrade the metadata schema to the current version"""
     with orm.Session(engine) as session:
-        system_info = session.query(SystemInfo).one().md
+        # Get exclusive lock on SystemInfo row
+        system_info = session.query(SystemInfo).with_for_update().one().md
         md_version = system_info['schema_version']
         assert isinstance(md_version, int)
         _logger.info(f'Current database version: {md_version}, installed version: {VERSION}')

pixeltable/metadata/converters/convert_18.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     # Migrate a few changed function names
     if k == 'path' and v == 'pixeltable.functions.string.str_format':
         return 'path', 'pixeltable.functions.string.format'

pixeltable/metadata/converters/convert_19.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -28,7 +28,7 @@ def _(engine: sql.engine.Engine) -> None:
                 conn.execute(sql.text(f'ALTER TABLE {store_name} ALTER COLUMN col_{col_id} TYPE TIMESTAMPTZ'))
 
 
-def __update_timestamp_literals(k: Any, v: Any) -> Optional[tuple[Any, Any]]:
+def __update_timestamp_literals(k: Any, v: Any) -> tuple[Any, Any] | None:
     if isinstance(v, dict) and 'val_t' in v:
         # It's a literal with an explicit 'val_t' field. In version 19 this can only mean a
         # timestamp literal, which (in version 19) is stored in the DB as a naive datetime.

pixeltable/metadata/converters/convert_20.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if isinstance(v, dict) and '_classname' in v:
         # The way InlineArray is represented changed in v20. Previously, literal values were stored
         # directly in the Inline expr; now we store them in Literal sub-exprs. This converter

pixeltable/metadata/converters/convert_21.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -24,7 +24,7 @@ def __update_schema_column(schema_column: dict) -> None:
     schema_column['media_validation'] = None
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if isinstance(v, dict) and '_classname' in v and v['_classname'] == 'ColumnRef':
         if 'perform_validation' not in v:
             v['perform_validation'] = False

pixeltable/metadata/converters/convert_22.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if isinstance(v, dict) and '_classname' in v and v['_classname'] == 'DataFrame':
         v['from_clause'] = {'tbls': [v['tbl']], 'join_clauses': []}
         return k, v

pixeltable/metadata/converters/convert_24.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     from pixeltable import func
     from pixeltable.func.globals import resolve_symbol
 

pixeltable/metadata/converters/convert_25.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if k == 'path' and (
         v in ('pixeltable.functions.huggingface.clip_text', 'pixeltable.functions.huggingface.clip_image')
     ):

pixeltable/metadata/converters/convert_26.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     import pixeltable.type_system as ts
     from pixeltable.exprs.literal import Literal
 

pixeltable/metadata/converters/convert_29.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -12,7 +12,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     # Defaults are now stored as literals in signatures
     if k == 'parameters':
         for param in v:
@@ -55,8 +55,8 @@ def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], A
         # We need to expand ("unroll") any var-args or var-kwargs.
 
         new_args_len = len(new_args)
-        rolled_args: Optional[dict] = None
-        rolled_kwargs: Optional[dict] = None
+        rolled_args: dict | None = None
+        rolled_kwargs: dict | None = None
 
         if 'signature' in v['fn']:
             # If it's a pickled function, there's no signature, so we're out of luck; varargs in a pickled function

pixeltable/metadata/converters/convert_34.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -11,7 +11,7 @@ def _(engine: sql.engine.Engine) -> None:
     convert_table_md(engine, substitution_fn=__substitute_md)
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if isinstance(v, dict) and '_classname' in v and v['_classname'] == 'ColumnRef':
         # Add reference_tbl to ColumnRef; for historical metadata it is always equal to tbl
         assert 'reference_tbl' not in v

pixeltable/metadata/converters/convert_36.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, Optional
+from typing import Any
 from uuid import UUID
 
 import sqlalchemy as sql
@@ -30,7 +30,7 @@ def __update_table_md(table_md: dict, table_id: UUID) -> None:
         _logger.info(f'Updating view metadata for table: {table_id}')
 
 
-def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
     if isinstance(v, dict) and (v.get('_classname') == 'DataFrame'):
         if 'sample_clause' not in v:
             v['sample_clause'] = None

pixeltable/metadata/converters/convert_37.py

@@ -0,0 +1,15 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=37)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, _: UUID) -> None:
+    table_md['view_sn'] = 0

pixeltable/metadata/converters/convert_38.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=38)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, substitution_fn=__substitute_md)
+
+
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
+    if k == 'col_mapping':
+        assert isinstance(v, list)
+        return k, [__col_mapping_entry(e) for e in v]
+    if k == 'stored_proxies':
+        assert isinstance(v, list)
+        return k, [__stored_proxies_entry(e) for e in v]
+    return None
+
+
+def __col_mapping_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], str)
+    return [__col_handle(e[0]), e[1]]
+
+
+def __stored_proxies_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], dict)
+    return [__col_handle(e[0]), __col_handle(e[1])]
+
+
+def __col_handle(e: dict) -> dict:
+    return {'tbl_version': {'id': e['tbl_id'], 'effective_version': None}, 'col_id': e['col_id']}