pixeltable 0.0.0__py3-none-any.whl

pixeltable/iterators/document.py

@@ -0,0 +1,432 @@
+import dataclasses
+import enum
+import logging
+from typing import Dict, Any, List, Tuple, Optional, Iterable, Iterator
+
+import ftfy
+
+from pixeltable.env import Env
+from pixeltable.exceptions import Error
+from pixeltable.type_system import ColumnType, DocumentType, StringType, IntType, JsonType
+from pixeltable.utils.documents import get_document_handle
+from .base import ComponentIterator
+
+_logger = logging.getLogger('pixeltable')
+
+class ChunkMetadata(enum.Enum):
+    TITLE = 1
+    HEADING = 2
+    SOURCELINE = 3
+    PAGE = 4
+    BOUNDING_BOX = 5
+
+class Separator(enum.Enum):
+    HEADING = 1
+    PARAGRAPH = 2
+    SENTENCE = 3
+    TOKEN_LIMIT = 4
+    CHAR_LIMIT = 5
+    PAGE = 6
+
+@dataclasses.dataclass
+class DocumentSectionMetadata:
+    """Metadata for a subsection of a document (ie, a structural element like a heading or paragraph)"""
+    # html and markdown metadata
+    sourceline: Optional[int] = None
+    # the stack of headings up to the most recently observed one;
+    # eg, if the most recent one was an h2, 'headings' would contain keys 1 and 2, but nothing below that
+    heading: Optional[Dict[int, str]] = None
+
+    # pdf-specific metadata
+    page: Optional[int] = None
+    # bounding box as an {x1, y1, x2, y2} dictionary
+    bounding_box: Optional[Dict[str, float]] = None
+
+@dataclasses.dataclass
+class DocumentSection:
+    """A single document chunk, according to some of the splitting criteria"""
+    text: Optional[str]
+    metadata: Optional[DocumentSectionMetadata]
+
+
+def _parse_separators(separators: str) -> List[Separator]:
+    ret = []
+    for s in separators.split(','):
+        clean_s = s.strip().upper()
+        if not clean_s:
+            continue
+        if clean_s not in Separator.__members__:
+            raise Error(
+                f'Invalid separator: `{s.strip()}`. Valid separators are: {", ".join(Separator.__members__).lower()}'
+            )
+        ret.append(Separator[clean_s])
+    return ret
+
+
+def _parse_metadata(metadata: str) -> List[ChunkMetadata]:
+    ret = []
+    for m in metadata.split(','):
+        clean_m = m.strip().upper()
+        if not clean_m:
+            continue
+        if clean_m not in ChunkMetadata.__members__:
+            raise Error(
+                f'Invalid metadata: `{m.strip()}`. Valid metadata are: {", ".join(ChunkMetadata.__members__).lower()}'
+            )
+        ret.append(ChunkMetadata[clean_m])
+    return ret
+
+
+_HTML_HEADINGS = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+
+class DocumentSplitter(ComponentIterator):
+    """Iterator over pieces of a document. The document is split into chunks based on the specified separators.
+    The iterator output tuples are of schema {'text': StringType()}, but can include additional metadata fields if specified
+    in the `metadata` argument as explained below.
+    All chunk text is passed through `ftfy.fix_text` to fix up common problems with unicode sequences.
+
+    Args:
+        `metadata`: which additional metadata fields to include in the output schema:
+             'title', 'heading' (HTML and Markdown), 'sourceline' (HTML), 'page' (PDF), 'bounding_box' (PDF).
+             The input can be a comma-separated string of these values eg. 'title,heading,sourceline'.
+        `separators`: which separators to use to split the document into rows. Options are:
+             'heading', 'paragraph', 'sentence', 'token_limit', 'char_limit', 'page'. As with metadata, this is can be a
+                comma-separated string eg. 'heading, token_limit'.
+        `limit`: the maximum number of tokens or characters in each chunk if 'token_limit' or 'char_limit' is specified.
+    """
+    METADATA_COLUMN_TYPES = {
+        ChunkMetadata.TITLE: StringType(nullable=True),
+        ChunkMetadata.HEADING: JsonType(nullable=True),
+        ChunkMetadata.SOURCELINE: IntType(nullable=True),
+        ChunkMetadata.PAGE: IntType(nullable=True),
+        ChunkMetadata.BOUNDING_BOX: JsonType(nullable=True),
+    }
+
+    def __init__(
+            self, document: str, *, separators: str, limit: Optional[int] = None, overlap: Optional[int] = None, metadata: str = '',
+            html_skip_tags: Optional[List[str]] = None, tiktoken_encoding: Optional[str] = 'cl100k_base',
+            tiktoken_target_model: Optional[str] = None
+    ):
+        if html_skip_tags is None:
+            html_skip_tags = ['nav']
+        self._doc_handle = get_document_handle(document)
+        assert self._doc_handle is not None
+        # calling the output_schema method to validate the input arguments
+        self.output_schema(separators=separators, metadata=metadata, limit=limit, overlap=overlap)
+        self._separators = _parse_separators(separators)
+        self._metadata_fields = _parse_metadata(metadata)
+        if self._doc_handle.bs_doc is not None:
+            title = self._doc_handle.bs_doc.title
+            if title is None:
+                self._doc_title = ''
+            else:
+                self._doc_title = ftfy.fix_text(title.get_text().strip())
+        else:
+            self._doc_title = ''
+        self._limit = 0 if limit is None else limit
+        self._skip_tags = html_skip_tags
+        self._overlap = 0 if overlap is None else overlap
+        self._tiktoken_encoding = tiktoken_encoding
+        self._tiktoken_target_model = tiktoken_target_model
+
+        # set up processing pipeline
+        if self._doc_handle.format == DocumentType.DocumentFormat.HTML:
+            assert self._doc_handle.bs_doc is not None
+            self._sections = self._html_sections()
+        elif self._doc_handle.format == DocumentType.DocumentFormat.MD:
+            assert self._doc_handle.md_ast is not None
+            self._sections = self._markdown_sections()
+        elif self._doc_handle.format == DocumentType.DocumentFormat.PDF:
+            assert self._doc_handle.pdf_doc is not None
+            self._sections = self._pdf_sections()
+        else:
+            assert False, f'unknown document format: {self._doc_handle.format}'
+
+        if Separator.SENTENCE in self._separators:
+            self._sections = self._sentence_sections(self._sections)
+        if Separator.TOKEN_LIMIT in self._separators:
+            self._sections = self._token_chunks(self._sections)
+        if Separator.CHAR_LIMIT in self._separators:
+            self._sections = self._char_chunks(self._sections)
+
+    @classmethod
+    def input_schema(cls) -> Dict[str, ColumnType]:
+        return {
+            'document': DocumentType(nullable=False),
+            'separators': StringType(nullable=False),
+            'metadata': StringType(nullable=True),
+            'limit': IntType(nullable=True),
+            'overlap': IntType(nullable=True),
+            'skip_tags': StringType(nullable=True),
+            'tiktoken_encoding': StringType(nullable=True),
+            'tiktoken_target_model': StringType(nullable=True),
+        }
+
+    @classmethod
+    def output_schema(cls, *args: Any, **kwargs: Any) -> Tuple[Dict[str, ColumnType], List[str]]:
+        schema = {'text': StringType()}
+        md_fields = _parse_metadata(kwargs['metadata']) if 'metadata' in kwargs else []
+
+        for md_field in md_fields:
+            schema[md_field.name.lower()] = cls.METADATA_COLUMN_TYPES[md_field]
+
+        assert 'separators' in kwargs
+        separators = _parse_separators(kwargs['separators'])
+
+        limit = kwargs.get('limit')
+        overlap = kwargs.get('overlap')
+
+        if limit is not None or overlap is not None:
+            if Separator.TOKEN_LIMIT not in separators and Separator.CHAR_LIMIT not in separators:
+                raise Error('limit/overlap requires the "token_limit" or "char_limit" separator')
+            if limit is not None and limit <= 0:
+                raise Error('"limit" must be an integer > 0')
+            if overlap is not None and overlap < 0:
+                raise Error('"overlap" must be an integer >= 0')
+        if Separator.TOKEN_LIMIT in separators or Separator.CHAR_LIMIT in separators:
+            if Separator.TOKEN_LIMIT in separators and Separator.CHAR_LIMIT in separators:
+                raise Error('Cannot specify both "token_limit" and "char_limit" separators')
+            if kwargs.get('limit') is None:
+                raise Error('limit is required with "token_limit"/"char_limit" separators')
+
+        # check dependencies at the end
+        if Separator.SENTENCE in separators:
+            Env.get().require_package('spacy')
+        if Separator.TOKEN_LIMIT in separators:
+            Env.get().require_package('tiktoken')
+
+        return schema, []
+
+    def __next__(self) -> Dict[str, Any]:
+        while True:
+            section = next(self._sections)
+            if section.text is None:
+                continue
+            result = {'text': section.text}
+            for md_field in self._metadata_fields:
+                if md_field == ChunkMetadata.TITLE:
+                    result[md_field.name.lower()] = self._doc_title
+                elif md_field == ChunkMetadata.HEADING:
+                    result[md_field.name.lower()] = section.metadata.heading
+                elif md_field == ChunkMetadata.SOURCELINE:
+                    result[md_field.name.lower()] = section.metadata.sourceline
+                elif md_field == ChunkMetadata.PAGE:
+                    result[md_field.name.lower()] = section.metadata.page
+                elif md_field == ChunkMetadata.BOUNDING_BOX:
+                    result[md_field.name.lower()] = section.metadata.bounding_box
+            return result
+
+    def _html_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections reflecting the html-specific separators"""
+        import bs4
+        emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
+        emit_on_heading = Separator.HEADING in self._separators or emit_on_paragraph
+        # current state
+        accumulated_text = []  # currently accumulated text
+        # accumulate pieces then join before emit to avoid quadratic complexity of string concatenation
+
+        headings: Dict[int, str] = {}   # current state of observed headings (level -> text)
+        sourceline = 0  # most recently seen sourceline
+
+        def update_metadata(el: bs4.Tag) -> None:
+            # update current state
+            nonlocal headings, sourceline
+            sourceline = el.sourceline
+            if el.name in _HTML_HEADINGS:
+                level = int(el.name[1])
+                # remove the previously seen lower levels
+                lower_levels = [l for l in headings if l > level]
+                for l in lower_levels:
+                    del headings[l]
+                headings[level] = el.get_text().strip()
+
+        def emit() -> None:
+            nonlocal accumulated_text, headings, sourceline
+            if len(accumulated_text) > 0:
+                md = DocumentSectionMetadata(sourceline=sourceline, heading=headings.copy())
+                full_text = ' '.join(accumulated_text)
+                full_text = ftfy.fix_text(full_text)
+                yield DocumentSection(text=full_text, metadata=md)
+                accumulated_text = []
+
+        def process_element(el: bs4.PageElement) -> Iterator[DocumentSection]:
+            # process the element and emit sections as necessary
+            nonlocal accumulated_text, headings, sourceline, emit_on_heading, emit_on_paragraph
+            if el.name in self._skip_tags:
+                return
+
+            if isinstance(el, bs4.NavigableString):
+                # accumulate text until we see a tag we care about
+                text = el.get_text().strip()
+                if len(text) > 0:
+                    accumulated_text.append(text)
+                return
+
+            if el.name in _HTML_HEADINGS:
+                if emit_on_heading:
+                    yield from emit()
+                update_metadata(el)
+            elif el.name == 'p':
+                if emit_on_paragraph:
+                    yield from emit()
+                update_metadata(el)
+            for child in el.children:
+                yield from process_element(child)
+
+        yield from process_element(self._doc_handle.bs_doc)
+        yield from emit()
+
+    def _markdown_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections reflecting the html-specific separators"""
+        assert self._doc_handle.md_ast is not None
+        emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
+        emit_on_heading = Separator.HEADING in self._separators or emit_on_paragraph
+        # current state
+        accumulated_text = []  # currently accumulated text
+        # accumulate pieces then join before emit to avoid quadratic complexity of string concatenation
+        headings: Dict[int, str] = {}   # current state of observed headings (level -> text)
+
+        def update_headings(heading: Dict) -> None:
+            # update current state
+            nonlocal headings
+            assert 'type' in heading and heading['type'] == 'heading'
+            level = heading['attrs']['level']
+            text = heading['children'][0]['raw'].strip()
+            # remove the previously seen lower levels
+            lower_levels = [l for l in headings.keys() if l > level]
+            for l in lower_levels:
+                del headings[l]
+            headings[level] = text
+
+        def emit() -> None:
+            nonlocal accumulated_text, headings
+            if len(accumulated_text) > 0:
+                metadata = DocumentSectionMetadata(sourceline=0, heading=headings.copy())
+                yield DocumentSection(text=ftfy.fix_text(' '.join(accumulated_text)), metadata=metadata)
+                accumulated_text = []
+
+        def process_element(el: Dict) -> Iterator[DocumentSection]:
+            # process the element and emit sections as necessary
+            nonlocal accumulated_text, headings, emit_on_heading, emit_on_paragraph
+            assert 'type' in el
+
+            if el['type'] == 'text':
+                # accumulate text until we see a separator element
+                text = el['raw'].strip()
+                if len(text) > 0:
+                    accumulated_text.append(text)
+                return
+
+            if el['type'] == 'heading':
+                if emit_on_heading:
+                    yield from emit()
+                update_headings(el)
+            elif el['type'] == 'paragraph':
+                if emit_on_paragraph:
+                    yield from emit()
+            if 'children' not in el:
+                return
+            for child in el['children']:
+                yield from process_element(child)
+
+        for el in self._doc_handle.md_ast:
+            yield from process_element(el)
+        yield from emit()
+
+    def _pdf_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections reflecting the pdf-specific separators"""
+        import fitz
+        doc: fitz.Document = self._doc_handle.pdf_doc
+        assert doc is not None
+
+        emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
+        emit_on_page = Separator.PAGE in self._separators or emit_on_paragraph
+
+        accumulated_text = []  # invariant: all elements are ftfy clean and non-empty
+
+        def _add_cleaned_text(raw_text: str) -> None:
+            fixed = ftfy.fix_text(raw_text)
+            if fixed:
+                accumulated_text.append(fixed)
+
+        def _emit_text() -> str:
+            full_text = ''.join(accumulated_text)
+            accumulated_text.clear()
+            return full_text
+
+        for page_number, page in enumerate(doc.pages()):
+            for block in page.get_text('blocks'):
+                # there is no concept of paragraph in pdf, block is the closest thing
+                # we can get (eg a paragraph in text may cut across pages)
+                # see pymupdf docs https://pymupdf.readthedocs.io/en/latest/app1.html
+                # other libraries like pdfminer also lack an explicit paragraph concept
+                x1, y1, x2, y2, text, _, _ = block
+                _add_cleaned_text(text)
+                if accumulated_text and emit_on_paragraph:
+                    bbox = {'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2}
+                    metadata = DocumentSectionMetadata(page=page_number, bounding_box=bbox)
+                    yield DocumentSection(text=_emit_text(), metadata=metadata)
+
+            if accumulated_text and emit_on_page and not emit_on_paragraph:
+                yield DocumentSection(text=_emit_text(),
+                                      metadata=DocumentSectionMetadata(page=page_number))
+                accumulated_text = []
+
+        if accumulated_text and not emit_on_page:
+            yield DocumentSection(text=_emit_text(), metadata=DocumentSectionMetadata())
+
+    def _sentence_sections(self, input_sections: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
+        """Split the input sections into sentences"""
+        for section in input_sections:
+            if section.text is not None:
+                doc = Env.get().spacy_nlp(section.text)
+                for sent in doc.sents:
+                    yield DocumentSection(text=sent.text, metadata=section.metadata)
+
+    def _token_chunks(self, input: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
+        import tiktoken
+        if self._tiktoken_target_model is not None:
+            encoding = tiktoken.encoding_for_model(self._tiktoken_target_model)
+        else:
+            encoding = tiktoken.get_encoding(self._tiktoken_encoding)
+        assert self._limit > 0 and self._overlap >= 0
+
+        for section in input:
+            if section.text is None:
+                continue
+            tokens = encoding.encode(section.text)
+            start_idx = 0
+            text = None
+            while start_idx < len(tokens):
+                end_idx = min(start_idx + self._limit, len(tokens))
+                while end_idx > start_idx:
+                    # find a cutoff point that doesn't cut in the middle of utf8 multi-byte sequences
+                    try:
+                        # check that the truncated data can be properly decoded
+                        text = encoding.decode(tokens[start_idx:end_idx], errors='strict')
+                        break
+                    except UnicodeDecodeError:
+                        # we split the token array at a point where the utf8 encoding is broken
+                        end_idx -= 1
+
+                assert end_idx > start_idx
+                assert text
+                yield DocumentSection(text=text, metadata=section.metadata)
+                start_idx = max(start_idx + 1, end_idx - self._overlap)  # ensure we make progress
+
+    def _char_chunks(self, input: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
+        for section in input:
+            if section.text is None:
+                continue
+            start_idx = 0
+            while start_idx < len(section.text):
+                end_idx = min(start_idx + self._limit, len(section.text))
+                text = section.text[start_idx:end_idx]
+                yield DocumentSection(text=text, metadata=section.metadata)
+                start_idx += self._limit - self._overlap
+
+    def close(self) -> None:
+        pass
+
+    def set_pos(self, pos: int) -> None:
+        pass

pixeltable/iterators/video.py

@@ -0,0 +1,88 @@
+import logging
+import math
+from pathlib import Path
+from typing import Dict, Any, List, Tuple
+
+import PIL.Image
+import cv2
+
+from pixeltable import exprs
+from pixeltable.exceptions import Error
+from pixeltable.type_system import ColumnType, VideoType, ImageType, IntType, FloatType
+from .base import ComponentIterator
+
+_logger = logging.getLogger('pixeltable')
+
+class FrameIterator(ComponentIterator):
+    def __init__(self, video: str, *, fps: float = 0.0):
+        video_path = Path(video)
+        assert video_path.exists() and video_path.is_file()
+        self.video_path = video_path
+        self.fps = fps
+        self.video_reader = cv2.VideoCapture(str(video_path))
+        if not self.video_reader.isOpened():
+            raise Error(f'Failed to open video: {video}')
+        video_fps = int(self.video_reader.get(cv2.CAP_PROP_FPS))
+        if fps > video_fps:
+            raise Error(f'Video {video}: requested fps ({fps}) exceeds that of the video ({video_fps})')
+        self.frame_freq = int(video_fps / fps) if fps > 0 else 1
+        num_video_frames = int(self.video_reader.get(cv2.CAP_PROP_FRAME_COUNT))
+        if num_video_frames == 0:
+            raise Error(f'Video {video}: failed to get number of frames')
+        # ceil: round up to ensure we count frame 0
+        self.num_frames = math.ceil(num_video_frames / self.frame_freq) if fps > 0 else num_video_frames
+        _logger.debug(f'FrameIterator: path={self.video_path} fps={self.fps}')
+
+        self.next_frame_idx = 0
+
+    @classmethod
+    def input_schema(cls) -> Dict[str, ColumnType]:
+        return {
+            'video': VideoType(nullable=False),
+            'fps': FloatType()
+        }
+
+    @classmethod
+    def output_schema(cls, *args: Any, **kwargs: Any) -> Tuple[Dict[str, ColumnType], List[str]]:
+        return {
+            'frame_idx': IntType(),
+            'pos_msec': FloatType(),
+            'pos_frame': FloatType(),
+            'frame': ImageType(),
+        }, ['frame']
+
+    def __next__(self) -> Dict[str, Any]:
+        while True:
+            pos_msec = self.video_reader.get(cv2.CAP_PROP_POS_MSEC)
+            pos_frame = self.video_reader.get(cv2.CAP_PROP_POS_FRAMES)
+            status, img = self.video_reader.read()
+            if not status:
+                _logger.debug(f'releasing video reader for {self.video_path}')
+                self.video_reader.release()
+                self.video_reader = None
+                raise StopIteration
+            if pos_frame % self.frame_freq == 0:
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+                result = {
+                    'frame_idx': self.next_frame_idx,
+                    'pos_msec': pos_msec,
+                    'pos_frame': pos_frame,
+                    'frame': PIL.Image.fromarray(img),
+                }
+                self.next_frame_idx += 1
+                # frame_freq > 1: jumping to the target frame here with video_reader.set() is far slower than just
+                # skipping the unwanted frames
+                return result
+
+    def close(self) -> None:
+        if self.video_reader is not None:
+            self.video_reader.release()
+            self.video_reader = None
+
+    def set_pos(self, pos: int) -> None:
+        """Seek to frame idx"""
+        if pos == self.next_frame_idx:
+            return
+        _logger.debug(f'seeking to frame {pos}')
+        self.video_reader.set(cv2.CAP_PROP_POS_FRAMES, pos * self.frame_freq)
+        self.next_frame_idx = pos

pixeltable/metadata/__init__.py

@@ -0,0 +1,58 @@
+import dataclasses
+import importlib
+import os
+import pkgutil
+from typing import Callable, Dict
+
+import sqlalchemy as sql
+import sqlalchemy.orm as orm
+
+from .schema import SystemInfo, SystemInfoMd
+
+# current version of the metadata; this is incremented whenever the metadata schema changes
+VERSION = 14
+
+
+def create_system_info(engine: sql.engine.Engine) -> None:
+    """Create the systemmetadata record"""
+    system_md = SystemInfoMd(schema_version=VERSION)
+    record = SystemInfo(md=dataclasses.asdict(system_md))
+    with orm.Session(engine, future=True) as session:
+        session.add(record)
+        session.flush()
+        session.commit()
+
+# conversion functions for upgrading the metadata schema from one version to the following
+# key: old schema version
+converter_cbs: Dict[int, Callable[[sql.engine.Engine], None]] = {}
+
+def register_converter(version: int, cb: Callable[[sql.engine.Engine], None]) -> None:
+    global converter_cbs
+    converter_cbs[version] = cb
+
+def noop_converter(engine: sql.engine.Engine) -> None:
+    # Converter to use when incrementing the schema version, but without any functional changes
+    pass
+
+# load all converter modules
+for _, modname, _ in pkgutil.iter_modules([os.path.dirname(__file__) + '/converters']):
+    importlib.import_module('pixeltable.metadata.converters.' + modname)
+
+def upgrade_md(engine: sql.engine.Engine) -> None:
+    """Upgrade the metadata schema to the current version"""
+    with orm.Session(engine) as session:
+        system_info = session.query(SystemInfo).one().md
+        md_version = system_info['schema_version']
+        if md_version == VERSION:
+            return
+        while md_version < VERSION:
+            if md_version not in converter_cbs:
+                raise RuntimeError(f'No metadata converter for version {md_version}')
+            print(f'Converting metadata from version {md_version} to {md_version + 1}')
+            converter_cbs[md_version](engine)
+            md_version += 1
+        # update system info
+        conn = session.connection()
+        system_info_md = SystemInfoMd(schema_version=VERSION)
+        conn.execute(SystemInfo.__table__.update().values(md=dataclasses.asdict(system_info_md)))
+        session.commit()

pixeltable/metadata/converters/convert_10.py

@@ -0,0 +1,18 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata.schema import Table, TableSchemaVersion
+from pixeltable.metadata import register_converter
+
+
+def convert_10(engine: sql.engine.Engine) -> None:
+    default_table_attrs = {"comment": None, "num_retained_versions": 10}
+    with engine.begin() as conn:
+        # Because `parameters` wasn't actually used for anything,
+        # we can simply delete it without any data loss.
+        conn.execute(sql.update(Table).values(md=Table.md - 'parameters'))
+        # Add `table_attrs` to all instances of tableschemaversions.md.
+        conn.execute(sql.update(TableSchemaVersion).values(md=TableSchemaVersion.md.concat(default_table_attrs)))
+    return
+
+
+register_converter(10, convert_10)

pixeltable/metadata/converters/convert_12.py

@@ -0,0 +1,3 @@
+from pixeltable.metadata import register_converter, noop_converter
+
+register_converter(12, noop_converter)

pixeltable/metadata/converters/convert_13.py

@@ -0,0 +1,41 @@
+import logging
+from typing import Any
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.schema import Table
+
+_logger = logging.getLogger('pixeltable')
+
+
+def convert_13(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        for row in conn.execute(sql.select(Table)):
+            id = row[0]
+            md = row[2]
+            updated_md = _update_md(md)
+            if updated_md != md:
+                _logger.info(f'Updating schema for table: {id}')
+                conn.execute(sql.update(Table).where(Table.id == id).values(md=updated_md))
+
+
+# Traverse the schema dictionary and replace instances of `ExplicitBatchedFunction` with
+# `CallableFunction`. DB versions prior to 14 can't contain serialized batched functions,
+# so this is all we need to do.
+def _update_md(md: Any) -> Any:
+    if isinstance(md, dict):
+        updated_md = {}
+        for k, v in md.items():
+            if k == '_classpath' and v == 'pixeltable.func.batched_function.ExplicitBatchedFunction':
+                updated_md[k] = 'pixeltable.func.callable_function.CallableFunction'
+            else:
+                updated_md[k] = _update_md(v)
+        return updated_md
+    elif isinstance(md, list):
+        return [_update_md(v) for v in md]
+    else:
+        return md
+
+
+register_converter(13, convert_13)