pixeltable 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

pixeltable/iterators/document.py

@@ -1,23 +1,25 @@
-from typing import Dict, Any, List, Tuple, Generator, Optional, Iterable
-import logging
 import dataclasses
 import enum
+import logging
+from typing import Dict, Any, List, Tuple, Optional, Iterable, Iterator
 
-from .base import ComponentIterator
+import ftfy
 
-from pixeltable.type_system import ColumnType, DocumentType, StringType, IntType, JsonType
-from pixeltable.exceptions import Error
 from pixeltable.env import Env
+from pixeltable.exceptions import Error
+from pixeltable.type_system import ColumnType, DocumentType, StringType, IntType, JsonType
 from pixeltable.utils.documents import get_document_handle
-
+from .base import ComponentIterator
 
 _logger = logging.getLogger('pixeltable')
 
 
 class ChunkMetadata(enum.Enum):
     TITLE = 1
-    HEADINGS = 2
+    HEADING = 2
     SOURCELINE = 3
+    PAGE = 4
+    BOUNDING_BOX = 5
 
 
 class Separator(enum.Enum):
@@ -26,52 +28,115 @@ class Separator(enum.Enum):
     SENTENCE = 3
     TOKEN_LIMIT = 4
     CHAR_LIMIT = 5
+    PAGE = 6
 
 
 @dataclasses.dataclass
-class DocumentSectionMd:
+class DocumentSectionMetadata:
     """Metadata for a subsection of a document (ie, a structural element like a heading or paragraph)"""
-    source_line: int
-
+    # html and markdown metadata
+    sourceline: Optional[int] = None
     # the stack of headings up to the most recently observed one;
     # eg, if the most recent one was an h2, 'headings' would contain keys 1 and 2, but nothing below that
-    headings: Dict[int, str]
+    heading: Optional[Dict[int, str]] = None
+
+    # pdf-specific metadata
+    page: Optional[int] = None
+    # bounding box as an {x1, y1, x2, y2} dictionary
+    bounding_box: Optional[Dict[str, float]] = None
 
 
 @dataclasses.dataclass
 class DocumentSection:
     """A single document chunk, according to some of the splitting criteria"""
     text: Optional[str]
-    md: Optional[DocumentSectionMd]
+    metadata: Optional[DocumentSectionMetadata]
+
+
+def _parse_separators(separators: str) -> List[Separator]:
+    ret = []
+    for s in separators.split(','):
+        clean_s = s.strip().upper()
+        if not clean_s:
+            continue
+        if clean_s not in Separator.__members__:
+            raise Error(
+                f'Invalid separator: `{s.strip()}`. Valid separators are: {", ".join(Separator.__members__).lower()}'
+            )
+        ret.append(Separator[clean_s])
+    return ret
+
+
+def _parse_metadata(metadata: str) -> List[ChunkMetadata]:
+    ret = []
+    for m in metadata.split(','):
+        clean_m = m.strip().upper()
+        if not clean_m:
+            continue
+        if clean_m not in ChunkMetadata.__members__:
+            raise Error(
+                f'Invalid metadata: `{m.strip()}`. Valid metadata are: {", ".join(ChunkMetadata.__members__).lower()}'
+            )
+        ret.append(ChunkMetadata[clean_m])
+    return ret
+
+
+_HTML_HEADINGS = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
 
 
 class DocumentSplitter(ComponentIterator):
-    """"Iterator over pieces of a document"""
-    MD_COLUMN_TYPES = {
-        ChunkMetadata.TITLE: StringType(),
-        ChunkMetadata.HEADINGS: JsonType(),
-        ChunkMetadata.SOURCELINE: IntType()
+    """Iterator over chunks of a document. The document is chunked according to the specified `separators`.
+
+    The iterator yields a `text` field containing the text of the chunk, and it may also
+    include additional metadata fields if specified in the `metadata` parameter, as explained below.
+
+    Chunked text will be cleaned with `ftfy.fix_text` to fix up common problems with unicode sequences.
+    """
+    METADATA_COLUMN_TYPES = {
+        ChunkMetadata.TITLE: StringType(nullable=True),
+        ChunkMetadata.HEADING: JsonType(nullable=True),
+        ChunkMetadata.SOURCELINE: IntType(nullable=True),
+        ChunkMetadata.PAGE: IntType(nullable=True),
+        ChunkMetadata.BOUNDING_BOX: JsonType(nullable=True),
     }
 
     def __init__(
-            self, document: str, *, separators: str, limit: int = 0, overlap: int = 0, metadata: str = '',
-            html_skip_tags: List[str] = None, tiktoken_encoding: Optional[str] = 'cl100k_base',
+            self, document: str, *, separators: str, limit: Optional[int] = None, overlap: Optional[int] = None,
+            metadata: str = '',
+            html_skip_tags: Optional[list[str]] = None, tiktoken_encoding: Optional[str] = 'cl100k_base',
             tiktoken_target_model: Optional[str] = None
     ):
-        import bs4
+        """Init method for `DocumentSplitter` class.
+
+        Args:
+            separators: separators to use to chunk the document. Options are:
+                 `'heading'`, `'paragraph'`, `'sentence'`, `'token_limit'`, `'char_limit'`, `'page'`.
+                 This may be a comma-separated string, e.g., `'heading,token_limit'`.
+            limit: the maximum number of tokens or characters in each chunk, if `'token_limit'`
+                 or `'char_limit'` is specified.
+            metadata: additional metadata fields to include in the output. Options are:
+                 `'title'`, `'heading'` (HTML and Markdown), `'sourceline'` (HTML), `'page'` (PDF), `'bounding_box'`
+                 (PDF). The input may be a comma-separated string, e.g., `'title,heading,sourceline'`.
+        """
         if html_skip_tags is None:
             html_skip_tags = ['nav']
-        with open(document, 'r', encoding='utf8') as fh:
-            s = fh.read()
-            self._doc_handle = get_document_handle(s)
-            assert self._doc_handle is not None
-        self._separators = [Separator[s.upper()] for s in separators.split(',')]
-        self._md_fields = [ChunkMetadata[m.upper()] for m in metadata.split(',')] if len(metadata) > 0 else []
-        self._doc_title = \
-            self._doc_handle.bs_doc.title.get_text().strip() if self._doc_handle.bs_doc is not None else ''
-        self._limit = limit
+        self._doc_handle = get_document_handle(document)
+        assert self._doc_handle is not None
+        # calling the output_schema method to validate the input arguments
+        self.output_schema(separators=separators, metadata=metadata, limit=limit, overlap=overlap)
+        self._separators = _parse_separators(separators)
+        self._metadata_fields = _parse_metadata(metadata)
+        if self._doc_handle.bs_doc is not None:
+            title = self._doc_handle.bs_doc.title
+            if title is None:
+                self._doc_title = ''
+            else:
+                self._doc_title = ftfy.fix_text(title.get_text().strip())
+        else:
+            self._doc_title = ''
+        self._limit = 0 if limit is None else limit
         self._skip_tags = html_skip_tags
-        self._overlap = overlap
+        self._overlap = 0 if overlap is None else overlap
         self._tiktoken_encoding = tiktoken_encoding
         self._tiktoken_target_model = tiktoken_target_model
 
@@ -79,9 +144,15 @@ class DocumentSplitter(ComponentIterator):
         if self._doc_handle.format == DocumentType.DocumentFormat.HTML:
             assert self._doc_handle.bs_doc is not None
             self._sections = self._html_sections()
-        else:
+        elif self._doc_handle.format == DocumentType.DocumentFormat.MD:
             assert self._doc_handle.md_ast is not None
             self._sections = self._markdown_sections()
+        elif self._doc_handle.format == DocumentType.DocumentFormat.PDF:
+            assert self._doc_handle.pdf_doc is not None
+            self._sections = self._pdf_sections()
+        else:
+            assert False, f'unknown document format: {self._doc_handle.format}'
+
         if Separator.SENTENCE in self._separators:
             self._sections = self._sentence_sections(self._sections)
         if Separator.TOKEN_LIMIT in self._separators:
@@ -105,38 +176,36 @@ class DocumentSplitter(ComponentIterator):
     @classmethod
     def output_schema(cls, *args: Any, **kwargs: Any) -> Tuple[Dict[str, ColumnType], List[str]]:
         schema = {'text': StringType()}
-        if 'metadata' in kwargs and len(kwargs['metadata']) > 0:
-            md_fields = kwargs['metadata'].split(',')
-            for md_field in md_fields:
-                if not hasattr(ChunkMetadata, md_field.upper()):
-                    raise Error(f'Invalid metadata field {md_field}')
-                schema[md_field.lower()] = cls.MD_COLUMN_TYPES[ChunkMetadata[md_field.upper()]]
+        md_fields = _parse_metadata(kwargs['metadata']) if 'metadata' in kwargs else []
+
+        for md_field in md_fields:
+            schema[md_field.name.lower()] = cls.METADATA_COLUMN_TYPES[md_field]
 
         assert 'separators' in kwargs
-        separators = kwargs['separators'].split(',')
-        for separator in separators:
-            if not hasattr(Separator, separator.upper()):
-                raise Error(f'Invalid separator {separator}')
+        separators = _parse_separators(kwargs['separators'])
 
-        # check dependencies
-        if 'sentence' in separators:
-            Env.get().require_package('spacy')
-        if 'token_limit' in separators:
-            Env.get().require_package('tiktoken')
+        limit = kwargs.get('limit')
+        overlap = kwargs.get('overlap')
 
-        if 'limit' in kwargs or 'overlap' in kwargs:
-            if 'token_limit' not in separators and 'char_limit' not in separators:
+        if limit is not None or overlap is not None:
+            if Separator.TOKEN_LIMIT not in separators and Separator.CHAR_LIMIT not in separators:
                 raise Error('limit/overlap requires the "token_limit" or "char_limit" separator')
-            if 'limit' in kwargs and int(kwargs['limit']) <= 0:
+            if limit is not None and limit <= 0:
                 raise Error('"limit" must be an integer > 0')
-            if 'overlap' in kwargs and int(kwargs['overlap']) < 0:
+            if overlap is not None and overlap < 0:
                 raise Error('"overlap" must be an integer >= 0')
-        if 'token_limit' in separators or 'char_limit' in separators:
-            if 'token_limit' in separators and 'char_limit' in separators:
+        if Separator.TOKEN_LIMIT in separators or Separator.CHAR_LIMIT in separators:
+            if Separator.TOKEN_LIMIT in separators and Separator.CHAR_LIMIT in separators:
                 raise Error('Cannot specify both "token_limit" and "char_limit" separators')
-            if 'limit' not in kwargs:
+            if kwargs.get('limit') is None:
                 raise Error('limit is required with "token_limit"/"char_limit" separators')
 
+        # check dependencies at the end
+        if Separator.SENTENCE in separators:
+            Env.get().require_package('spacy')
+        if Separator.TOKEN_LIMIT in separators:
+            Env.get().require_package('tiktoken')
+
         return schema, []
 
     def __next__(self) -> Dict[str, Any]:
@@ -145,47 +214,55 @@ class DocumentSplitter(ComponentIterator):
             if section.text is None:
                 continue
             result = {'text': section.text}
-            for md_field in self._md_fields:
+            for md_field in self._metadata_fields:
                 if md_field == ChunkMetadata.TITLE:
                     result[md_field.name.lower()] = self._doc_title
-                elif md_field == ChunkMetadata.HEADINGS:
-                    result[md_field.name.lower()] = section.md.headings
+                elif md_field == ChunkMetadata.HEADING:
+                    result[md_field.name.lower()] = section.metadata.heading
                 elif md_field == ChunkMetadata.SOURCELINE:
-                    result[md_field.name.lower()] = section.md.source_line
+                    result[md_field.name.lower()] = section.metadata.sourceline
+                elif md_field == ChunkMetadata.PAGE:
+                    result[md_field.name.lower()] = section.metadata.page
+                elif md_field == ChunkMetadata.BOUNDING_BOX:
+                    result[md_field.name.lower()] = section.metadata.bounding_box
             return result
 
-    def _html_sections(self) -> Generator[DocumentSection, None, None]:
+    def _html_sections(self) -> Iterator[DocumentSection]:
         """Create DocumentSections reflecting the html-specific separators"""
         import bs4
         emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
         emit_on_heading = Separator.HEADING in self._separators or emit_on_paragraph
         # current state
-        text_section = ''  # currently accumulated text
+        accumulated_text = []  # currently accumulated text
+        # accumulate pieces then join before emit to avoid quadratic complexity of string concatenation
+
         headings: Dict[int, str] = {}   # current state of observed headings (level -> text)
         sourceline = 0  # most recently seen sourceline
 
-        def update_md(el: bs4.Tag) -> None:
+        def update_metadata(el: bs4.Tag) -> None:
             # update current state
             nonlocal headings, sourceline
             sourceline = el.sourceline
-            if el.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+            if el.name in _HTML_HEADINGS:
                 level = int(el.name[1])
                 # remove the previously seen lower levels
-                lower_levels = [l for l in headings.keys() if l > level]
+                lower_levels = [l for l in headings if l > level]
                 for l in lower_levels:
                     del headings[l]
                 headings[level] = el.get_text().strip()
 
         def emit() -> None:
-            nonlocal text_section, headings, sourceline
-            if len(text_section) > 0:
-                md = DocumentSectionMd(sourceline, headings.copy())
-                yield DocumentSection(text=text_section, md=md)
-                text_section = ''
-
-        def process_element(el: bs4.PageElement) -> Generator[DocumentSection, None, None]:
+            nonlocal accumulated_text, headings, sourceline
+            if len(accumulated_text) > 0:
+                md = DocumentSectionMetadata(sourceline=sourceline, heading=headings.copy())
+                full_text = ' '.join(accumulated_text)
+                full_text = ftfy.fix_text(full_text)
+                yield DocumentSection(text=full_text, metadata=md)
+                accumulated_text = []
+
+        def process_element(el: bs4.PageElement) -> Iterator[DocumentSection]:
             # process the element and emit sections as necessary
-            nonlocal text_section, headings, sourceline, emit_on_heading, emit_on_paragraph
+            nonlocal accumulated_text, headings, sourceline, emit_on_heading, emit_on_paragraph
             if el.name in self._skip_tags:
                 return
 
@@ -193,30 +270,31 @@ class DocumentSplitter(ComponentIterator):
                 # accumulate text until we see a tag we care about
                 text = el.get_text().strip()
                 if len(text) > 0:
-                    text_section += ' ' + text
+                    accumulated_text.append(text)
                 return
 
-            if el.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+            if el.name in _HTML_HEADINGS:
                 if emit_on_heading:
                     yield from emit()
-                update_md(el)
+                update_metadata(el)
             elif el.name == 'p':
                 if emit_on_paragraph:
                     yield from emit()
-                update_md(el)
+                update_metadata(el)
             for child in el.children:
                 yield from process_element(child)
 
         yield from process_element(self._doc_handle.bs_doc)
         yield from emit()
 
-    def _markdown_sections(self) -> Generator[DocumentSection, None, None]:
+    def _markdown_sections(self) -> Iterator[DocumentSection]:
         """Create DocumentSections reflecting the html-specific separators"""
         assert self._doc_handle.md_ast is not None
         emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
         emit_on_heading = Separator.HEADING in self._separators or emit_on_paragraph
         # current state
-        text_section = ''  # currently accumulated text
+        accumulated_text = []  # currently accumulated text
+        # accumulate pieces then join before emit to avoid quadratic complexity of string concatenation
         headings: Dict[int, str] = {}   # current state of observed headings (level -> text)
 
         def update_headings(heading: Dict) -> None:
@@ -232,22 +310,22 @@ class DocumentSplitter(ComponentIterator):
             headings[level] = text
 
         def emit() -> None:
-            nonlocal text_section, headings
-            if len(text_section) > 0:
-                md = DocumentSectionMd(0, headings.copy())
-                yield DocumentSection(text=text_section, md=md)
-                text_section = ''
+            nonlocal accumulated_text, headings
+            if len(accumulated_text) > 0:
+                metadata = DocumentSectionMetadata(sourceline=0, heading=headings.copy())
+                yield DocumentSection(text=ftfy.fix_text(' '.join(accumulated_text)), metadata=metadata)
+                accumulated_text = []
 
-        def process_element(el: Dict) -> Generator[DocumentSection, None, None]:
+        def process_element(el: Dict) -> Iterator[DocumentSection]:
             # process the element and emit sections as necessary
-            nonlocal text_section, headings, emit_on_heading, emit_on_paragraph
+            nonlocal accumulated_text, headings, emit_on_heading, emit_on_paragraph
             assert 'type' in el
 
             if el['type'] == 'text':
                 # accumulate text until we see a separator element
                 text = el['raw'].strip()
                 if len(text) > 0:
-                    text_section += ' ' + text
+                    accumulated_text.append(text)
                 return
 
             if el['type'] == 'heading':
@@ -266,15 +344,57 @@ class DocumentSplitter(ComponentIterator):
             yield from process_element(el)
         yield from emit()
 
-    def _sentence_sections(self, input_sections: Iterable[DocumentSection]) -> Generator[DocumentSection, None, None]:
+    def _pdf_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections reflecting the pdf-specific separators"""
+        import fitz
+        doc: fitz.Document = self._doc_handle.pdf_doc
+        assert doc is not None
+
+        emit_on_paragraph = Separator.PARAGRAPH in self._separators or Separator.SENTENCE in self._separators
+        emit_on_page = Separator.PAGE in self._separators or emit_on_paragraph
+
+        accumulated_text = []  # invariant: all elements are ftfy clean and non-empty
+
+        def _add_cleaned_text(raw_text: str) -> None:
+            fixed = ftfy.fix_text(raw_text)
+            if fixed:
+                accumulated_text.append(fixed)
+
+        def _emit_text() -> str:
+            full_text = ''.join(accumulated_text)
+            accumulated_text.clear()
+            return full_text
+
+        for page_number, page in enumerate(doc.pages()):
+            for block in page.get_text('blocks'):
+                # there is no concept of paragraph in pdf, block is the closest thing
+                # we can get (eg a paragraph in text may cut across pages)
+                # see pymupdf docs https://pymupdf.readthedocs.io/en/latest/app1.html
+                # other libraries like pdfminer also lack an explicit paragraph concept
+                x1, y1, x2, y2, text, _, _ = block
+                _add_cleaned_text(text)
+                if accumulated_text and emit_on_paragraph:
+                    bbox = {'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2}
+                    metadata = DocumentSectionMetadata(page=page_number, bounding_box=bbox)
+                    yield DocumentSection(text=_emit_text(), metadata=metadata)
+
+            if accumulated_text and emit_on_page and not emit_on_paragraph:
+                yield DocumentSection(text=_emit_text(),
+                                      metadata=DocumentSectionMetadata(page=page_number))
+                accumulated_text = []
+
+        if accumulated_text and not emit_on_page:
+            yield DocumentSection(text=_emit_text(), metadata=DocumentSectionMetadata())
+
+    def _sentence_sections(self, input_sections: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
         """Split the input sections into sentences"""
         for section in input_sections:
             if section.text is not None:
                 doc = Env.get().spacy_nlp(section.text)
                 for sent in doc.sents:
-                    yield DocumentSection(text=sent.text, md=section.md)
+                    yield DocumentSection(text=sent.text, metadata=section.metadata)
 
-    def _token_chunks(self, input: Iterable[DocumentSection]) -> Generator[DocumentSection, None, None]:
+    def _token_chunks(self, input: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
         import tiktoken
         if self._tiktoken_target_model is not None:
             encoding = tiktoken.encoding_for_model(self._tiktoken_target_model)
@@ -287,13 +407,25 @@ class DocumentSplitter(ComponentIterator):
                 continue
             tokens = encoding.encode(section.text)
             start_idx = 0
+            text = None
             while start_idx < len(tokens):
                 end_idx = min(start_idx + self._limit, len(tokens))
-                text = encoding.decode(tokens[start_idx:end_idx])
-                yield DocumentSection(text=text, md=section.md)
-                start_idx += self._limit - self._overlap
-
-    def _char_chunks(self, input: Iterable[DocumentSection]) -> Generator[DocumentSection, None, None]:
+                while end_idx > start_idx:
+                    # find a cutoff point that doesn't cut in the middle of utf8 multi-byte sequences
+                    try:
+                        # check that the truncated data can be properly decoded
+                        text = encoding.decode(tokens[start_idx:end_idx], errors='strict')
+                        break
+                    except UnicodeDecodeError:
+                        # we split the token array at a point where the utf8 encoding is broken
+                        end_idx -= 1
+
+                assert end_idx > start_idx
+                assert text
+                yield DocumentSection(text=text, metadata=section.metadata)
+                start_idx = max(start_idx + 1, end_idx - self._overlap)  # ensure we make progress
+
+    def _char_chunks(self, input: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
         for section in input:
             if section.text is None:
                 continue
@@ -301,7 +433,7 @@ class DocumentSplitter(ComponentIterator):
             while start_idx < len(section.text):
                 end_idx = min(start_idx + self._limit, len(section.text))
                 text = section.text[start_idx:end_idx]
-                yield DocumentSection(text=text, md=section.md)
+                yield DocumentSection(text=text, metadata=section.metadata)
                 start_idx += self._limit - self._overlap
 
     def close(self) -> None:

pixeltable/iterators/video.py

@@ -1,21 +1,28 @@
-from typing import Dict, Any, List, Tuple
-from pathlib import Path
-import math
 import logging
+import math
+from pathlib import Path
+from typing import Dict, Any, List, Tuple
 
-import cv2
 import PIL.Image
+import cv2
 
-from .base import ComponentIterator
-
-from pixeltable.type_system import ColumnType, VideoType, ImageType, IntType, FloatType
 from pixeltable.exceptions import Error
-
+from pixeltable.type_system import ColumnType, VideoType, ImageType, IntType, FloatType
+from .base import ComponentIterator
 
 _logger = logging.getLogger('pixeltable')
 
+
 class FrameIterator(ComponentIterator):
-    def __init__(self, video: str, fps: float = 0.0):
+    """Iterator over frames of a video.
+
+        Args:
+            video: URL or file of the video to use for frame extraction
+            fps: number of frames to extract per second of video. This may be a fractional value, such as 0.5.
+                If set to 0.0, then the native framerate of the video will be used (all frames will be extracted).
+                Default: 0.0
+    """
+    def __init__(self, video: str, *, fps: float = 0.0):
         video_path = Path(video)
         assert video_path.exists() and video_path.is_file()
         self.video_path = video_path

pixeltable/metadata/__init__.py

@@ -10,11 +10,11 @@ import sqlalchemy.orm as orm
 from .schema import SystemInfo, SystemInfoMd
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 12
+VERSION = 16
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:
-    """Create the systemmetadata record"""
+    """Create the system metadata record"""
     system_md = SystemInfoMd(schema_version=VERSION)
     record = SystemInfo(md=dataclasses.asdict(system_md))
     with orm.Session(engine, future=True) as session:
@@ -30,17 +30,21 @@ def register_converter(version: int, cb: Callable[[sql.engine.Engine], None]) ->
     global converter_cbs
     converter_cbs[version] = cb
 
+def noop_converter(engine: sql.engine.Engine) -> None:
+    # Converter to use when incrementing the schema version, but without any functional changes
+    pass
+
 # load all converter modules
 for _, modname, _ in pkgutil.iter_modules([os.path.dirname(__file__) + '/converters']):
     importlib.import_module('pixeltable.metadata.converters.' + modname)
 
 def upgrade_md(engine: sql.engine.Engine) -> None:
     """Upgrade the metadata schema to the current version"""
-    with orm.Session(engine, future=True) as session:
+    with orm.Session(engine) as session:
         system_info = session.query(SystemInfo).one().md
         md_version = system_info['schema_version']
         if md_version == VERSION:
-                return
+            return
         while md_version < VERSION:
             if md_version not in converter_cbs:
                 raise RuntimeError(f'No metadata converter for version {md_version}')

pixeltable/metadata/converters/convert_12.py

@@ -0,0 +1,3 @@
+from pixeltable.metadata import register_converter, noop_converter
+
+register_converter(12, noop_converter)

pixeltable/metadata/converters/convert_13.py

@@ -0,0 +1,41 @@
+import logging
+from typing import Any
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.schema import Table
+
+_logger = logging.getLogger('pixeltable')
+
+
+def convert_13(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        for row in conn.execute(sql.select(Table)):
+            id = row[0]
+            md = row[2]
+            updated_md = _update_md(md)
+            if updated_md != md:
+                _logger.info(f'Updating schema for table: {id}')
+                conn.execute(sql.update(Table).where(Table.id == id).values(md=updated_md))
+
+
+# Traverse the schema dictionary and replace instances of `ExplicitBatchedFunction` with
+# `CallableFunction`. DB versions prior to 14 can't contain serialized batched functions,
+# so this is all we need to do.
+def _update_md(md: Any) -> Any:
+    if isinstance(md, dict):
+        updated_md = {}
+        for k, v in md.items():
+            if k == '_classpath' and v == 'pixeltable.func.batched_function.ExplicitBatchedFunction':
+                updated_md[k] = 'pixeltable.func.callable_function.CallableFunction'
+            else:
+                updated_md[k] = _update_md(v)
+        return updated_md
+    elif isinstance(md, list):
+        return [_update_md(v) for v in md]
+    else:
+        return md
+
+
+register_converter(13, convert_13)

pixeltable/metadata/converters/convert_14.py

@@ -0,0 +1,13 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata.schema import Table
+from pixeltable.metadata import register_converter
+
+
+def convert_14(engine: sql.engine.Engine) -> None:
+    default_remotes = {'remotes': []}
+    with engine.begin() as conn:
+        conn.execute(sql.update(Table).where(Table.md['remotes'] == None).values(md=Table.md.concat(default_remotes)))
+
+
+register_converter(14, convert_14)

pixeltable/metadata/converters/convert_15.py

@@ -0,0 +1,29 @@
+import uuid
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+def convert_15(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, column_md_updater=update_column_md, remote_md_updater=update_remote_md)
+
+
+def update_column_md(column_md: dict) -> None:
+    column_md['proxy_base'] = None
+
+
+def update_remote_md(remote_md: dict) -> None:
+    remote_md['class'] = f'{remote_md["module"]}.{remote_md["class"]}'
+    del remote_md['module']
+    if remote_md['class'] == 'pixeltable.datatransfer.remote.MockRemote':
+        remote_md['remote_md']['name'] = f'remote_{uuid.uuid4()}'
+    elif remote_md['class'] == 'pixeltable.datatransfer.label_studio.LabelStudioProject':
+        # 'post' is the media_import_method for legacy LabelStudioProject remotes
+        remote_md['remote_md']['media_import_method'] = 'post'
+    else:
+        assert False, remote_md['class']
+
+
+register_converter(15, convert_15)