pixeltable 0.3.14__py3-none-any.whl → 0.3.15__py3-none-any.whl

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.3.14'
-__version_tuple__ = (0, 3, 14)
+__version__ = '0.3.15'
+__version_tuple__ = (0, 3, 15)

pixeltable/catalog/table_version.py

@@ -324,6 +324,7 @@ class TableVersion:
     @classmethod
     def create_replica(cls, md: schema.FullTableMd) -> TableVersion:
         tbl_id = UUID(md.tbl_md.tbl_id)
+        _logger.info(f'Creating replica table version {tbl_id}:{md.version_md.version}.')
         view_md = md.tbl_md.view_md
         base_path = pxt.catalog.TableVersionPath.from_md(view_md.base_versions) if view_md is not None else None
         base = base_path.tbl_version if base_path is not None else None
@@ -331,6 +332,7 @@ class TableVersion:
             tbl_id, md.tbl_md, md.version_md.version, md.schema_version_md, [], base_path=base_path, base=base
         )
         tbl_version.store_tbl.create()
+        tbl_version.store_tbl.ensure_columns_exist(col for col in tbl_version.cols if col.is_stored)
         return tbl_version
 
     def drop(self) -> None:

pixeltable/catalog/table_version_path.py

@@ -98,13 +98,6 @@ class TableVersionPath:
             return None
         return self.base.find_tbl_version(id)
 
-    @property
-    def ancestor_paths(self) -> list[TableVersionPath]:
-        if self.base is None:
-            return [self]
-        else:
-            return [self, *self.base.ancestor_paths]
-
     def columns(self) -> list[Column]:
         """Return all user columns visible in this tbl version path, including columns from bases"""
         result = list(self.tbl_version.get().cols_by_name.values())

pixeltable/functions/gemini.py

@@ -5,10 +5,16 @@ first `pip install google-genai` and configure your Gemini credentials, as descr
 the [Working with Gemini](https://pixeltable.readme.io/docs/working-with-gemini) tutorial.
 """
 
+import asyncio
+import io
+import tempfile
+from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
+import PIL.Image
+
 import pixeltable as pxt
-from pixeltable import env
+from pixeltable import env, exceptions as excs, exprs
 
 if TYPE_CHECKING:
     from google import genai
@@ -27,23 +33,11 @@ def _genai_client() -> 'genai.client.Client':
 
 @pxt.udf(resource_pool='request-rate:gemini')
 async def generate_content(
-    contents: str,
-    *,
-    model: str,
-    candidate_count: Optional[int] = None,
-    stop_sequences: Optional[list[str]] = None,
-    max_output_tokens: Optional[int] = None,
-    temperature: Optional[float] = None,
-    top_p: Optional[float] = None,
-    top_k: Optional[int] = None,
-    response_mime_type: Optional[str] = None,
-    response_schema: Optional[dict] = None,
-    presence_penalty: Optional[float] = None,
-    frequency_penalty: Optional[float] = None,
+    contents: str, *, model: str, config: Optional[dict] = None, tools: Optional[list[dict]] = None
 ) -> dict:
     """
     Generate content from the specified model. For additional details, see:
-    <https://ai.google.dev/gemini-api/docs>
+    <https://ai.google.dev/gemini-api/docs/text-generation>
 
     Request throttling:
     Applies the rate limit set in the config (section `gemini`, key `rate_limit`). If no rate
@@ -56,38 +50,177 @@ async def generate_content(
     Args:
         contents: The input content to generate from.
         model: The name of the model to use.
-
-    For details on the other parameters, see: <https://ai.google.dev/gemini-api/docs>
+        config: Configuration for generation, corresponding to keyword arguments of
+            `genai.types.GenerateContentConfig`. For details on the parameters, see:
+            <https://googleapis.github.io/python-genai/genai.html#module-genai.types>
+        tools: Optional list of Pixeltable tools to use. It is also possible to specify tools manually via the
+            `config.tools` parameter, but at most one of `config.tools` or `tools` may be used.
 
     Returns:
         A dictionary containing the response and other metadata.
 
     Examples:
-        Add a computed column that applies the model `gemini-1.5-flash`
+        Add a computed column that applies the model `gemini-2.0-flash`
         to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
 
-        >>> tbl.add_computed_column(response=generate_content(tbl.prompt, model='gemini-1.5-flash'))
+        >>> tbl.add_computed_column(response=generate_content(tbl.prompt, model='gemini-2.0-flash'))
     """
     env.Env.get().require_package('google.genai')
     from google.genai import types
 
-    config = types.GenerateContentConfig(
-        candidate_count=candidate_count,
-        stop_sequences=stop_sequences,
-        max_output_tokens=max_output_tokens,
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        response_mime_type=response_mime_type,
-        response_schema=response_schema,
-        presence_penalty=presence_penalty,
-        frequency_penalty=frequency_penalty,
-    )
-
-    response = await _genai_client().aio.models.generate_content(model=model, contents=contents, config=config)
+    config_: types.GenerateContentConfig
+    if config is None and tools is None:
+        config_ = None
+    else:
+        if config is None:
+            config_ = types.GenerateContentConfig()
+        else:
+            config_ = types.GenerateContentConfig(**config)
+        if tools is not None:
+            gemini_tools = [__convert_pxt_tool(tool) for tool in tools]
+            config_.tools = [types.Tool(function_declarations=gemini_tools)]
+
+    response = await _genai_client().aio.models.generate_content(model=model, contents=contents, config=config_)
     return response.model_dump()
 
 
+def __convert_pxt_tool(pxt_tool: dict) -> dict:
+    return {
+        'name': pxt_tool['name'],
+        'description': pxt_tool['description'],
+        'parameters': {
+            'type': 'object',
+            'properties': pxt_tool['parameters']['properties'],
+            'required': pxt_tool['required'],
+        },
+    }
+
+
+def invoke_tools(tools: pxt.func.Tools, response: exprs.Expr) -> exprs.InlineDict:
+    """Converts an OpenAI response dict to Pixeltable tool invocation format and calls `tools._invoke()`."""
+    return tools._invoke(_gemini_response_to_pxt_tool_calls(response))
+
+
+@pxt.udf
+def _gemini_response_to_pxt_tool_calls(response: dict) -> Optional[dict]:
+    print(response)
+    pxt_tool_calls: dict[str, list[dict]] = {}
+    for part in response['candidates'][0]['content']['parts']:
+        tool_call = part.get('function_call')
+        if tool_call is not None:
+            tool_name = tool_call['name']
+            if tool_name not in pxt_tool_calls:
+                pxt_tool_calls[tool_name] = []
+            pxt_tool_calls[tool_name].append({'args': tool_call['args']})
+    if len(pxt_tool_calls) == 0:
+        return None
+    return pxt_tool_calls
+
+
 @generate_content.resource_pool
 def _(model: str) -> str:
     return f'request-rate:gemini:{model}'
+
+
+@pxt.udf(resource_pool='request-rate:imagen')
+async def generate_images(prompt: str, *, model: str, config: Optional[dict] = None) -> PIL.Image.Image:
+    """
+    Generates images based on a text description and configuration. For additional details, see:
+    <https://ai.google.dev/gemini-api/docs/image-generation>
+
+    __Requirements:__
+
+    - `pip install google-genai`
+
+    Args:
+        prompt: A text description of the images to generate.
+        model: The model to use.
+        config: Configuration for generation, corresponding to keyword arguments of
+            `genai.types.GenerateImagesConfig`. For details on the parameters, see:
+            <https://googleapis.github.io/python-genai/genai.html#module-genai.types>
+
+    Returns:
+        The generated image.
+
+    Examples:
+        Add a computed column that applies the model `imagen-3.0-generate-002`
+        to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
+
+        >>> tbl.add_computed_column(response=generate_images(tbl.prompt, model='imagen-3.0-generate-002'))
+    """
+    env.Env.get().require_package('google.genai')
+    from google.genai.types import GenerateImagesConfig
+
+    config_ = GenerateImagesConfig(**config) if config else None
+    response = await _genai_client().aio.models.generate_images(model=model, prompt=prompt, config=config_)
+    return response.generated_images[0].image._pil_image
+
+
+@generate_images.resource_pool
+def _(model: str) -> str:
+    return f'request-rate:imagen:{model}'
+
+
+@pxt.udf(resource_pool='request-rate:veo')
+async def generate_videos(
+    prompt: Optional[str] = None, image: Optional[PIL.Image.Image] = None, *, model: str, config: Optional[dict] = None
+) -> pxt.Video:
+    """
+    Generates videos based on a text description and configuration. For additional details, see:
+    <https://ai.google.dev/gemini-api/docs/video-generation>
+
+    __Requirements:__
+
+    - `pip install google-genai`
+
+    Args:
+        prompt: A text description of the videos to generate.
+        image: An optional image to use as the first frame of the video. At least one of `prompt` or `image` must be
+            provided. (It is ok to specify both.)
+        model: The model to use.
+        config: Configuration for generation, corresponding to keyword arguments of
+            `genai.types.GenerateVideosConfig`. For details on the parameters, see:
+            <https://googleapis.github.io/python-genai/genai.html#module-genai.types>
+
+    Returns:
+        The generated video.
+
+    Examples:
+        Add a computed column that applies the model `veo-2.0-generate-001`
+        to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
+
+        >>> tbl.add_computed_column(response=generate_videos(tbl.prompt, model='veo-2.0-generate-001'))
+    """
+    env.Env.get().require_package('google.genai')
+    from google.genai import types
+
+    if prompt is None and image is None:
+        raise excs.Error('At least one of `prompt` or `image` must be provided.')
+
+    image_: Optional[types.Image] = None
+    if image is not None:
+        with io.BytesIO() as buffer:
+            image.save(buffer, format='jpeg')
+            image_ = types.Image(image_bytes=buffer.getvalue(), mime_type='image/jpeg')
+
+    config_ = types.GenerateVideosConfig(**config) if config else None
+    operation = await _genai_client().aio.models.generate_videos(
+        model=model, prompt=prompt, image=image_, config=config_
+    )
+    while not operation.done:
+        await asyncio.sleep(3)
+        operation = await _genai_client().aio.operations.get(operation)
+
+    video = operation.response.generated_videos[0]
+
+    video_bytes = await _genai_client().aio.files.download(file=video.video)  # type: ignore[arg-type]
+    assert video_bytes is not None
+
+    _, output_filename = tempfile.mkstemp(suffix='.mp4', dir=str(env.Env.get().tmp_dir))
+    Path(output_filename).write_bytes(video_bytes)
+    return output_filename
+
+
+@generate_videos.resource_pool
+def _(model: str) -> str:
+    return f'request-rate:veo:{model}'

pixeltable/functions/math.py

@@ -100,6 +100,69 @@ def _(self: sql.ColumnElement, digits: Optional[sql.ColumnElement] = None) -> sq
         return sql.func.round(sql.cast(self, sql.Numeric), sql.cast(digits, sql.Integer))
 
 
+@pxt.udf(is_method=True)
+def pow(self: int, other: int) -> float:
+    """
+    Raise `self` to the power of `other`.
+
+    Equivalent to Python [`self ** other`](https://docs.python.org/3/library/functions.html#pow).
+    """
+    return self**other
+
+
+@pow.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return sql.func.pow(self, other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_and(self: int, other: int) -> int:
+    """
+    Bitwise AND of two integers.
+
+    Equivalent to Python
+    [`self & other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self & other
+
+
+@bitwise_and.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_and(other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_or(self: int, other: int) -> int:
+    """
+    Bitwise OR of two integers.
+
+    Equivalent to Python
+    [`self | other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self | other
+
+
+@bitwise_or.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_or(other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_xor(self: int, other: int) -> int:
+    """
+    Bitwise XOR of two integers.
+
+    Equivalent to Python
+    [`self ^ other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self ^ other
+
+
+@bitwise_xor.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_xor(other)
+
+
 __all__ = local_public_names(__name__)
 
 

pixeltable/io/parquet.py

@@ -112,11 +112,11 @@ def export_parquet(
                         length = len(val)
                     elif col_type.is_string_type():
                         length = len(val)
-                    elif col_type.is_video_type():
+                    elif col_type.is_video_type() or col_type.is_audio_type():
                         if data_row.file_paths is not None and data_row.file_paths[e.slot_idx] is not None:
                             val = data_row.file_paths[e.slot_idx]
                         else:
-                            raise excs.Error(f'unknown video type {type(val)}')
+                            raise excs.Error(f'unknown audio/video type {type(val)}')
                         length = len(val)
                     elif col_type.is_json_type():
                         val = json.dumps(val)

pixeltable/share/packager.py

@@ -17,6 +17,7 @@ import pixeltable as pxt
 from pixeltable import catalog, exceptions as excs, metadata
 from pixeltable.env import Env
 from pixeltable.metadata import schema
+from pixeltable.utils import sha256sum
 from pixeltable.utils.media_store import MediaStore
 
 _logger = logging.getLogger('pixeltable')
@@ -88,7 +89,7 @@ class TablePackager:
         assert any(tv.id == base.id for base in self.table._tbl_version_path.get_tbl_versions())
         sql_types = {col.name: col.type for col in tv.store_tbl.sa_tbl.columns}
         media_cols: set[str] = set()
-        for col in tv.cols_by_name.values():
+        for col in tv.cols:
             if col.is_stored and col.col_type.is_media_type():
                 media_cols.add(col.store_name())
 
@@ -182,7 +183,12 @@ class TablePackager:
             path = Path(urllib.parse.unquote(urllib.request.url2pathname(parsed_url.path)))
             if path not in self.media_files:
                 # Create a new entry in the `media_files` dict so that we can copy the file into the tarball later.
-                dest_name = f'{uuid.uuid4().hex}{path.suffix}'
+                # We name the media files in the archive by their SHA256 hash. This ensures that we can properly
+                # deduplicate and validate them later.
+                # If we get a collision, it's not a problem; it just means we have two identical files (which will
+                # be conveniently deduplicated in the bundle).
+                sha = sha256sum(path)
+                dest_name = f'{sha}{path.suffix}'
                 self.media_files[path] = dest_name
             return f'pxtmedia://{self.media_files[path]}'
         # For any type of URL other than a local file, just return the URL as-is.
@@ -276,11 +282,182 @@ class TableRestorer:
         tbl_id = uuid.UUID(tbl_md.tbl_md.tbl_id)
         parquet_dir = bundle_path / 'tables' / f'tbl_{tbl_id.hex}'
         parquet_table = pq.read_table(str(parquet_dir))
-
-        for batch in parquet_table.to_batches():
+        replica_version = tv.version
+
+        conn = Env.get().conn
+        store_sa_tbl = tv.store_tbl.sa_tbl
+        store_sa_tbl_name = tv.store_tbl._storage_name()
+
+        # Sometimes we are importing a table that has never been seen before. Other times, however, we are importing
+        # an existing replica table, and the table version and/or row selection differs from what was imported
+        # previously. Care must be taken to ensure that the new data is merged with existing data in a way that
+        # yields an internally consistent version history for each row.
+
+        # The overall strategy is this:
+        # 1. Import the parquet data into a temporary table;
+        # 2. "rectify" the v_max values in both the temporary table and the existing table (more on this below);
+        # 3. Delete any row instances from the temporary table that are already present in the existing table;
+        # 4. Copy the remaining rows from the temporary table into the existing table.
+
+        # Create a temporary table for the initial data load, containing columns for all columns present in the
+        # parquet table. The parquet columns have identical names to those in the store table, so we can use the
+        # store table schema to get their SQL types (which are not necessarily derivable from their Parquet types,
+        # e.g., pa.string() may hold either VARCHAR or serialized JSONB).
+        temp_cols: dict[str, sql.Column] = {}
+        for field in parquet_table.schema:
+            assert field.name in store_sa_tbl.columns
+            col_type = store_sa_tbl.columns[field.name].type
+            temp_cols[field.name] = sql.Column(field.name, col_type)
+        temp_sa_tbl_name = f'temp_{uuid.uuid4().hex}'
+        _logger.debug(f'Creating temporary table: {temp_sa_tbl_name}')
+        temp_md = sql.MetaData()
+        temp_sa_tbl = sql.Table(temp_sa_tbl_name, temp_md, *temp_cols.values(), prefixes=['TEMPORARY'])
+        temp_sa_tbl.create(conn)
+
+        # Populate the temporary table with data from the Parquet file.
+        _logger.debug(f'Loading {parquet_table.num_rows} row(s) into temporary table: {temp_sa_tbl_name}')
+        for batch in parquet_table.to_batches(max_chunksize=10_000):
             pydict = batch.to_pydict()
             rows = self.__from_pa_pydict(tv, pydict)
-            tv.store_tbl.load_rows(rows)
+            conn.execute(sql.insert(temp_sa_tbl), rows)
+
+        # Each row version is identified uniquely by its pk, a tuple (row_id, pos_0, pos_1, ..., pos_k, v_min).
+        # Conversely, v_max is not part of the primary key, but is simply a bookkeeping device.
+        # In an original table, v_max is always equal to the v_min of the succeeding row instance with the same
+        # row id, or MAX_VERSION if no such row instance exists. But in the replica, we need to be careful, since
+        # we might see only a subset of the original table's versions, and we might see them out of order.
+
+        # We'll adjust the v_max values according to the principle of "latest provable v_max":
+        # they will always correspond to the latest version for which we can prove the row instance was alive. This
+        # will enable us to maintain consistency of the v_max values if additional table versions are later imported,
+        # regardless of the order in which they are seen. It also means that replica tables (unlike original tables)
+        # may have gaps in their row version histories, but this is fine; the gaps simply correspond to table versions
+        # that have never been observed.
+
+        pk_predicates = [col == temp_cols[col.name] for col in tv.store_tbl.pk_columns()]
+        pk_clause = sql.and_(*pk_predicates)
+
+        # If the same pk exists in both the temporary table and the existing table, then the corresponding row data
+        # must be identical; the rows can differ only in their v_max value. As a sanity check, we go through the
+        # motion of verifying this; a failure implies data corruption in either the replica being imported or in a
+        # previously imported replica.
+
+        system_col_names = {col.name for col in tv.store_tbl.system_columns()}
+        media_col_names = {col.store_name() for col in tv.cols if col.col_type.is_media_type() and col.is_stored}
+        value_store_cols = [
+            store_sa_tbl.c[col_name]
+            for col_name in temp_cols
+            if col_name not in system_col_names and col_name not in media_col_names
+        ]
+        value_temp_cols = [
+            col
+            for col_name, col in temp_cols.items()
+            if col_name not in system_col_names and col_name not in media_col_names
+        ]
+        mismatch_predicates = [store_col != temp_col for store_col, temp_col in zip(value_store_cols, value_temp_cols)]
+        mismatch_clause = sql.or_(*mismatch_predicates)
+
+        # This query looks for rows that have matching primary keys (rowid + pos_k + v_min), but differ in at least
+        # one value column. Pseudo-SQL:
+        #
+        # SELECT store_tbl.col_0, ..., store_tbl.col_n, temp_tbl.col_0, ...,  temp_tbl.col_n
+        # FROM store_tbl, temp_tbl
+        # WHERE store_tbl.rowid = temp_tbl.rowid
+        #     AND store_tbl.pos_0 = temp_tbl.pos_0
+        #     AND ... AND store_tbl.pos_k = temp_tbl.pos_k
+        #     AND store_tbl.v_min = temp_tbl.v_min
+        #     AND (
+        #         store_tbl.col_0 != temp_tbl.col_0
+        #         OR store_tbl.col_1 != temp_tbl.col_1
+        #         OR ... OR store_tbl.col_n != temp_tbl.col_n
+        #     )
+        #
+        # The value column comparisons (store_tbl.col_0 != temp_tbl.col_0, etc.) will always be false for rows where
+        # either column is NULL; this is what we want, since it may indicate a column that is present in one version
+        # but not the other.
+        q = sql.select(*value_store_cols, *value_temp_cols).where(pk_clause).where(mismatch_clause)
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        if result.rowcount > 0:
+            _logger.debug(
+                f'Data corruption error between {temp_sa_tbl_name!r} and {store_sa_tbl_name!r}: '
+                f'{result.rowcount} inconsistent row(s).'
+            )
+            row = result.first()
+            _logger.debug('Example mismatch:')
+            _logger.debug(f'{store_sa_tbl_name}: {row[: len(value_store_cols)]}')
+            _logger.debug(f'{temp_sa_tbl_name}: {row[len(value_store_cols) :]}')
+            raise excs.Error(
+                'Data corruption error: the replica data are inconsistent with data retrieved from a previous replica.'
+            )
+        _logger.debug(f'Verified data integrity between {store_sa_tbl_name!r} and {temp_sa_tbl_name!r}.')
+
+        # Now rectify the v_max values in the temporary table.
+        # If a row instance has a concrete v_max value, then we know it's genuine: it's the unique and immutable
+        # version when the row was deleted. (This can only happen if later versions of the base table already
+        # existed at the time this replica was published.)
+        # But if a row instance has a v_max value of MAX_VERSION, then we don't know anything about its future.
+        # It might live indefinitely, or it might be deleted as early as version `n + 1`. Following the principle
+        # of "latest provable v_max", we simply set v_max equal to `n + 1`.
+        q = (
+            temp_sa_tbl.update()
+            .values(v_max=(replica_version + 1))
+            .where(temp_sa_tbl.c.v_max == schema.Table.MAX_VERSION)
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Rectified {result.rowcount} row(s) in {temp_sa_tbl_name!r}.')
+
+        # Now rectify the v_max values in the existing table. This is done by simply taking the later of the two v_max
+        # values (the existing one and the new one) for each row instance, following the "latest provable v_max"
+        # principle. Obviously we only need to do this for rows that exist in both tables (it's a simple join).
+        q = (
+            store_sa_tbl.update()
+            .values(v_max=sql.func.greatest(store_sa_tbl.c.v_max, temp_sa_tbl.c.v_max))
+            .where(pk_clause)
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Rectified {result.rowcount} row(s) in {store_sa_tbl_name!r}.')
+
+        # Now we need to update rows in the existing table that are also present in the temporary table. This is to
+        # account for the scenario where the temporary table has columns that are not present in the existing table.
+        # (We can't simply replace the rows with their versions in the temporary table, because the converse scenario
+        # might also occur; there may be columns in the existing table that are not present in the temporary table.)
+        value_update_clauses: dict[str, sql.ColumnElement] = {}
+        for temp_col in temp_cols.values():
+            if temp_col.name not in system_col_names:
+                store_col = store_sa_tbl.c[temp_col.name]
+                # Prefer the value from the existing table, substituting the value from the temporary table if it's
+                # NULL. This works in all cases (including media columns, where we prefer the existing media file).
+                clause = sql.case((store_col == None, temp_col), else_=store_col)
+                value_update_clauses[temp_col.name] = clause
+        if len(value_update_clauses) > 0:
+            q = store_sa_tbl.update().values(**value_update_clauses).where(pk_clause)
+            _logger.debug(q.compile())
+            result = conn.execute(q)
+            _logger.debug(
+                f'Merged values from {temp_sa_tbl_name!r} into {store_sa_tbl_name!r} for {result.rowcount} row(s).'
+            )
+
+        # Now drop any rows from the temporary table that are also present in the existing table.
+        # The v_max values have been rectified, data has been merged into NULL cells, and all other row values have
+        # been verified identical.
+        # TODO: Delete any media files that were orphaned by this operation (they're necessarily duplicates of media
+        #     files that are already present in the existing table).
+        q = temp_sa_tbl.delete().where(pk_clause)
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Deleted {result.rowcount} row(s) from {temp_sa_tbl_name!r}.')
+
+        # Finally, copy the remaining data (consisting entirely of new row instances) from the temporary table into
+        # the actual table.
+        q = store_sa_tbl.insert().from_select(
+            [store_sa_tbl.c[col_name] for col_name in temp_cols], sql.select(*temp_cols.values())
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Inserted {result.rowcount} row(s) from {temp_sa_tbl_name!r} into {store_sa_tbl_name!r}.')
 
     def __from_pa_pydict(self, tv: catalog.TableVersion, pydict: dict[str, Any]) -> list[dict[str, Any]]:
         # Data conversions from pyarrow to Pixeltable
@@ -289,7 +466,7 @@ class TableRestorer:
             assert col_name in tv.store_tbl.sa_tbl.columns
             sql_types[col_name] = tv.store_tbl.sa_tbl.columns[col_name].type
         media_col_ids: dict[str, int] = {}
-        for col in tv.cols_by_name.values():
+        for col in tv.cols:
             if col.is_stored and col.col_type.is_media_type():
                 media_col_ids[col.store_name()] = col.id
 

pixeltable/store.py

@@ -54,6 +54,9 @@ class StoreBase:
         self.base = tbl_version.base.get().store_tbl if tbl_version.base is not None else None
         self.create_sa_tbl()
 
+    def system_columns(self) -> list[sql.Column]:
+        return [*self._pk_cols, self.v_max_col]
+
     def pk_columns(self) -> list[sql.Column]:
         return self._pk_cols
 
@@ -215,6 +218,15 @@ class StoreBase:
         log_stmt(_logger, stmt)
         Env.get().conn.execute(stmt)
 
+    def ensure_columns_exist(self, cols: Iterable[catalog.Column]) -> None:
+        conn = Env.get().conn
+        sql_text = f'SELECT column_name FROM information_schema.columns WHERE table_name = {self._storage_name()!r}'
+        result = conn.execute(sql.text(sql_text))
+        existing_cols = {row[0] for row in result}
+        for col in cols:
+            if col.store_name() not in existing_cols:
+                self.add_column(col)
+
     def load_column(
         self, col: catalog.Column, exec_plan: ExecNode, value_expr_slot_idx: int, on_error: Literal['abort', 'ignore']
     ) -> int:

{pixeltable-0.3.14.dist-info → pixeltable-0.3.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pixeltable
-Version: 0.3.14
+Version: 0.3.15
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 License: Apache-2.0
 Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai

{pixeltable-0.3.14.dist-info → pixeltable-0.3.15.dist-info}/RECORD

@@ -1,5 +1,5 @@
 pixeltable/__init__.py,sha256=-uXHuiXH98kAlCupUTPbkBh4ToZgxcYUOk7-c9hqCC8,1439
-pixeltable/__version__.py,sha256=XhiJRUjdjrwJPbZIKiObIa0uwoOqWB_MxrLdbtgVGS8,114
+pixeltable/__version__.py,sha256=HD2KV9QVAFaAv42gJZzYHDLMgRFeCQBTYL5eYWvU_Kk,114
 pixeltable/catalog/__init__.py,sha256=rQmjveID4bk6NI4Ql09lGsZ0K0HVE2l1yqKAveipHzc,558
 pixeltable/catalog/catalog.py,sha256=E-Fqf3xDsFWLuau41trwqFRQbmNjy-LlT_votgYRM8k,48780
 pixeltable/catalog/column.py,sha256=f_jSvdV7DIe3MYXc4n4tSSJmAKIiNfPyT6i97rt2ewA,11133
@@ -10,9 +10,9 @@ pixeltable/catalog/named_function.py,sha256=vZ-j7P4HugWh9OmUzBMwyRYvO3tQn9jWyJz_
 pixeltable/catalog/path.py,sha256=gk8TIlO_7Jpji5mAN0dUNvHmvU0uneTHeB_qCTWnszQ,2529
 pixeltable/catalog/schema_object.py,sha256=J96iXsKMvqTNN0jbcMOPZDSZDNq8688Vkybs5bFcqNk,1818
 pixeltable/catalog/table.py,sha256=mlMbV6cWafm6OC9AHmMfpHuTYTyhGf5rhMHV2537tF8,65129
-pixeltable/catalog/table_version.py,sha256=1Yff0GylzRpsZZS0I8sEGkSxJUMdGqpGoB1QsYjtjGs,62249
+pixeltable/catalog/table_version.py,sha256=BlxPBec17v_zcZ2pJGqyT9rJfj2Jyxau472DfZYc3jg,62440
 pixeltable/catalog/table_version_handle.py,sha256=LYJFTdRssPu4uOBPbP93wKqXygJXr3Gwdc9wHzzZRag,1654
-pixeltable/catalog/table_version_path.py,sha256=r4WHtP2dkF05UafiQe47RWkszPiy8ZsdcTMA5mEzRp4,6847
+pixeltable/catalog/table_version_path.py,sha256=_g9knGenpMOlhaK8DZa8iLz5CorsMcbnFqTLnvaUkYM,6653
 pixeltable/catalog/view.py,sha256=u9c9YL9dHw21qGYf-QYST7nULhut-ECK-DWR1_MH3ik,13027
 pixeltable/config.py,sha256=gnRI4G9GE7mQJDcMcn8JsEzYk8oKVfHB-BwoLRWnRDo,3971
 pixeltable/dataframe.py,sha256=n_ZF_JdCq-DkpOzLhlnCspjDS2I8_7M0nlkTcDQVqyA,51593
@@ -84,13 +84,13 @@ pixeltable/functions/bedrock.py,sha256=lTCFHjYunF3minBGWcjXR90yJ8resFjXr4niyKhfx
 pixeltable/functions/date.py,sha256=WUwqyrOWB8A00cTNEd6Vd7anQZo40_-7EWhpfpI-P6c,5323
 pixeltable/functions/deepseek.py,sha256=KYIa-UJJUTOt9cCfmP6k6nM4MpKm1MBU8F-jWk3CycY,3827
 pixeltable/functions/fireworks.py,sha256=k0vUXxeeNYWfL_tdLgF9c-vOilr0g2tTeLkAL9SJ6ws,4972
-pixeltable/functions/gemini.py,sha256=GTtYBCNQG0DXBrBCdd92A2KVfP6JLuBESdCJ2XRrlBU,2969
+pixeltable/functions/gemini.py,sha256=zWPsvtq0mPFBC_-4N7NDuhTYZfAzRMmZa9S4GFjIpLg,8328
 pixeltable/functions/globals.py,sha256=ZXBV2LPXT2-yQYHHE7q8N1WdAr0WxiIO1ax0qwxhmK8,5118
 pixeltable/functions/huggingface.py,sha256=KM1OH0Jt6XWF2jfpHb6rGhi1mV-AQNYAsHAyQfzW4qw,20560
 pixeltable/functions/image.py,sha256=IKXljMma-uU88efptC3F4aywau7DYcD-Nqd3YpmRNRw,13971
 pixeltable/functions/json.py,sha256=d7-AvwytUQtQYF_JnWJkptT_Yq0NgMpWfVk-m3U6qTY,807
 pixeltable/functions/llama_cpp.py,sha256=uf7WSZIhKDa492snnQv5ojGVLNdBWvuw0Ou3Mch1c_I,3874
-pixeltable/functions/math.py,sha256=gmkeAWm_FbWqiekVOK8fyRs7A87kKE9rCdYE0ETfGj4,3357
+pixeltable/functions/math.py,sha256=eZEFjXxNHDHjcCsOMhzfNbJthTsmtNxtSFV8AEeRIfM,4979
 pixeltable/functions/mistralai.py,sha256=yZge5T385RoiFGXEZ6OhwWHj0JnsZ8tN8Jb3VkfDmXc,6274
 pixeltable/functions/ollama.py,sha256=AmkP532HwWeTyWkTnHm_hIk0CFjzV5MwCCPnM9Kb7KM,4231
 pixeltable/functions/openai.py,sha256=aDh1L2mBbSlrM8c1Rbh2QsCnmBESItLqzZ-frdgb05k,29259
@@ -115,7 +115,7 @@ pixeltable/io/globals.py,sha256=Z8ww-Pcm59ql1tvame8z0Mu1thIy5BPbW-TswGRXt4s,1136
 pixeltable/io/hf_datasets.py,sha256=gWyBH_0iFvxcrrxMY9_W399ZRcNDCmWFOAMmb1apnY0,5246
 pixeltable/io/label_studio.py,sha256=uB-LReXf1l2OMuzJEENxJP-0C14r14VEmsIulK8Yr3s,31261
 pixeltable/io/pandas.py,sha256=AbOeRDlA4MvUvianSKixsU-x-64nasPWw4HCHD6emz4,8981
-pixeltable/io/parquet.py,sha256=MC2n1ybf0l9O2h873SuNEJHv1bTGA2cV0ei_wQCgbwo,7757
+pixeltable/io/parquet.py,sha256=Aav5tkqY22gHSWTxsAnlk6MkRLi0OarXgp_N0JYOfHA,7791
 pixeltable/io/table_data_conduit.py,sha256=gdjr82HxJpDfH55xmbIUCX5V-Hkaj6Kmo25NESKChtk,23205
 pixeltable/io/utils.py,sha256=YMfhpqMitWz1PhXJGkCNOgNtEM1AZ55S0zXVhljC5kY,4260
 pixeltable/iterators/__init__.py,sha256=bU4EmbX85J1URmRw6G71f2I77b1ctqngEOwDmRB3T0w,455
@@ -156,9 +156,9 @@ pixeltable/metadata/schema.py,sha256=EKmx29vfQo3eGD2uCJW_lPalPialSb2oUSBGTyewduE
 pixeltable/plan.py,sha256=VfXTvEYYiiLPBdw0hoTmdXHE5IeQKZc1ej8l9a3XAns,43632
 pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pixeltable/share/__init__.py,sha256=AtR4nS6YkfkFRkXA-zZXFTK5pSQjHry8MnxdVLUk5SA,68
-pixeltable/share/packager.py,sha256=hF1_jFgEFq-94zF-PykwcgXbl4h40dgemBwMeSj8e9M,15590
+pixeltable/share/packager.py,sha256=hYpss8DKggZcY9-6Tl8oX6IRUq97p1bqduyfMSfjaZ4,26675
 pixeltable/share/publish.py,sha256=MZ_tsCSM9nUnrz8P1gbwatlpbS6EJYjYAd7S06lHw9M,6533
-pixeltable/store.py,sha256=j9rieIVD2fD7aIEcJ2xPf0vGCi0OirYk-A_JKEB9HVo,23693
+pixeltable/store.py,sha256=HVe99eQ_fk9CYZHFjYFfFy-sc95R9ADc4clPnHZeNZ0,24233
 pixeltable/type_system.py,sha256=DSrof2NgKhBzvt7pbDNrGlZ3rkkDJ7MQsQ9rqk9N9pA,53988
 pixeltable/utils/__init__.py,sha256=Pwgu-Sg1XkxzdCZ4ZhWP77UgLP3tnQsyCKaUJLF4ajo,1741
 pixeltable/utils/arrow.py,sha256=74wIy58rDYZJBVQ1g85NqzFyiQBvEQhnJ0Gi82iZ0dw,6421
@@ -179,8 +179,8 @@ pixeltable/utils/pytorch.py,sha256=564VHRdDHwD9h0v5lBHEDTJ8c6zx8wuzWYx8ZYjBxlI,3
 pixeltable/utils/s3.py,sha256=pxip2MlCqd2Qon2dzJXzfxvwtZyc-BAsjAnLL4J_OXY,587
 pixeltable/utils/sql.py,sha256=Sa4Lh-VGe8GToU5W7DRiWf2lMl9B6saPqemiT0ZdHEc,806
 pixeltable/utils/transactional_directory.py,sha256=OFKmu90oP7KwBAljwjnzP_w8euGdAXob3y4Nx9SCNHA,1357
-pixeltable-0.3.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pixeltable-0.3.14.dist-info/METADATA,sha256=Z7hUCAAUayqhytbLmw1_uIW_IS2lUBq-rRd9sSOoHx8,20540
-pixeltable-0.3.14.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-pixeltable-0.3.14.dist-info/entry_points.txt,sha256=ToOd-pRgG7AitEBgYoBCRRB4-KVDQ0pj_9T4a1LgwA4,97
-pixeltable-0.3.14.dist-info/RECORD,,
+pixeltable-0.3.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pixeltable-0.3.15.dist-info/METADATA,sha256=YAC_YaXgK70Eo19mvT9rhBj0WKkwYVrMvz4b9YMB8Ag,20540
+pixeltable-0.3.15.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+pixeltable-0.3.15.dist-info/entry_points.txt,sha256=ToOd-pRgG7AitEBgYoBCRRB4-KVDQ0pj_9T4a1LgwA4,97
+pixeltable-0.3.15.dist-info/RECORD,,