pixeltable 0.2.19__py3-none-any.whl → 0.2.20__py3-none-any.whl

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.19"
-__version_tuple__ = (0, 2, 19)
+__version__ = "0.2.20"
+__version_tuple__ = (0, 2, 20)

pixeltable/catalog/insertable_table.py

@@ -10,6 +10,7 @@ import pixeltable as pxt
 import pixeltable.type_system as ts
 from pixeltable import exceptions as excs
 from pixeltable.env import Env
+from pixeltable.utils.filecache import FileCache
 
 from .catalog import Catalog
 from .globals import UpdateStatus
@@ -101,21 +102,22 @@ class InsertableTable(Table):
             if not isinstance(row, dict):
                 raise excs.Error('rows must be a list of dictionaries')
         self._validate_input_rows(rows)
-        result = self._tbl_version.insert(rows, None, print_stats=print_stats, fail_on_exception=fail_on_exception)
+        status = self._tbl_version.insert(rows, None, print_stats=print_stats, fail_on_exception=fail_on_exception)
 
-        if result.num_excs == 0:
+        if status.num_excs == 0:
             cols_with_excs_str = ''
         else:
             cols_with_excs_str = \
-                f' across {len(result.cols_with_excs)} column{"" if len(result.cols_with_excs) == 1 else "s"}'
-            cols_with_excs_str += f' ({", ".join(result.cols_with_excs)})'
+                f' across {len(status.cols_with_excs)} column{"" if len(status.cols_with_excs) == 1 else "s"}'
+            cols_with_excs_str += f' ({", ".join(status.cols_with_excs)})'
         msg = (
-            f'Inserted {result.num_rows} row{"" if result.num_rows == 1 else "s"} '
-            f'with {result.num_excs} error{"" if result.num_excs == 1 else "s"}{cols_with_excs_str}.'
+            f'Inserted {status.num_rows} row{"" if status.num_rows == 1 else "s"} '
+            f'with {status.num_excs} error{"" if status.num_excs == 1 else "s"}{cols_with_excs_str}.'
         )
         print(msg)
         _logger.info(f'InsertableTable {self._name}: {msg}')
-        return result
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def _validate_input_rows(self, rows: List[Dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""

pixeltable/catalog/table.py

@@ -20,6 +20,7 @@ import pixeltable.exprs as exprs
 import pixeltable.index as index
 import pixeltable.metadata.schema as schema
 import pixeltable.type_system as ts
+from pixeltable.utils.filecache import FileCache
 
 from .column import Column
 from .globals import _ROWID_COLUMN_NAME, UpdateStatus, is_system_column_name, is_valid_identifier
@@ -33,7 +34,12 @@ if TYPE_CHECKING:
 _logger = logging.getLogger('pixeltable')
 
 class Table(SchemaObject):
-    """Base class for table objects (base tables, views, snapshots)."""
+    """
+    Base class for table objects (base tables, views, snapshots).
+
+    Every user-invoked operation that runs an ExecNode tree (directly or indirectly) needs to call
+    FileCache.emit_eviction_warnings() at the end of the operation.
+    """
 
     def __init__(self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath):
         super().__init__(id, name, dir_id)
@@ -374,7 +380,10 @@ class Table(SchemaObject):
 
         new_col = self._create_columns({col_name: col_schema})[0]
         self._verify_column(new_col, set(self._schema.keys()), set(self._query_names))
-        return self._tbl_version.add_column(new_col, print_stats=print_stats)
+        status = self._tbl_version.add_column(new_col, print_stats=print_stats)
+        FileCache.get().emit_eviction_warnings()
+        return status
+
 
     @classmethod
     def _validate_column_spec(cls, name: str, spec: dict[str, Any]) -> None:
@@ -587,6 +596,7 @@ class Table(SchemaObject):
         idx = EmbeddingIndex(col, metric=metric, string_embed=string_embed, image_embed=image_embed)
         status = self._tbl_version.add_index(col, idx_name=idx_name, idx=idx)
         # TODO: how to deal with exceptions here? drop the index and raise?
+        FileCache.get().emit_eviction_warnings()
 
     def drop_embedding_index(self, *, column_name: Optional[str] = None, idx_name: Optional[str] = None) -> None:
         """Drop an embedding index from the table.
@@ -732,7 +742,9 @@ class Table(SchemaObject):
             >>> tbl.update({'int_col': tbl.int_col + 1}, where=tbl.int_col == 0)
         """
         self._check_is_dropped()
-        return self._tbl_version.update(value_spec, where, cascade)
+        status = self._tbl_version.update(value_spec, where, cascade)
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def batch_update(
             self, rows: Iterable[dict[str, Any]], cascade: bool = True,
@@ -789,9 +801,11 @@ class Table(SchemaObject):
                     missing_cols = pk_col_names - set(col.name for col in col_vals.keys())
                     raise excs.Error(f'Primary key columns ({", ".join(missing_cols)}) missing in {row_spec}')
             row_updates.append(col_vals)
-        return self._tbl_version.batch_update(
+        status = self._tbl_version.batch_update(
             row_updates, rowids, error_if_not_exists=if_not_exists == 'error',
             insert_if_not_exists=if_not_exists == 'insert', cascade=cascade)
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def delete(self, where: Optional['pixeltable.exprs.Expr'] = None) -> UpdateStatus:
         """Delete rows in this table.

pixeltable/env.py

@@ -8,6 +8,7 @@ import importlib.util
 import inspect
 import logging
 import os
+import shutil
 import subprocess
 import sys
 import threading
@@ -15,12 +16,12 @@ import uuid
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import pixeltable_pgserver
 import sqlalchemy as sql
-import yaml
+import toml
 from tqdm import TqdmWarning
 
 import pixeltable.exceptions as excs
@@ -64,7 +65,7 @@ class Env:
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
     _config_file: Optional[Path]
-    _config: Optional[dict[str, Any]]
+    _config: Optional[Config]
     _stdout_handler: logging.StreamHandler
     _initialized: bool
 
@@ -110,6 +111,7 @@ class Env:
         self._log_to_stdout = False
         self._module_log_level = {}  # module name -> log level
 
+        # config
         self._config_file = None
         self._config = None
 
@@ -119,7 +121,8 @@ class Env:
         self._initialized = False
 
     @property
-    def config(self):
+    def config(self) -> Config:
+        assert self._config is not None
         return self._config
 
     @property
@@ -227,30 +230,13 @@ class Env:
         home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
         assert self._home is None or self._home == home
         self._home = home
-        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.yaml')))
+        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.toml')))
         self._media_dir = self._home / 'media'
         self._file_cache_dir = self._home / 'file_cache'
         self._dataset_cache_dir = self._home / 'dataset_cache'
         self._log_dir = self._home / 'logs'
         self._tmp_dir = self._home / 'tmp'
 
-        # Read in the config
-        if os.path.isfile(self._config_file):
-            with open(self._config_file, 'r') as stream:
-                try:
-                    self._config = yaml.safe_load(stream)
-                except yaml.YAMLError as exc:
-                    self._logger.error(f'Could not read config file: {self._config_file}')
-                    self._config = {}
-        else:
-            self._config = {}
-
-        # Disable spurious warnings
-        warnings.simplefilter('ignore', category=TqdmWarning)
-        if 'hide_warnings' in self._config and self._config['hide_warnings']:
-            # Disable more warnings
-            warnings.simplefilter('ignore', category=UserWarning)
-
         if self._home.exists() and not self._home.is_dir():
             raise RuntimeError(f'{self._home} is not a directory')
 
@@ -274,6 +260,22 @@ class Env:
         if not self._tmp_dir.exists():
             self._tmp_dir.mkdir()
 
+        # Read in the config
+        self._config = Config.from_file(self._config_file)
+        self._file_cache_size_g = self._config.get_float_value('file_cache_size_g')
+        if self._file_cache_size_g is None:
+            raise excs.Error(
+                'pixeltable/file_cache_size_g is missing from configuration\n'
+                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {self._config_file},\n'
+                'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
+            )
+
+        # Disable spurious warnings
+        warnings.simplefilter('ignore', category=TqdmWarning)
+        if self._config.get_bool_value('hide_warnings'):
+            # Disable more warnings
+            warnings.simplefilter('ignore', category=UserWarning)
+
         # configure _logger to log to a file
         self._logfilename = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + '.log'
         fh = logging.FileHandler(self._log_dir / self._logfilename, mode='w')
@@ -313,7 +315,7 @@ class Env:
         self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
         self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
 
-        tz_name = os.environ.get('PXT_TIME_ZONE', self._config.get('pxt_time_zone', None))
+        tz_name = self.config.get_string_value('time_zone')
         if tz_name is not None:
             # Validate tzname
             if not isinstance(tz_name, str):
@@ -440,21 +442,18 @@ class Env:
         if cl.client_obj is not None:
             return cl.client_obj  # Already initialized
 
-        # Construct a client. For each client parameter, first check if the parameter is in the environment;
-        # if not, look in Pixeltable config from `config.yaml`.
+        # Construct a client, retrieving each parameter from config.
 
         init_kwargs: dict[str, str] = {}
         for param in cl.param_names:
-            environ = f'{name.upper()}_{param.upper()}'
-            if environ in os.environ:
-                init_kwargs[param] = os.environ[environ]
-            elif name.lower() in self._config and param in self._config[name.lower()]:
-                init_kwargs[param] = self._config[name.lower()][param.lower()]
-            if param not in init_kwargs or init_kwargs[param] == '':
+            arg = self._config.get_string_value(param, section=name)
+            if arg is not None and len(arg) > 0:
+                init_kwargs[param] = arg
+            else:
                 raise excs.Error(
                     f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{environ}` environment variable, or put `{param.lower()}` in '
-                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.yaml.'
+                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, or put `{param.lower()}` in '
+                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
         cl.client_obj = cl.init_fn(**init_kwargs)
@@ -506,7 +505,6 @@ class Env:
         self.__register_package('spacy')
         self.__register_package('tiktoken')
         self.__register_package('together')
-        self.__register_package('toml')
         self.__register_package('torch')
         self.__register_package('torchvision')
         self.__register_package('transformers')
@@ -643,7 +641,7 @@ def register_client(name: str) -> Callable:
     Pixeltable will attempt to load the client parameters from config. For each
     config parameter:
     - If an environment variable named MY_CLIENT_API_KEY (for example) is set, use it;
-    - Otherwise, look for 'api_key' in the 'my_client' section of config.yaml.
+    - Otherwise, look for 'api_key' in the 'my_client' section of config.toml.
 
     If all config parameters are found, Pixeltable calls the initialization function;
     otherwise it throws an exception.
@@ -660,6 +658,79 @@ def register_client(name: str) -> Callable:
     return decorator
 
 
+class Config:
+    """
+    The (global) Pixeltable configuration, as loaded from `config.toml`. Provides methods for retrieving
+    configuration values, which can be set in the config file or as environment variables.
+    """
+    __config: dict[str, Any]
+
+    T = TypeVar('T')
+
+    @classmethod
+    def from_file(cls, path: Path) -> Config:
+        """
+        Loads configuration from the specified TOML file. If the file does not exist, it will be
+        created and populated with the default configuration.
+        """
+        if os.path.isfile(path):
+            with open(path, 'r') as stream:
+                try:
+                    config_dict = toml.load(stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not read config file: {str(path)}') from exc
+        else:
+            config_dict = cls.__create_default_config(path)
+            with open(path, 'w') as stream:
+                try:
+                    toml.dump(config_dict, stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not write config file: {str(path)}') from exc
+            logging.getLogger('pixeltable').info(f'Created default config file at: {str(path)}')
+        return cls(config_dict)
+
+    @classmethod
+    def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
+        free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
+        # Default cache size is 1/5 of free disk space
+        file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
+        return {
+            'pixeltable': {
+                'file_cache_size_g': round(file_cache_size_g, 1),
+                'hide_warnings': False,
+            }
+        }
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self.__config = config
+
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+        env_var = f'{section.upper()}_{key.upper()}'
+        if env_var in os.environ:
+            value = os.environ[env_var]
+        elif section in self.__config and key in self.__config[section]:
+            value = self.__config[section][key]
+        else:
+            return None
+
+        try:
+            return expected_type(value)  # type: ignore[call-arg]
+        except ValueError:
+            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
+
+    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
+        return self.get_value(key, str, section)
+
+    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
+        return self.get_value(key, int, section)
+
+    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
+        return self.get_value(key, float, section)
+
+    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
+        return self.get_value(key, bool, section)
+
+
 _registered_clients: dict[str, ApiClient] = {}
 
 

pixeltable/exceptions.py

@@ -1,6 +1,9 @@
-from typing import List, Any
-from types import TracebackType
 from dataclasses import dataclass
+from types import TracebackType
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pixeltable import exprs
 
 
 class Error(Exception):
@@ -9,11 +12,11 @@ class Error(Exception):
 
 @dataclass
 class ExprEvalError(Exception):
-    expr: Any  # exprs.Expr, but we're not importing pixeltable.exprs to avoid circular imports
+    expr: 'exprs.Expr'
     expr_msg: str
     exc: Exception
     exc_tb: TracebackType
-    input_vals: List[Any]
+    input_vals: list[Any]
     row_num: int
 
 

pixeltable/exec/component_iteration_node.py

@@ -1,10 +1,12 @@
-from typing import Generator, Optional
+import inspect
+from typing import Iterator, Optional
 
-from .data_row_batch import DataRowBatch
-from .exec_node import ExecNode
 import pixeltable.catalog as catalog
-import pixeltable.exprs as exprs
 import pixeltable.exceptions as excs
+import pixeltable.exprs as exprs
+
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
 
 
 class ComponentIterationNode(ExecNode):
@@ -12,7 +14,7 @@ class ComponentIterationNode(ExecNode):
 
     Returns row batches of OUTPUT_BATCH_SIZE size.
     """
-    OUTPUT_BATCH_SIZE = 1024
+    __OUTPUT_BATCH_SIZE = 1024
 
     def __init__(self, view: catalog.TableVersion, input: ExecNode):
         assert view.is_component_view()
@@ -23,57 +25,76 @@ class ComponentIterationNode(ExecNode):
         self.iterator_args = iterator_args[0]
         assert isinstance(self.iterator_args, exprs.InlineDict)
         self.iterator_args_ctx = self.row_builder.create_eval_ctx([self.iterator_args])
-        self.iterator_output_schema, self.unstored_column_names = \
+        self.iterator_output_schema, self.unstored_column_names = (
             self.view.iterator_cls.output_schema(**self.iterator_args.to_kwargs())
+        )
         self.iterator_output_fields = list(self.iterator_output_schema.keys())
-        self.iterator_output_cols = \
-            {field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields}
+        self.iterator_output_cols = {
+            field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields
+        }
         # referenced iterator output fields
         self.refd_output_slot_idxs = {
             e.col.name: e.slot_idx for e in self.row_builder.unique_exprs
             if isinstance(e, exprs.ColumnRef) and e.col.name in self.iterator_output_fields
         }
-        self._output: Optional[Generator[DataRowBatch, None, None]] = None
+        self.__output: Optional[Iterator[DataRowBatch]] = None
 
-    def _output_batches(self) -> Generator[DataRowBatch, None, None]:
+    def __output_batches(self) -> Iterator[DataRowBatch]:
         output_batch = DataRowBatch(self.view, self.row_builder)
         for input_batch in self.input:
             for input_row in input_batch:
                 self.row_builder.eval(input_row, self.iterator_args_ctx)
                 iterator_args = input_row[self.iterator_args.slot_idx]
-                iterator = self.view.iterator_cls(**iterator_args)
-                for pos, component_dict in enumerate(iterator):
-                    output_row = output_batch.add_row()
-                    input_row.copy(output_row)
-                    # we're expanding the input and need to add the iterator position to the pk
-                    pk = output_row.pk[:-1] + (pos,) + output_row.pk[-1:]
-                    output_row.set_pk(pk)
-
-                    # verify and copy component_dict fields to their respective slots in output_row
-                    for field_name, field_val in component_dict.items():
-                        if field_name not in self.iterator_output_fields:
-                            raise excs.Error(
-                                f'Invalid field name {field_name} in output of {self.view.iterator_cls.__name__}')
-                        if field_name not in self.refd_output_slot_idxs:
-                            # we can ignore this
-                            continue
-                        output_col = self.iterator_output_cols[field_name]
-                        output_col.col_type.validate_literal(field_val)
-                        output_row[self.refd_output_slot_idxs[field_name]] = field_val
-                    if len(component_dict) != len(self.iterator_output_fields):
-                        missing_fields = set(self.refd_output_slot_idxs.keys()) - set(component_dict.keys())
-                        raise excs.Error(
-                            f'Invalid output of {self.view.iterator_cls.__name__}: '
-                            f'missing fields {", ".join(missing_fields)}')
-
-                    if len(output_batch) == self.OUTPUT_BATCH_SIZE:
-                        yield output_batch
-                        output_batch = DataRowBatch(self.view, self.row_builder)
+                assert isinstance(iterator_args, dict)
+                # We need to ensure that all of the required (non-nullable) parameters of the iterator are
+                # specified and are not null. If any of them are null, then we skip this row (i.e., we emit 0
+                # output rows for this input row).
+                if self.__non_nullable_args_specified(iterator_args):
+                    iterator = self.view.iterator_cls(**iterator_args)
+                    for pos, component_dict in enumerate(iterator):
+                        output_row = output_batch.add_row()
+                        input_row.copy(output_row)
+                        # we're expanding the input and need to add the iterator position to the pk
+                        self.__populate_output_row(output_row, pos, component_dict)
+                        if len(output_batch) == self.__OUTPUT_BATCH_SIZE:
+                            yield output_batch
+                            output_batch = DataRowBatch(self.view, self.row_builder)
 
         if len(output_batch) > 0:
             yield output_batch
 
+    def __non_nullable_args_specified(self, iterator_args: dict) -> bool:
+        """
+        Returns true if all non-nullable iterator arguments are not `None`.
+        """
+        input_schema = self.view.iterator_cls.input_schema()
+        for arg_name, arg_value in iterator_args.items():
+            col_type = input_schema[arg_name]
+            if arg_value is None and not col_type.nullable:
+                return False
+        return True
+
+    def __populate_output_row(self, output_row: exprs.DataRow, pos: int, component_dict: dict) -> None:
+        pk = output_row.pk[:-1] + (pos,) + output_row.pk[-1:]
+        output_row.set_pk(pk)
+        # verify and copy component_dict fields to their respective slots in output_row
+        for field_name, field_val in component_dict.items():
+            if field_name not in self.iterator_output_fields:
+                raise excs.Error(
+                    f'Invalid field name {field_name} in output of {self.view.iterator_cls.__name__}')
+            if field_name not in self.refd_output_slot_idxs:
+                # we can ignore this
+                continue
+            output_col = self.iterator_output_cols[field_name]
+            output_col.col_type.validate_literal(field_val)
+            output_row[self.refd_output_slot_idxs[field_name]] = field_val
+        if len(component_dict) != len(self.iterator_output_fields):
+            missing_fields = set(self.refd_output_slot_idxs.keys()) - set(component_dict.keys())
+            raise excs.Error(
+                f'Invalid output of {self.view.iterator_cls.__name__}: '
+                f'missing fields {", ".join(missing_fields)}')
+
     def __next__(self) -> DataRowBatch:
-        if self._output is None:
-            self._output = self._output_batches()
-        return next(self._output)
+        if self.__output is None:
+            self.__output = self.__output_batches()
+        return next(self.__output)

pixeltable/exprs/json_path.py

@@ -105,12 +105,9 @@ class JsonPath(Expr):
         return JsonPath(self._anchor, self.path_elements + [name])
 
     def __getitem__(self, index: object) -> 'JsonPath':
-        if isinstance(index, str):
-            if index != '*':
-                raise excs.Error(f'Invalid json list index: {index}')
-        elif not isinstance(index, (int, slice)):
-                raise excs.Error(f'Invalid json list index: {index}')
-        return JsonPath(self._anchor, self.path_elements + [index])
+        if isinstance(index, (int, slice, str)):
+            return JsonPath(self._anchor, self.path_elements + [index])
+        raise excs.Error(f'Invalid json list index: {index}')
 
     def __rshift__(self, other: object) -> 'JsonMapper':
         rhs_expr = Expr.from_object(other)

pixeltable/functions/together.py

@@ -7,13 +7,15 @@ the [Working with Together AI](https://pixeltable.readme.io/docs/together-ai) tu
 
 import base64
 import io
-from typing import TYPE_CHECKING, Callable, Optional
+from typing import TYPE_CHECKING, Callable, Optional, TypeVar
 
 import numpy as np
 import PIL.Image
+import requests
 import tenacity
 
 import pixeltable as pxt
+import pixeltable.exceptions as excs
 from pixeltable import env
 from pixeltable.func import Batch
 from pixeltable.utils.code import local_public_names
@@ -32,7 +34,10 @@ def _together_client() -> 'together.Together':
     return env.Env.get().get_client('together')
 
 
-def _retry(fn: Callable) -> Callable:
+T = TypeVar('T')
+
+
+def _retry(fn: Callable[..., T]) -> Callable[..., T]:
     import together
     return tenacity.retry(
         retry=tenacity.retry_if_exception_type(together.error.RateLimitError),
@@ -249,20 +254,29 @@ def image_generations(
         The generated image.
 
     Examples:
-        Add a computed column that applies the model `runwayml/stable-diffusion-v1-5`
+        Add a computed column that applies the model `stabilityai/stable-diffusion-xl-base-1.0`
         to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
 
-        >>> tbl['response'] = image_generations(tbl.prompt, model='runwayml/stable-diffusion-v1-5')
+        >>> tbl['response'] = image_generations(tbl.prompt, model='stabilityai/stable-diffusion-xl-base-1.0')
     """
-    # TODO(aaron-siegel): Decompose CPU/GPU ops into separate functions
     result = _retry(_together_client().images.generate)(
         prompt=prompt, model=model, steps=steps, seed=seed, height=height, width=width, negative_prompt=negative_prompt
     )
-    b64_str = result.data[0].b64_json
-    b64_bytes = base64.b64decode(b64_str)
-    img = PIL.Image.open(io.BytesIO(b64_bytes))
-    img.load()
-    return img
+    if result.data[0].b64_json is not None:
+        b64_bytes = base64.b64decode(result.data[0].b64_json)
+        img = PIL.Image.open(io.BytesIO(b64_bytes))
+        img.load()
+        return img
+    if result.data[0].url is not None:
+        try:
+            resp = requests.get(result.data[0].url)
+            with io.BytesIO(resp.content) as fp:
+                image = PIL.Image.open(fp)
+                image.load()
+                return image
+        except Exception as exc:
+            raise excs.Error('Failed to download generated image from together.ai.') from exc
+    raise excs.Error('Response does not contain a generated image.')
 
 
 __all__ = local_public_names(__name__)

pixeltable/globals.py

@@ -16,6 +16,7 @@ from pixeltable.dataframe import DataFrameResultSet
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
+from pixeltable.utils.filecache import FileCache
 
 _logger = logging.getLogger('pixeltable')
 
@@ -193,6 +194,7 @@ def create_view(
     )
     Catalog.get().paths[path] = view
     _logger.info(f'Created view `{path_str}`.')
+    FileCache.get().emit_eviction_warnings()
     return view
 
 

pixeltable/io/globals.py

@@ -43,7 +43,7 @@ def create_label_studio_project(
     The API key and URL for a valid Label Studio server must be specified in Pixeltable config. Either:
 
     * Set the `LABEL_STUDIO_API_KEY` and `LABEL_STUDIO_URL` environment variables; or
-    * Specify `api_key` and `url` fields in the `label-studio` section of `$PIXELTABLE_HOME/config.yaml`.
+    * Specify `api_key` and `url` fields in the `label-studio` section of `$PIXELTABLE_HOME/config.toml`.
 
     __Requirements:__
 

pixeltable/io/hf_datasets.py

@@ -34,9 +34,7 @@ _hf_to_pxt: dict[str, ts.ColumnType] = {
 }
 
 
-def _to_pixeltable_type(
-    feature_type: Union[datasets.ClassLabel, datasets.Value, datasets.Sequence],
-) -> Optional[ts.ColumnType]:
+def _to_pixeltable_type(feature_type: Any) -> Optional[ts.ColumnType]:
     """Convert a huggingface feature type to a pixeltable ColumnType if one is defined."""
     import datasets
 
@@ -51,6 +49,8 @@ def _to_pixeltable_type(
         dtype = _to_pixeltable_type(feature_type.feature)
         length = feature_type.length if feature_type.length != -1 else None
         return ts.ArrayType(shape=(length,), dtype=dtype)
+    elif isinstance(feature_type, datasets.Image):
+        return ts.ImageType(nullable=True)
     else:
         return None
 

pixeltable/iterators/document.py

@@ -166,7 +166,7 @@ class DocumentSplitter(ComponentIterator):
         return {
             'document': DocumentType(nullable=False),
             'separators': StringType(nullable=False),
-            'metadata': StringType(nullable=True),
+            'metadata': StringType(nullable=False),
             'limit': IntType(nullable=True),
             'overlap': IntType(nullable=True),
             'skip_tags': StringType(nullable=True),

pixeltable/tool/create_test_db_dump.py

@@ -36,7 +36,7 @@ class Dumper:
         mock_home_dir = self.output_dir / '.pixeltable'
         mock_home_dir.mkdir(parents=True, exist_ok=True)
         os.environ['PIXELTABLE_HOME'] = str(mock_home_dir)
-        os.environ['PIXELTABLE_CONFIG'] = str(shared_home / 'config.yaml')
+        os.environ['PIXELTABLE_CONFIG'] = str(shared_home / 'config.toml')
         os.environ['PIXELTABLE_DB'] = db_name
         os.environ['PIXELTABLE_PGDATA'] = str(shared_home / 'pgdata')
 

pixeltable/utils/filecache.py

@@ -1,28 +1,33 @@
 from __future__ import annotations
-from typing import Optional, List, Tuple, Dict
-from collections import OrderedDict, defaultdict, namedtuple
-import os
+
 import glob
-from pathlib import Path
-from time import time
+import hashlib
 import logging
+import os
+import warnings
+from collections import OrderedDict, defaultdict, namedtuple
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
 from uuid import UUID
-import hashlib
 
+import pixeltable.exceptions as excs
 from pixeltable.env import Env
 
-
 _logger = logging.getLogger('pixeltable')
 
+@dataclass
 class CacheEntry:
-    def __init__(self, key: str, tbl_id: UUID, col_id: int, size: int, last_accessed_ts: int, ext: str):
-        self.key = key
-        self.tbl_id = tbl_id
-        self.col_id = col_id
-        self.size = size
-        self.last_accessed_ts = last_accessed_ts
-        self.ext = ext
 
+    key: str
+    tbl_id: UUID
+    col_id: int
+    size: int
+    last_used: datetime
+    ext: str
+
+    @property
     def path(self) -> Path:
         return Env.get().file_cache_dir / f'{self.tbl_id.hex}_{self.col_id}_{self.key}{self.ext}'
 
@@ -34,7 +39,11 @@ class CacheEntry:
         col_id = int(components[1])
         key = components[2]
         file_info = os.stat(str(path))
-        return cls(key, tbl_id, col_id, file_info.st_size, file_info.st_mtime, path.suffix)
+        # We use the last modified time (file_info.st_mtime) as the timestamp; `FileCache` will touch the file
+        # each time it is retrieved, so that the mtime of the file will always represent the last used time of
+        # the cache entry.
+        last_used = datetime.fromtimestamp(file_info.st_mtime, tz=timezone.utc)
+        return cls(key, tbl_id, col_id, file_info.st_size, last_used, path.suffix)
 
 
 class FileCache:
@@ -45,31 +54,60 @@ class FileCache:
     access of a cache entries is its file's mtime.
 
     TODO:
-    - enforce a maximum capacity with LRU eviction
     - implement MRU eviction for queries that exceed the capacity
     """
-    _instance: Optional[FileCache] = None
-    ColumnStats = namedtuple('FileCacheColumnStats', ['tbl_id', 'col_id', 'num_files', 'total_size'])
+    __instance: Optional[FileCache] = None
+
+    cache: OrderedDict[str, CacheEntry]
+    total_size: int
+    capacity_bytes: int
+    num_requests: int
+    num_hits: int
+    num_evictions: int
+    keys_retrieved: set[str]  # keys retrieved (downloaded or accessed) this session
+    keys_evicted_after_retrieval: set[str]  # keys that were evicted after having been retrieved this session
+
+    # A key is added to this set when it is already present in `keys_evicted_this_session` and is downloaded again.
+    # In other words, for a key to be added to this set, the following sequence of events must occur in this order:
+    # - It is retrieved during this session (either because it was newly downloaded, or because it was in the cache
+    #   at the start of the session and was accessed at some point during the session)
+    # - It is subsequently evicted
+    # - It is subsequently retrieved a second time ("download after a previous retrieval")
+    # The contents of this set will be used to generate a more informative warning.
+    evicted_working_set_keys: set[str]
+    new_redownload_witnessed: bool  # whether a new re-download has occurred since the last time a warning was issued
+
+    ColumnStats = namedtuple('FileCacheColumnStats', ('tbl_id', 'col_id', 'num_files', 'total_size'))
     CacheStats = namedtuple(
-        'FileCacheStats', ['total_size', 'num_requests', 'num_hits', 'num_evictions', 'column_stats'])
+        'FileCacheStats',
+        ('total_size', 'num_requests', 'num_hits', 'num_evictions', 'column_stats')
+    )
 
     @classmethod
     def get(cls) -> FileCache:
-        if cls._instance is None:
-            cls._instance = cls()
-        return cls._instance
+        if cls.__instance is None:
+            cls.init()
+        return cls.__instance
+
+    @classmethod
+    def init(cls) -> None:
+        cls.__instance = cls()
 
     def __init__(self):
-        self.cache: OrderedDict[str, CacheEntry] = OrderedDict()  # ordered by entry.last_accessed_ts
+        self.cache = OrderedDict()
         self.total_size = 0
-        #self.capacity = Env.get().max_filecache_size
+        self.capacity_bytes = Env.get()._file_cache_size_g * (1 << 30)
         self.num_requests = 0
         self.num_hits = 0
         self.num_evictions = 0
+        self.keys_retrieved = set()
+        self.keys_evicted_after_retrieval = set()
+        self.evicted_working_set_keys = set()
+        self.new_redownload_witnessed = False
         paths = glob.glob(str(Env.get().file_cache_dir / '*'))
         entries = [CacheEntry.from_file(Path(path_str)) for path_str in paths]
-        # we need to insert entries in order of last_accessed_ts
-        entries.sort(key=lambda e: e.last_accessed_ts)
+        # we need to insert entries in access order
+        entries.sort(key=lambda e: e.last_used)
         for entry in entries:
             self.cache[entry.key] = entry
             self.total_size += entry.size
@@ -82,30 +120,43 @@ class FileCache:
     def num_files(self, tbl_id: Optional[UUID] = None) -> int:
         if tbl_id is None:
             return len(self.cache)
-        entries = [e for e in self.cache.values() if e.tbl_id == tbl_id]
-        return len(entries)
+        return sum(e.tbl_id == tbl_id for e in self.cache.values())
 
-    def clear(self, tbl_id: Optional[UUID] = None, capacity: Optional[int] = None) -> None:
+    def clear(self, tbl_id: Optional[UUID] = None) -> None:
         """
         For testing purposes: allow resetting capacity and stats.
         """
-        self.num_requests, self.num_hits, self.num_evictions = 0, 0, 0
-        entries = list(self.cache.values())  # list(): avoid dealing with values() return type
-        if tbl_id is not None:
-            entries = [e for e in entries if e.tbl_id == tbl_id]
-            _logger.debug(f'clearing {len(entries)} entries from file cache for table {tbl_id}')
+        if tbl_id is None:
+            # We need to store the entries to remove in a list, because we can't remove items from a dict while iterating
+            entries_to_remove = list(self.cache.values())
+            _logger.debug(f'clearing {self.num_files()} entries from file cache')
+            self.num_requests, self.num_hits, self.num_evictions = 0, 0, 0
+            self.keys_retrieved.clear()
+            self.keys_evicted_after_retrieval.clear()
+            self.new_redownload_witnessed = False
         else:
-            _logger.debug(f'clearing {len(entries)} entries from file cache')
-        for entry in entries:
+            entries_to_remove = [e for e in self.cache.values() if e.tbl_id == tbl_id]
+            _logger.debug(f'clearing {self.num_files(tbl_id)} entries from file cache for table {tbl_id}')
+        for entry in entries_to_remove:
+            os.remove(entry.path)
             del self.cache[entry.key]
             self.total_size -= entry.size
-            os.remove(entry.path())
-        # if capacity is not None:
-        #     self.capacity = capacity
-        # else:
-        #     # need to reset to default
-        #     self.capacity = Env.get().max_filecache_size
-        # _logger.debug(f'setting file cache capacity to {self.capacity}')
+
+    def emit_eviction_warnings(self) -> None:
+        if self.new_redownload_witnessed:
+            # Compute the additional capacity that would be needed in order to retain all the re-downloaded files
+            extra_capacity_needed = sum(self.cache[key].size for key in self.evicted_working_set_keys)
+            suggested_cache_size = self.capacity_bytes + extra_capacity_needed + (1 << 30)
+            warnings.warn(
+                f'{len(self.evicted_working_set_keys)} media file(s) had to be downloaded multiple times this session, '
+                'because they were evicted\nfrom the file cache after their first access. The total size '
+                f'of the evicted file(s) is {round(extra_capacity_needed / (1 << 30), 1)} GiB.\n'
+                f'Consider increasing the cache size to at least {round(suggested_cache_size / (1 << 30), 1)} GiB '
+                f'(it is currently {round(self.capacity_bytes / (1 << 30), 1)} GiB).\n'
+                f'You can do this by setting the value of `file_cache_size_g` in: {str(Env.get()._config_file)}',
+                excs.PixeltableWarning
+            )
+            self.new_redownload_witnessed = False
 
     def _url_hash(self, url: str) -> str:
         h = hashlib.sha256()
@@ -120,66 +171,62 @@ class FileCache:
             _logger.debug(f'file cache miss for {url}')
             return None
         # update mtime and cache
-        path = entry.path()
+        path = entry.path
         path.touch(exist_ok=True)
         file_info = os.stat(str(path))
-        entry.last_accessed_ts = file_info.st_mtime
+        entry.last_used = file_info.st_mtime
         self.cache.move_to_end(key, last=True)
         self.num_hits += 1
+        self.keys_retrieved.add(key)
         _logger.debug(f'file cache hit for {url}')
         return path
 
-    # def can_admit(self, query_ts: int) -> bool:
-    #     if self.total_size + self.avg_file_size <= self.capacity:
-    #         return True
-    #     assert len(self.cache) > 0
-    #     # check whether we can evict the current lru entry
-    #     lru_entry = next(iter(self.cache.values()))
-    #     if lru_entry.last_accessed_ts >= query_ts:
-    #         # the current query brought this entry in: we're not going to evict it
-    #         return False
-    #     return True
-
     def add(self, tbl_id: UUID, col_id: int, url: str, path: Path) -> Path:
         """Adds url at 'path' to cache and returns its new path.
         'path' will not be accessible after this call. Retains the extension of 'path'.
         """
         file_info = os.stat(str(path))
-        _ = time()
-        #if self.total_size + file_info.st_size > self.capacity:
-        if False:
-            if len(self.cache) == 0:
-                # nothing to evict
-                return
-            # evict entries until we're below the limit or until we run into entries the current query brought in
-            while True:
-                lru_entry = next(iter(self.cache.values()))
-                if lru_entry.last_accessed_ts >= query_ts:
-                    # the current query brought this entry in: switch to MRU and ignore this put()
-                    _logger.debug('file cache switched to MRU')
-                    return
-                self.cache.popitem(last=False)
-                self.total_size -= lru_entry.size
-                self.num_evictions += 1
-                os.remove(str(lru_entry.path()))
-                _logger.debug(f'evicted entry for cell {lru_entry.cell_id} from file cache')
-                if self.total_size + file_info.st_size <= self.capacity:
-                    break
-
+        self.ensure_capacity(file_info.st_size)
         key = self._url_hash(url)
         assert key not in self.cache
+        if key in self.keys_evicted_after_retrieval:
+            # This key was evicted after being retrieved earlier this session, and is now being retrieved again.
+            # Add it to `keys_multiply_downloaded` so that we may generate a warning later.
+            self.evicted_working_set_keys.add(key)
+            self.new_redownload_witnessed = True
+        self.keys_retrieved.add(key)
         entry = CacheEntry(key, tbl_id, col_id, file_info.st_size, file_info.st_mtime, path.suffix)
         self.cache[key] = entry
         self.total_size += entry.size
-        new_path = entry.path()
+        new_path = entry.path
         os.rename(str(path), str(new_path))
+        new_path.touch(exist_ok=True)
         _logger.debug(f'added entry for cell {url} to file cache')
         return new_path
 
+    def ensure_capacity(self, size: int) -> None:
+        """
+        Evict entries from the cache until there is at least 'size' bytes of free space.
+        """
+        while len(self.cache) > 0 and self.total_size + size > self.capacity_bytes:
+            _, lru_entry = self.cache.popitem(last=False)
+            self.total_size -= lru_entry.size
+            self.num_evictions += 1
+            if lru_entry.key in self.keys_retrieved:
+                # This key was retrieved at some point earlier this session and is now being evicted.
+                # Make a record of the eviction, so that we can generate a warning later if the key is retrieved again.
+                self.keys_evicted_after_retrieval.add(lru_entry.key)
+            os.remove(str(lru_entry.path))
+            _logger.debug(f'evicted entry for cell {lru_entry.key} from file cache (of size {lru_entry.size // (1 << 20)} MiB)')
+
+    def set_capacity(self, capacity_bytes: int) -> None:
+        self.capacity_bytes = capacity_bytes
+        self.ensure_capacity(0)  # evict entries if necessary
+
     def stats(self) -> CacheStats:
         # collect column stats
         # (tbl_id, col_id) -> (num_files, total_size)
-        d: Dict[Tuple[int, int], List[int]] = defaultdict(lambda: [0, 0])
+        d: dict[tuple[int, int], list[int]] = defaultdict(lambda: [0, 0])
         for entry in self.cache.values():
             t = d[(entry.tbl_id, entry.col_id)]
             t[0] += 1

{pixeltable-0.2.19.dist-info → pixeltable-0.2.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.19
+Version: 0.2.20
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Pixeltable, Inc.
 Author-email: contact@pixeltable.com
@@ -31,6 +31,7 @@ Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0)
 Requires-Dist: tenacity (>=8.2,<9.0)
+Requires-Dist: toml (>=0.10)
 Requires-Dist: tqdm (>=4.64)
 Description-Content-Type: text/markdown
 
@@ -46,10 +47,17 @@ Description-Content-Type: text/markdown
 [![tests status](https://github.com/pixeltable/pixeltable/actions/workflows/nightly.yml/badge.svg)](https://github.com/pixeltable/pixeltable/actions/workflows/nightly.yml)
 [![PyPI Package](https://img.shields.io/pypi/v/pixeltable?color=darkorange)](https://pypi.org/project/pixeltable/)
 
-[Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.readme.io/) | [API Reference](https://pixeltable.github.io/pixeltable/) | [Code Samples](https://pixeltable.readme.io/recipes) | [Examples](https://github.com/pixeltable/pixeltable/tree/release/docs/release/tutorials)
+[Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.readme.io/) | [API Reference](https://pixeltable.github.io/pixeltable/) | [Code Samples](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#-code-samples) | [Computer Vision](https://docs.pixeltable.com/docs/object-detection-in-videos) | [LLM](https://docs.pixeltable.com/docs/document-indexing-and-rag)
 </div>
 
-Pixeltable is a Python library providing a declarative interface for multimodal data (text, images, audio, video). It features built-in versioning, lineage tracking, and incremental updates, enabling users to store, transform, index, and iterate on data for their ML workflows. Data transformations, model inference, and custom logic are embedded as computed columns.
+Pixeltable is a Python library providing a declarative interface for multimodal data (text, images, audio, video). It features built-in versioning, lineage tracking, and incremental updates, enabling users to **store**, **transform**, **index**, and **iterate** on data for their ML workflows.
+
+Data transformations, model inference, and custom logic are embedded as **computed columns**.
+- **Load/Query all data types**: Interact with [video data](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#import-media-data-into-pixeltable-videos-images-audio) at the [frame level](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#text-and-image-similarity-search-on-video-frames-with-embedding-indexes) and documents at the [chunk level](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#automate-data-operations-with-views-eg-split-documents-into-chunks)
+- **Incremental updates for data transformation**: Maintain an [embedding index](https://docs.pixeltable.com/docs/embedding-vector-indexes) colocated with your data
+- **Lazy evaluation and cache management**: Eliminates the need for [manual frame extraction](https://docs.pixeltable.com/docs/object-detection-in-videos)
+- **Integrates with any Python libraries**: Use [built-in and custom functions (UDFs)](https://docs.pixeltable.com/docs/user-defined-functions-udfs) without complex pipelines
+- **Data format agnostic and extensibility**: Access tables as Parquet files, [PyTorch datasets](https://pixeltable.github.io/pixeltable/api/data-frame/#pixeltable.DataFrame.to_pytorch_dataset), or [COCO annotations](https://pixeltable.github.io/pixeltable/api/table/#pixeltable.Table.to_coco_dataset)
 
 ## 💾 Installation
 

{pixeltable-0.2.19.dist-info → pixeltable-0.2.20.dist-info}/RECORD

@@ -1,26 +1,26 @@
 pixeltable/__init__.py,sha256=t1uRHKta7mPH9_KgkUpOWBu6AewA7DRdSGGyrm0OcSQ,1279
-pixeltable/__version__.py,sha256=_o-1q05Ttbs4cHNYgDuwFGf4i38fZkI3gdZZEbf8-co,114
+pixeltable/__version__.py,sha256=RrlUQ8lgLgO05DWPZnMK9eYJ0O2bfE8N7RBzpECP5f8,114
 pixeltable/catalog/__init__.py,sha256=E41bxaPeQIcgRYzTWc2vkDOboQhRymrJf4IcHQO7o_8,453
 pixeltable/catalog/catalog.py,sha256=tyDyI5wQw7vV6_FChrp9qgGCRClcjiSdW3eygYT0p9s,7849
 pixeltable/catalog/column.py,sha256=Be3WmOadMROS2s4IgtG_Ohjkr07eU9GJItl6WhNishQ,9683
 pixeltable/catalog/dir.py,sha256=fG_BQM-fLuABpTstMVH-9dvZPx7kqi3sgTQgKveVXJI,922
 pixeltable/catalog/globals.py,sha256=XeOeDqq1nDEcpqkY7PYBosoL6tXVAfkJSLJN9aQ_9Fg,1850
-pixeltable/catalog/insertable_table.py,sha256=XpA87IoYtEe1xVbbahYRowhGq3fHHc4M9GpIViggNCU,6724
+pixeltable/catalog/insertable_table.py,sha256=DFL93x8ihYEnK_yCR8EdYHDQqAomJdU11ygTF0jEFWY,6822
 pixeltable/catalog/named_function.py,sha256=W8vikP_3jMJ9pQQsksO2EfQAlaVxuQHBlo65M4924dc,1150
 pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
 pixeltable/catalog/path_dict.py,sha256=4b9_Ax7Q8tkmoCYPaKNedpQkU17pE0oGDd2XB53eNZA,5979
 pixeltable/catalog/schema_object.py,sha256=qhpeeUPOYT5doDbsyUNBcPm5QzAQPCAsikqh1PQ6d1k,2226
-pixeltable/catalog/table.py,sha256=eVGCXZxtEaG8Y4IEe-8IgPtIFAaUsycJqMdVE_WpKPA,40811
+pixeltable/catalog/table.py,sha256=NQMZwG6wPu8DzJmZLXTkDm_Dth0AmNXhcixNqiXlPuc,41307
 pixeltable/catalog/table_version.py,sha256=4_djeYLGu9ljRSXe_f14c3HvXL0o0P2-sOZ-1bBQzYw,56991
 pixeltable/catalog/table_version_path.py,sha256=Ee6nPh5Jgbp91qFSKkCwdzIpQ3gJqv3SG06bFFLhbBE,6139
 pixeltable/catalog/view.py,sha256=RfQRldjPUZ7W8jMMdXJFSjbjCUe-3ynxDFvg4W27qXc,10642
 pixeltable/dataframe.py,sha256=kAPv9YjOEx0xZViFG3fi6eXsX6zUhm3F2x5U7qDOrJU,34378
-pixeltable/env.py,sha256=fAxc0vXRN_OkooqepIbHQJPyoY2aQwWmfxYmFCq-8pc,27374
-pixeltable/exceptions.py,sha256=pgMRe11dqUdSWaTssib4bXj9R_HNqwhdqKSnMsRkSvE,422
+pixeltable/env.py,sha256=XHxv2P5Aj1dvUxwlvfoxahVemhJFyapmW7p3pf8Vq7g,30133
+pixeltable/exceptions.py,sha256=NuFY2WtkQpLfLHT_J70kOw9Tr0kEDkkgo-u7As4Gaq4,410
 pixeltable/exec/__init__.py,sha256=VRENEONsAv3PPoBV0r7h-7nAB7SWM4Uglmu1FVQE5uQ,507
 pixeltable/exec/aggregation_node.py,sha256=-DunTLlVh3OflpwTIjkwKGczotl4i3oUqrvfyvRjv6Q,3452
 pixeltable/exec/cache_prefetch_node.py,sha256=d5pEuR6AtJQkEVy9X3XeYFI_q0szMtoNAH96vYdtBE0,5241
-pixeltable/exec/component_iteration_node.py,sha256=d5XHahyq-cosYW6BaIeFLuU8zNRmDL8eGtiN_eYY8ZM,4068
+pixeltable/exec/component_iteration_node.py,sha256=ABuXGbDRQWLGuaBnfK7bvOxCrz81vMMiAvXHHI8SX4c,4930
 pixeltable/exec/data_row_batch.py,sha256=1IDYHBkSQ60dwOnAGnS-Wpp3AsnbMqKcY40zUT7ku-Q,3392
 pixeltable/exec/exec_context.py,sha256=0rg5V8HzSy-BvqmSbGr-U4aJ4eOZg2JN0x6zjYQGtBc,1090
 pixeltable/exec/exec_node.py,sha256=ixkv3p_EfF53UDWgwLjQGKR1LNIQxzgDXsTzzJj6ea4,2211
@@ -45,7 +45,7 @@ pixeltable/exprs/in_predicate.py,sha256=vJwT07SlDXBYMbqpf-dgV2gr6je5DehrpkPBapnZ
 pixeltable/exprs/inline_expr.py,sha256=FIQsgwfz-9qmghnaTSTL3522Mhr9GQUKM_SDxzA4P5w,7055
 pixeltable/exprs/is_null.py,sha256=qkzxr0NPuID77gs-J_tXj0MYuoCPBEd3Iq6MUWJ_dSc,1101
 pixeltable/exprs/json_mapper.py,sha256=grr-9xVOU_TUL1wtON7wNqZ10-p3mGp66cTCofQKkqc,4590
-pixeltable/exprs/json_path.py,sha256=cdAA73af_ZK2hZAzROA9fpP8dTBrHYzcTi4APgtJNdk,7051
+pixeltable/exprs/json_path.py,sha256=xlwUeYL8D--dPTMhzoyCtkQVeik0sfwI7k_XlNs0eS4,6912
 pixeltable/exprs/literal.py,sha256=ofhMe2kiT4tWNuzf2zKOiGY5pml10dRqbV0e9HGVcbs,3780
 pixeltable/exprs/method_ref.py,sha256=6TQnl5JhsUqKNPFUbu2tzu5svF_BZf5rfm2cZo740Ts,2600
 pixeltable/exprs/object_ref.py,sha256=UDLfpFXrOTrYZOVWH6G5dx4Ax_BxFTpLOaIab3MuyyI,1282
@@ -81,26 +81,26 @@ pixeltable/functions/mistralai.py,sha256=U7f6g4EyHMsik8HMIdJIKn6xFSCdQH6950AAOYL
 pixeltable/functions/openai.py,sha256=yr2hgUa0ZtUJOezSC9aVqp-BoxADf-gmYoK8FE2jbVU,15930
 pixeltable/functions/string.py,sha256=RCGj5bXx7MWgcdcOuy1IMTn3vBvGzjgxudyUrDqWdAg,20153
 pixeltable/functions/timestamp.py,sha256=lyWPv2sCpejD2t9DB62nxJEm0kWLNsAW8yMiT5iEsOo,9121
-pixeltable/functions/together.py,sha256=h2FqZomgLhTkpwJ-t1mFgbBrFPbNzKE4CLlqTEc-qwY,8947
+pixeltable/functions/together.py,sha256=pmd_Xo9XaJ9M8-Zx1bDb4pnomHGZ5swBENHYx-uhmPs,9480
 pixeltable/functions/util.py,sha256=F2iiIL7UfhYdCVzdCa3efYqWbaeLKFrbycKnuPkG57M,650
 pixeltable/functions/video.py,sha256=qaPkeU4qO_g_lQhiMcytAOiJbwtfO89amGVxsT86MZQ,7180
 pixeltable/functions/vision.py,sha256=K_E1Q-n2plPuFoOPlbKWRMiJp9dPgftIJ2T_o3TNL3I,15594
 pixeltable/functions/whisper.py,sha256=VvGVWEsANHH2oCabT1bFTXoDEn5g90gQT_PCh56W4n4,3377
-pixeltable/globals.py,sha256=Jc9XgtCUTjM7O2EJLnrDXkjQL--ROSMlRl5MBP7ndC8,16661
+pixeltable/globals.py,sha256=dbLCAuobQAJgjlTASp9bGRLwOYEyBntKLl3-GP7GTgU,16755
 pixeltable/index/__init__.py,sha256=XBwetNQQwnz0fiKwonOKhyy_U32l_cjt77kNvEIdjWs,102
 pixeltable/index/base.py,sha256=YAQ5Dz1mfI0dfu9rxWHWroE8TjB90yKfPtXAzoADq38,1568
 pixeltable/index/btree.py,sha256=NE4GYhcJWYJhdKyeHI0sQBlFvUaIgGOF9KLyCZOfFjE,1822
 pixeltable/index/embedding_index.py,sha256=U1wAjcTYvw3uJf3QHIOzBV8FLOUn8IeaFsLzUb_QTmc,7829
 pixeltable/io/__init__.py,sha256=bJGWPhKfgoMrSKFdXhLGupjQQbIXt7JaoUPwilND2PE,519
 pixeltable/io/external_store.py,sha256=iRqvMx9QuCKmOKBe12hoY1KfXyGvDHL-q1CjaZr3Fkk,16466
-pixeltable/io/globals.py,sha256=3kgeSFX6kFXGFMs9GpmLlU1sY9aZrfddMhVrQEOlJIs,13330
-pixeltable/io/hf_datasets.py,sha256=mVZYvjh7DVCmyKSQNnsJ8oLspSQw2WRnSm65F_pzf-A,8255
+pixeltable/io/globals.py,sha256=ZmjbLy9EMhJjXKeNVgPhsi1dmllwJ1rsHu1XHadHtgM,13330
+pixeltable/io/hf_datasets.py,sha256=E5E2yfaHo9Hf9gFI9ZhzaztHtRC_xDL6mIyeeond2Uo,8284
 pixeltable/io/label_studio.py,sha256=m1-ayI7S8Lxv2R1agrO-32xXyB8Z-YPP_ErAqu22c7o,31023
 pixeltable/io/pandas.py,sha256=7eHg7wnAfRA9eBk4iC0iSSVTKOM59Ne4pXokKWdt3dY,9793
 pixeltable/io/parquet.py,sha256=bUBJmnTFrlBZ8yIesqUJ1JufXZ76pm7vQ3Fq48hVijA,7853
 pixeltable/iterators/__init__.py,sha256=sjldFckkT8aVRiKgEP6faeAK2NQBdzbmpwAeRhI1FkM,366
 pixeltable/iterators/base.py,sha256=cnEh1tNN2JAxRzrLTg3dhun3N1oNQ8vifCm6ts3_UiE,1687
-pixeltable/iterators/document.py,sha256=dz7bfukocBS_2xBTD4gBS8EMaUP5fnkmQL-kWGxqdTA,19454
+pixeltable/iterators/document.py,sha256=dAJjCRY0HUxrdMlGjf19ZLChARmWonYoJ0QvgfxkQyQ,19455
 pixeltable/iterators/string.py,sha256=NG_fWc_GAITDfzl6MvrDOMrSoMcZdMZf6hPQztCSatE,1305
 pixeltable/iterators/video.py,sha256=Glp7qNjyrH8X5S4WJOEsZhCa4yChalTICiR9bbMsHlo,5734
 pixeltable/metadata/__init__.py,sha256=8mYxCsc_uvN3tqwrmIbB9iBkQ9r9ybsdpFCMsrzNaNw,2172
@@ -119,7 +119,7 @@ pixeltable/metadata/notes.py,sha256=1Hk6TGy69a4jgqqLoaUlQPtzANMvMGkifKC5rjqeOeA,
 pixeltable/metadata/schema.py,sha256=H2NjpNBxZNDw_VV3UK97fKs30dh81uQf8F3vexKeePo,8567
 pixeltable/plan.py,sha256=pHTJxv2WzsDXtnBd9RvXtUnEFvIQjXV7NX_BIQRQiHs,38544
 pixeltable/store.py,sha256=zlVG9rs5k0k8wcfYF2jcgAQgIOfanJ9YjIDs_kacRIQ,21106
-pixeltable/tool/create_test_db_dump.py,sha256=YpVGwe6mn22p1sEGbXcEpOyF0RQwKgd3gCRA0Wl2phY,11980
+pixeltable/tool/create_test_db_dump.py,sha256=y4LotPVbcQeqnarpISmVPWoURBVnjKjSl9Yi2MmCZE0,11980
 pixeltable/tool/create_test_video.py,sha256=OLfccymYReIpzE8osZn4rQvLXxxiPC_l0vc06U74hVM,2899
 pixeltable/tool/doc_plugins/griffe.py,sha256=Q6ARBlQNBm8J21G_p625TB5c8MQ8r6hJlm7I2LoBon0,3422
 pixeltable/tool/doc_plugins/mkdocstrings.py,sha256=afq7XOaSC5WRmugkh-FMFMK8PqOgIlDIsJdD8cuPhtE,207
@@ -131,7 +131,7 @@ pixeltable/utils/arrow.py,sha256=83_7aG5UR2qtTktw_otLkQs-RQbLk0VVM0JLJkbweNU,369
 pixeltable/utils/coco.py,sha256=ISpFBhR4eO1jOcg_SPb0thVI4KdS6H0RyNQauZIA5A4,7287
 pixeltable/utils/code.py,sha256=AOw1u2r8_DQXpX-lxJhyHWARGrCRDXOJHFVgKOi54Uc,1231
 pixeltable/utils/documents.py,sha256=UQq2F-W4stDuldFDSGHwUe5PK1dPoalN8SfYRoGqd14,2038
-pixeltable/utils/filecache.py,sha256=UoNONG2VaAc2IBB0e3sQdsvyOPOes2XSDc5_CsA4qek,7839
+pixeltable/utils/filecache.py,sha256=hQOSz5VmC2MBk0F4RaZKgG1OQFFXOyFfanp-cQMOsU4,10553
 pixeltable/utils/formatter.py,sha256=XOuNAhZKCvA9Dlj1QYHB_ovwWUuznvvvdkWgjl4bWq0,9239
 pixeltable/utils/help.py,sha256=cCnxJ4VP9MJ57iDqExmnDcM-JG3a1lw_q7g-D7bpSVI,252
 pixeltable/utils/http_server.py,sha256=WQ5ILMzlz4TlwI9j5YqAPgEZyhrN1GytMNDbLD9occk,2422
@@ -140,8 +140,8 @@ pixeltable/utils/pytorch.py,sha256=VWczSB_FT_aOU5Xqv4T5ONTsnQN6KDlZmMkuoBuji08,3
 pixeltable/utils/s3.py,sha256=DBfXp0SYubhiKckdAD7PsiVBX_YfVP8Rcu6DCG_3SaQ,433
 pixeltable/utils/sql.py,sha256=5n5_OmXAGtqFdL6z5XvgnU-vlx6Ba6f1WJrO1ZwUle8,765
 pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
-pixeltable-0.2.19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pixeltable-0.2.19.dist-info/METADATA,sha256=elg8Pycf1lqUyJVyCbPHjWNXQqHqCKei_wRlKSI6A_o,13538
-pixeltable-0.2.19.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pixeltable-0.2.19.dist-info/entry_points.txt,sha256=TNI1Gb5vPwFrTdw6TimSYjO8FeK8c_HuPr28vcf7o_I,108
-pixeltable-0.2.19.dist-info/RECORD,,
+pixeltable-0.2.20.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pixeltable-0.2.20.dist-info/METADATA,sha256=QYhIK4U4RMLo_B3lTevJoPXFTM3hP8qfqO1A89R9Qjo,14972
+pixeltable-0.2.20.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pixeltable-0.2.20.dist-info/entry_points.txt,sha256=TNI1Gb5vPwFrTdw6TimSYjO8FeK8c_HuPr28vcf7o_I,108
+pixeltable-0.2.20.dist-info/RECORD,,