pixeltable 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

pixeltable/__init__.py

@@ -4,7 +4,7 @@ from .exceptions import Error
 from .exprs import RELATIVE_PATH_ROOT
 from .func import Function, udf, Aggregator, uda, expr_udf
 from .globals import init, create_table, create_view, get_table, move, drop_table, list_tables, create_dir, drop_dir, \
-    list_dirs, list_functions, configure_logging
+    list_dirs, list_functions, configure_logging, array
 from .type_system import (
     ColumnType,
     StringType,

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.18"
-__version_tuple__ = (0, 2, 18)
+__version__ = "0.2.20"
+__version_tuple__ = (0, 2, 20)

pixeltable/catalog/insertable_table.py

@@ -10,6 +10,7 @@ import pixeltable as pxt
 import pixeltable.type_system as ts
 from pixeltable import exceptions as excs
 from pixeltable.env import Env
+from pixeltable.utils.filecache import FileCache
 
 from .catalog import Catalog
 from .globals import UpdateStatus
@@ -101,21 +102,22 @@ class InsertableTable(Table):
             if not isinstance(row, dict):
                 raise excs.Error('rows must be a list of dictionaries')
         self._validate_input_rows(rows)
-        result = self._tbl_version.insert(rows, None, print_stats=print_stats, fail_on_exception=fail_on_exception)
+        status = self._tbl_version.insert(rows, None, print_stats=print_stats, fail_on_exception=fail_on_exception)
 
-        if result.num_excs == 0:
+        if status.num_excs == 0:
             cols_with_excs_str = ''
         else:
             cols_with_excs_str = \
-                f' across {len(result.cols_with_excs)} column{"" if len(result.cols_with_excs) == 1 else "s"}'
-            cols_with_excs_str += f' ({", ".join(result.cols_with_excs)})'
+                f' across {len(status.cols_with_excs)} column{"" if len(status.cols_with_excs) == 1 else "s"}'
+            cols_with_excs_str += f' ({", ".join(status.cols_with_excs)})'
         msg = (
-            f'Inserted {result.num_rows} row{"" if result.num_rows == 1 else "s"} '
-            f'with {result.num_excs} error{"" if result.num_excs == 1 else "s"}{cols_with_excs_str}.'
+            f'Inserted {status.num_rows} row{"" if status.num_rows == 1 else "s"} '
+            f'with {status.num_excs} error{"" if status.num_excs == 1 else "s"}{cols_with_excs_str}.'
         )
         print(msg)
         _logger.info(f'InsertableTable {self._name}: {msg}')
-        return result
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def _validate_input_rows(self, rows: List[Dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""

pixeltable/catalog/table.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import abc
 import builtins
-import itertools
 import json
 import logging
 from pathlib import Path
@@ -21,6 +20,7 @@ import pixeltable.exprs as exprs
 import pixeltable.index as index
 import pixeltable.metadata.schema as schema
 import pixeltable.type_system as ts
+from pixeltable.utils.filecache import FileCache
 
 from .column import Column
 from .globals import _ROWID_COLUMN_NAME, UpdateStatus, is_system_column_name, is_valid_identifier
@@ -34,7 +34,12 @@ if TYPE_CHECKING:
 _logger = logging.getLogger('pixeltable')
 
 class Table(SchemaObject):
-    """Base class for table objects (base tables, views, snapshots)."""
+    """
+    Base class for table objects (base tables, views, snapshots).
+
+    Every user-invoked operation that runs an ExecNode tree (directly or indirectly) needs to call
+    FileCache.emit_eviction_warnings() at the end of the operation.
+    """
 
     def __init__(self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath):
         super().__init__(id, name, dir_id)
@@ -375,7 +380,10 @@ class Table(SchemaObject):
 
         new_col = self._create_columns({col_name: col_schema})[0]
         self._verify_column(new_col, set(self._schema.keys()), set(self._query_names))
-        return self._tbl_version.add_column(new_col, print_stats=print_stats)
+        status = self._tbl_version.add_column(new_col, print_stats=print_stats)
+        FileCache.get().emit_eviction_warnings()
+        return status
+
 
     @classmethod
     def _validate_column_spec(cls, name: str, spec: dict[str, Any]) -> None:
@@ -588,6 +596,7 @@ class Table(SchemaObject):
         idx = EmbeddingIndex(col, metric=metric, string_embed=string_embed, image_embed=image_embed)
         status = self._tbl_version.add_index(col, idx_name=idx_name, idx=idx)
         # TODO: how to deal with exceptions here? drop the index and raise?
+        FileCache.get().emit_eviction_warnings()
 
     def drop_embedding_index(self, *, column_name: Optional[str] = None, idx_name: Optional[str] = None) -> None:
         """Drop an embedding index from the table.
@@ -733,7 +742,9 @@ class Table(SchemaObject):
             >>> tbl.update({'int_col': tbl.int_col + 1}, where=tbl.int_col == 0)
         """
         self._check_is_dropped()
-        return self._tbl_version.update(value_spec, where, cascade)
+        status = self._tbl_version.update(value_spec, where, cascade)
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def batch_update(
             self, rows: Iterable[dict[str, Any]], cascade: bool = True,
@@ -790,9 +801,11 @@ class Table(SchemaObject):
                     missing_cols = pk_col_names - set(col.name for col in col_vals.keys())
                     raise excs.Error(f'Primary key columns ({", ".join(missing_cols)}) missing in {row_spec}')
             row_updates.append(col_vals)
-        return self._tbl_version.batch_update(
+        status = self._tbl_version.batch_update(
             row_updates, rowids, error_if_not_exists=if_not_exists == 'error',
             insert_if_not_exists=if_not_exists == 'insert', cascade=cascade)
+        FileCache.get().emit_eviction_warnings()
+        return status
 
     def delete(self, where: Optional['pixeltable.exprs.Expr'] = None) -> UpdateStatus:
         """Delete rows in this table.

pixeltable/catalog/table_version.py

@@ -147,7 +147,7 @@ class TableVersion:
             module = importlib.import_module(module_name)
             self.iterator_cls = getattr(module, class_name)
             self.iterator_args = exprs.InlineDict.from_dict(tbl_md.view_md.iterator_args)
-            output_schema, _ = self.iterator_cls.output_schema(**self.iterator_args.to_dict())
+            output_schema, _ = self.iterator_cls.output_schema(**self.iterator_args.to_kwargs())
             self.num_iterator_cols = len(output_schema)
             assert tbl_md.view_md.iterator_args is not None
 

pixeltable/catalog/view.py

@@ -94,7 +94,7 @@ class View(Table):
                 ]
                 sig = func.Signature(InvalidType(), params)
                 from pixeltable.exprs import FunctionCall
-                FunctionCall.normalize_args(sig, bound_args)
+                FunctionCall.normalize_args(iterator_cls.__name__, sig, bound_args)
             except TypeError as e:
                 raise Error(f'Cannot instantiate iterator with given arguments: {e}')
 

pixeltable/dataframe.py

@@ -501,7 +501,7 @@ class DataFrame:
             elif isinstance(raw_expr, dict):
                 select_list.append((exprs.InlineDict(raw_expr), name))
             elif isinstance(raw_expr, list):
-                select_list.append((exprs.InlineArray(raw_expr), name))
+                select_list.append((exprs.InlineList(raw_expr), name))
             else:
                 select_list.append((exprs.Literal(raw_expr), name))
             expr = select_list[-1][0]

pixeltable/env.py

@@ -8,18 +8,20 @@ import importlib.util
 import inspect
 import logging
 import os
+import shutil
+import subprocess
 import sys
 import threading
 import uuid
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import pixeltable_pgserver
 import sqlalchemy as sql
-import yaml
+import toml
 from tqdm import TqdmWarning
 
 import pixeltable.exceptions as excs
@@ -63,7 +65,7 @@ class Env:
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
     _config_file: Optional[Path]
-    _config: Optional[dict[str, Any]]
+    _config: Optional[Config]
     _stdout_handler: logging.StreamHandler
     _initialized: bool
 
@@ -109,6 +111,7 @@ class Env:
         self._log_to_stdout = False
         self._module_log_level = {}  # module name -> log level
 
+        # config
         self._config_file = None
         self._config = None
 
@@ -118,7 +121,8 @@ class Env:
         self._initialized = False
 
     @property
-    def config(self):
+    def config(self) -> Config:
+        assert self._config is not None
         return self._config
 
     @property
@@ -226,30 +230,13 @@ class Env:
         home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
         assert self._home is None or self._home == home
         self._home = home
-        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.yaml')))
+        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.toml')))
         self._media_dir = self._home / 'media'
         self._file_cache_dir = self._home / 'file_cache'
         self._dataset_cache_dir = self._home / 'dataset_cache'
         self._log_dir = self._home / 'logs'
         self._tmp_dir = self._home / 'tmp'
 
-        # Read in the config
-        if os.path.isfile(self._config_file):
-            with open(self._config_file, 'r') as stream:
-                try:
-                    self._config = yaml.safe_load(stream)
-                except yaml.YAMLError as exc:
-                    self._logger.error(f'Could not read config file: {self._config_file}')
-                    self._config = {}
-        else:
-            self._config = {}
-
-        # Disable spurious warnings
-        warnings.simplefilter('ignore', category=TqdmWarning)
-        if 'hide_warnings' in self._config and self._config['hide_warnings']:
-            # Disable more warnings
-            warnings.simplefilter('ignore', category=UserWarning)
-
         if self._home.exists() and not self._home.is_dir():
             raise RuntimeError(f'{self._home} is not a directory')
 
@@ -273,6 +260,22 @@ class Env:
         if not self._tmp_dir.exists():
             self._tmp_dir.mkdir()
 
+        # Read in the config
+        self._config = Config.from_file(self._config_file)
+        self._file_cache_size_g = self._config.get_float_value('file_cache_size_g')
+        if self._file_cache_size_g is None:
+            raise excs.Error(
+                'pixeltable/file_cache_size_g is missing from configuration\n'
+                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {self._config_file},\n'
+                'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
+            )
+
+        # Disable spurious warnings
+        warnings.simplefilter('ignore', category=TqdmWarning)
+        if self._config.get_bool_value('hide_warnings'):
+            # Disable more warnings
+            warnings.simplefilter('ignore', category=UserWarning)
+
         # configure _logger to log to a file
         self._logfilename = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + '.log'
         fh = logging.FileHandler(self._log_dir / self._logfilename, mode='w')
@@ -312,7 +315,7 @@ class Env:
         self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
         self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
 
-        tz_name = os.environ.get('PXT_TIME_ZONE', self._config.get('pxt_time_zone', None))
+        tz_name = self.config.get_string_value('time_zone')
         if tz_name is not None:
             # Validate tzname
             if not isinstance(tz_name, str):
@@ -439,21 +442,18 @@ class Env:
         if cl.client_obj is not None:
             return cl.client_obj  # Already initialized
 
-        # Construct a client. For each client parameter, first check if the parameter is in the environment;
-        # if not, look in Pixeltable config from `config.yaml`.
+        # Construct a client, retrieving each parameter from config.
 
         init_kwargs: dict[str, str] = {}
         for param in cl.param_names:
-            environ = f'{name.upper()}_{param.upper()}'
-            if environ in os.environ:
-                init_kwargs[param] = os.environ[environ]
-            elif name.lower() in self._config and param in self._config[name.lower()]:
-                init_kwargs[param] = self._config[name.lower()][param.lower()]
-            if param not in init_kwargs or init_kwargs[param] == '':
+            arg = self._config.get_string_value(param, section=name)
+            if arg is not None and len(arg) > 0:
+                init_kwargs[param] = arg
+            else:
                 raise excs.Error(
                     f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{environ}` environment variable, or put `{param.lower()}` in '
-                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.yaml.'
+                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, or put `{param.lower()}` in '
+                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
         cl.client_obj = cl.init_fn(**init_kwargs)
@@ -486,6 +486,8 @@ class Env:
         """Check for and start runtime services"""
         self._start_web_server()
         self.__register_packages()
+        if self.is_installed_package('spacy'):
+            self.__init_spacy()
 
     def __register_packages(self) -> None:
         """Declare optional packages that are utilized by some parts of the code."""
@@ -500,10 +502,9 @@ class Env:
         self.__register_package('openpyxl')
         self.__register_package('pyarrow')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
-        self.__register_package('spacy')  # TODO: deal with en-core-web-sm
+        self.__register_package('spacy')
         self.__register_package('tiktoken')
         self.__register_package('together')
-        self.__register_package('toml')
         self.__register_package('torch')
         self.__register_package('torchvision')
         self.__register_package('transformers')
@@ -511,10 +512,6 @@ class Env:
         self.__register_package('whisperx')
         self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
 
-        if self.is_installed_package('spacy'):
-            import spacy
-            self._spacy_nlp = spacy.load('en_core_web_sm')
-
     def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
         self.__optional_packages[package_name] = PackageInfo(
             is_installed=importlib.util.find_spec(package_name) is not None,
@@ -556,6 +553,35 @@ class Env:
                 f'To fix this, run: `pip install -U {package_info.library_name}`'
             )
 
+    def __init_spacy(self) -> None:
+        """
+        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
+        dependency, we install it programmatically here. This should cause no problems, since the model packages
+        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
+        """
+        import spacy
+        from spacy.cli.download import get_model_filename
+        spacy_model = 'en_core_web_sm'
+        spacy_model_version = '3.7.1'
+        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
+        url = f'{spacy.about.__download_url__}/{filename}'
+        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
+        # a problem, because the model have been installed on a previous attempt.
+        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
+        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
+        if ret.returncode != 0:
+            self._logger.warn(f'pip install failed for spaCy model: {filename}')
+        try:
+            self._logger.info(f'Loading spaCy model: {spacy_model}')
+            self._spacy_nlp = spacy.load(spacy_model)
+        except Exception as exc:
+            self._logger.warn(f'Failed to load spaCy model: {spacy_model}', exc_info=exc)
+            warnings.warn(
+                f"Failed to load spaCy model '{spacy_model}'. spaCy features will not be available.",
+                excs.PixeltableWarning
+            )
+            self.__optional_packages['spacy'].is_installed = False
+
     def num_tmp_files(self) -> int:
         return len(glob.glob(f'{self._tmp_dir}/*'))
 
@@ -594,6 +620,7 @@ class Env:
 
     @property
     def spacy_nlp(self) -> spacy.Language:
+        Env.get().require_package('spacy')
         assert self._spacy_nlp is not None
         return self._spacy_nlp
 
@@ -614,7 +641,7 @@ def register_client(name: str) -> Callable:
     Pixeltable will attempt to load the client parameters from config. For each
     config parameter:
     - If an environment variable named MY_CLIENT_API_KEY (for example) is set, use it;
-    - Otherwise, look for 'api_key' in the 'my_client' section of config.yaml.
+    - Otherwise, look for 'api_key' in the 'my_client' section of config.toml.
 
     If all config parameters are found, Pixeltable calls the initialization function;
     otherwise it throws an exception.
@@ -631,6 +658,79 @@ def register_client(name: str) -> Callable:
     return decorator
 
 
+class Config:
+    """
+    The (global) Pixeltable configuration, as loaded from `config.toml`. Provides methods for retrieving
+    configuration values, which can be set in the config file or as environment variables.
+    """
+    __config: dict[str, Any]
+
+    T = TypeVar('T')
+
+    @classmethod
+    def from_file(cls, path: Path) -> Config:
+        """
+        Loads configuration from the specified TOML file. If the file does not exist, it will be
+        created and populated with the default configuration.
+        """
+        if os.path.isfile(path):
+            with open(path, 'r') as stream:
+                try:
+                    config_dict = toml.load(stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not read config file: {str(path)}') from exc
+        else:
+            config_dict = cls.__create_default_config(path)
+            with open(path, 'w') as stream:
+                try:
+                    toml.dump(config_dict, stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not write config file: {str(path)}') from exc
+            logging.getLogger('pixeltable').info(f'Created default config file at: {str(path)}')
+        return cls(config_dict)
+
+    @classmethod
+    def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
+        free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
+        # Default cache size is 1/5 of free disk space
+        file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
+        return {
+            'pixeltable': {
+                'file_cache_size_g': round(file_cache_size_g, 1),
+                'hide_warnings': False,
+            }
+        }
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self.__config = config
+
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+        env_var = f'{section.upper()}_{key.upper()}'
+        if env_var in os.environ:
+            value = os.environ[env_var]
+        elif section in self.__config and key in self.__config[section]:
+            value = self.__config[section][key]
+        else:
+            return None
+
+        try:
+            return expected_type(value)  # type: ignore[call-arg]
+        except ValueError:
+            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
+
+    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
+        return self.get_value(key, str, section)
+
+    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
+        return self.get_value(key, int, section)
+
+    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
+        return self.get_value(key, float, section)
+
+    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
+        return self.get_value(key, bool, section)
+
+
 _registered_clients: dict[str, ApiClient] = {}
 
 

pixeltable/exceptions.py

@@ -1,6 +1,9 @@
-from typing import List, Any
-from types import TracebackType
 from dataclasses import dataclass
+from types import TracebackType
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pixeltable import exprs
 
 
 class Error(Exception):
@@ -9,9 +12,13 @@ class Error(Exception):
 
 @dataclass
 class ExprEvalError(Exception):
-    expr: Any  # exprs.Expr, but we're not importing pixeltable.exprs to avoid circular imports
+    expr: 'exprs.Expr'
     expr_msg: str
     exc: Exception
     exc_tb: TracebackType
-    input_vals: List[Any]
-    row_num: int
+    input_vals: list[Any]
+    row_num: int
+
+
+class PixeltableWarning(Warning):
+    pass

pixeltable/exec/component_iteration_node.py

@@ -1,10 +1,12 @@
-from typing import Generator, Optional
+import inspect
+from typing import Iterator, Optional
 
-from .data_row_batch import DataRowBatch
-from .exec_node import ExecNode
 import pixeltable.catalog as catalog
-import pixeltable.exprs as exprs
 import pixeltable.exceptions as excs
+import pixeltable.exprs as exprs
+
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
 
 
 class ComponentIterationNode(ExecNode):
@@ -12,7 +14,7 @@ class ComponentIterationNode(ExecNode):
 
     Returns row batches of OUTPUT_BATCH_SIZE size.
     """
-    OUTPUT_BATCH_SIZE = 1024
+    __OUTPUT_BATCH_SIZE = 1024
 
     def __init__(self, view: catalog.TableVersion, input: ExecNode):
         assert view.is_component_view()
@@ -23,57 +25,76 @@ class ComponentIterationNode(ExecNode):
         self.iterator_args = iterator_args[0]
         assert isinstance(self.iterator_args, exprs.InlineDict)
         self.iterator_args_ctx = self.row_builder.create_eval_ctx([self.iterator_args])
-        self.iterator_output_schema, self.unstored_column_names = \
-            self.view.iterator_cls.output_schema(**self.iterator_args.to_dict())
+        self.iterator_output_schema, self.unstored_column_names = (
+            self.view.iterator_cls.output_schema(**self.iterator_args.to_kwargs())
+        )
         self.iterator_output_fields = list(self.iterator_output_schema.keys())
-        self.iterator_output_cols = \
-            {field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields}
+        self.iterator_output_cols = {
+            field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields
+        }
         # referenced iterator output fields
         self.refd_output_slot_idxs = {
             e.col.name: e.slot_idx for e in self.row_builder.unique_exprs
             if isinstance(e, exprs.ColumnRef) and e.col.name in self.iterator_output_fields
         }
-        self._output: Optional[Generator[DataRowBatch, None, None]] = None
+        self.__output: Optional[Iterator[DataRowBatch]] = None
 
-    def _output_batches(self) -> Generator[DataRowBatch, None, None]:
+    def __output_batches(self) -> Iterator[DataRowBatch]:
         output_batch = DataRowBatch(self.view, self.row_builder)
         for input_batch in self.input:
             for input_row in input_batch:
                 self.row_builder.eval(input_row, self.iterator_args_ctx)
                 iterator_args = input_row[self.iterator_args.slot_idx]
-                iterator = self.view.iterator_cls(**iterator_args)
-                for pos, component_dict in enumerate(iterator):
-                    output_row = output_batch.add_row()
-                    input_row.copy(output_row)
-                    # we're expanding the input and need to add the iterator position to the pk
-                    pk = output_row.pk[:-1] + (pos,) + output_row.pk[-1:]
-                    output_row.set_pk(pk)
-
-                    # verify and copy component_dict fields to their respective slots in output_row
-                    for field_name, field_val in component_dict.items():
-                        if field_name not in self.iterator_output_fields:
-                            raise excs.Error(
-                                f'Invalid field name {field_name} in output of {self.view.iterator_cls.__name__}')
-                        if field_name not in self.refd_output_slot_idxs:
-                            # we can ignore this
-                            continue
-                        output_col = self.iterator_output_cols[field_name]
-                        output_col.col_type.validate_literal(field_val)
-                        output_row[self.refd_output_slot_idxs[field_name]] = field_val
-                    if len(component_dict) != len(self.iterator_output_fields):
-                        missing_fields = set(self.refd_output_slot_idxs.keys()) - set(component_dict.keys())
-                        raise excs.Error(
-                            f'Invalid output of {self.view.iterator_cls.__name__}: '
-                            f'missing fields {", ".join(missing_fields)}')
-
-                    if len(output_batch) == self.OUTPUT_BATCH_SIZE:
-                        yield output_batch
-                        output_batch = DataRowBatch(self.view, self.row_builder)
+                assert isinstance(iterator_args, dict)
+                # We need to ensure that all of the required (non-nullable) parameters of the iterator are
+                # specified and are not null. If any of them are null, then we skip this row (i.e., we emit 0
+                # output rows for this input row).
+                if self.__non_nullable_args_specified(iterator_args):
+                    iterator = self.view.iterator_cls(**iterator_args)
+                    for pos, component_dict in enumerate(iterator):
+                        output_row = output_batch.add_row()
+                        input_row.copy(output_row)
+                        # we're expanding the input and need to add the iterator position to the pk
+                        self.__populate_output_row(output_row, pos, component_dict)
+                        if len(output_batch) == self.__OUTPUT_BATCH_SIZE:
+                            yield output_batch
+                            output_batch = DataRowBatch(self.view, self.row_builder)
 
         if len(output_batch) > 0:
             yield output_batch
 
+    def __non_nullable_args_specified(self, iterator_args: dict) -> bool:
+        """
+        Returns true if all non-nullable iterator arguments are not `None`.
+        """
+        input_schema = self.view.iterator_cls.input_schema()
+        for arg_name, arg_value in iterator_args.items():
+            col_type = input_schema[arg_name]
+            if arg_value is None and not col_type.nullable:
+                return False
+        return True
+
+    def __populate_output_row(self, output_row: exprs.DataRow, pos: int, component_dict: dict) -> None:
+        pk = output_row.pk[:-1] + (pos,) + output_row.pk[-1:]
+        output_row.set_pk(pk)
+        # verify and copy component_dict fields to their respective slots in output_row
+        for field_name, field_val in component_dict.items():
+            if field_name not in self.iterator_output_fields:
+                raise excs.Error(
+                    f'Invalid field name {field_name} in output of {self.view.iterator_cls.__name__}')
+            if field_name not in self.refd_output_slot_idxs:
+                # we can ignore this
+                continue
+            output_col = self.iterator_output_cols[field_name]
+            output_col.col_type.validate_literal(field_val)
+            output_row[self.refd_output_slot_idxs[field_name]] = field_val
+        if len(component_dict) != len(self.iterator_output_fields):
+            missing_fields = set(self.refd_output_slot_idxs.keys()) - set(component_dict.keys())
+            raise excs.Error(
+                f'Invalid output of {self.view.iterator_cls.__name__}: '
+                f'missing fields {", ".join(missing_fields)}')
+
     def __next__(self) -> DataRowBatch:
-        if self._output is None:
-            self._output = self._output_batches()
-        return next(self._output)
+        if self.__output is None:
+            self.__output = self.__output_batches()
+        return next(self.__output)

pixeltable/exprs/__init__.py

@@ -9,8 +9,7 @@ from .expr import Expr
 from .expr_set import ExprSet
 from .function_call import FunctionCall
 from .in_predicate import InPredicate
-from .inline_array import InlineArray
-from .inline_dict import InlineDict
+from .inline_expr import InlineArray, InlineDict, InlineList
 from .is_null import IsNull
 from .json_mapper import JsonMapper
 from .json_path import RELATIVE_PATH_ROOT, JsonPath

pixeltable/exprs/expr.py

@@ -356,15 +356,14 @@ class Expr(abc.ABC):
         """
         if isinstance(o, Expr):
             return o
-        # Try to create a literal. We need to check for InlineArray/InlineDict
-        # first, to prevent arrays from inappropriately being interpreted as JsonType
+        # Try to create a literal. We need to check for InlineList/InlineDict
+        # first, to prevent them from inappropriately being interpreted as JsonType
         # literals.
-        # TODO: general cleanup of InlineArray/InlineDict
         if isinstance(o, list):
-            from .inline_array import InlineArray
-            return InlineArray(tuple(o))
+            from .inline_expr import InlineList
+            return InlineList(o)
         if isinstance(o, dict):
-            from .inline_dict import InlineDict
+            from .inline_expr import InlineDict
             return InlineDict(o)
         obj_type = ts.ColumnType.infer_literal_type(o)
         if obj_type is not None: