pixeltable 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

pixeltable/env.py

@@ -16,19 +16,19 @@ import threading
 import uuid
 import warnings
 from abc import abstractmethod
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from sys import stdout
-from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import pixeltable_pgserver
 import sqlalchemy as sql
-import toml
 from tqdm import TqdmWarning
 
-import pixeltable.exceptions as excs
-from pixeltable import metadata
+from pixeltable import exceptions as excs
+from pixeltable.config import Config
 from pixeltable.utils.console_output import ConsoleLogger, ConsoleMessageFilter, ConsoleOutputHandler, map_level
 from pixeltable.utils.http_server import make_server
 
@@ -47,9 +47,9 @@ class Env:
     """
 
     _instance: Optional[Env] = None
+    __initializing: bool = False
     _log_fmt_str = '%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d: %(message)s'
 
-    _home: Optional[Path]
     _media_dir: Optional[Path]
     _file_cache_dir: Optional[Path]  # cached media files with external URL
     _dataset_cache_dir: Optional[Path]  # cached datasets (eg, pytorch or COCO)
@@ -69,19 +69,18 @@ class Env:
     _httpd: Optional[http.server.HTTPServer]
     _http_address: Optional[str]
     _logger: logging.Logger
-    _console_logger: ConsoleLogger
     _default_log_level: int
     _logfilename: Optional[str]
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
-    _config_file: Optional[Path]
-    _config: Optional[Config]
     _file_cache_size_g: float
     _pxt_api_key: Optional[str]
     _stdout_handler: logging.StreamHandler
     _initialized: bool
 
     _resource_pool_info: dict[str, Any]
+    _current_conn: Optional[sql.Connection]
+    _current_session: Optional[sql.orm.Session]
 
     @classmethod
     def get(cls) -> Env:
@@ -91,15 +90,17 @@ class Env:
 
     @classmethod
     def _init_env(cls, reinit_db: bool = False) -> None:
+        assert not cls.__initializing, 'Circular env initialization detected.'
+        cls.__initializing = True
         env = Env()
         env._set_up(reinit_db=reinit_db)
         env._upgrade_metadata()
         cls._instance = env
+        cls.__initializing = False
 
     def __init__(self):
         assert self._instance is None, 'Env is a singleton; use Env.get() to access the instance'
 
-        self._home = None
         self._media_dir = None  # computed media files
         self._file_cache_dir = None  # cached media files with external URL
         self._dataset_cache_dir = None  # cached datasets (eg, pytorch or COCO)
@@ -127,21 +128,14 @@ class Env:
         self._log_to_stdout = False
         self._module_log_level = {}  # module name -> log level
 
-        # config
-        self._config_file = None
-        self._config = None
-
         # create logging handler to also log to stdout
         self._stdout_handler = logging.StreamHandler(stream=sys.stdout)
         self._stdout_handler.setFormatter(logging.Formatter(self._log_fmt_str))
         self._initialized = False
 
         self._resource_pool_info = {}
-
-    @property
-    def config(self) -> Config:
-        assert self._config is not None
-        return self._config
+        self._current_conn = None
+        self._current_session = None
 
     @property
     def db_url(self) -> str:
@@ -166,6 +160,33 @@ class Env:
         self.engine.dispose()
         self._create_engine(time_zone_name=tz_name)
 
+    @property
+    def conn(self) -> Optional[sql.Connection]:
+        assert self._current_conn is not None
+        return self._current_conn
+
+    @property
+    def session(self) -> Optional[sql.orm.Session]:
+        assert self._current_session is not None
+        return self._current_session
+
+    @contextmanager
+    def begin_xact(self) -> Iterator[sql.Connection]:
+        """Return a context manager that yields a connection to the database. Idempotent."""
+        if self._current_conn is None:
+            assert self._current_session is None
+            with self.engine.begin() as conn, sql.orm.Session(conn) as session:
+                self._current_conn = conn
+                self._current_session = session
+                try:
+                    yield conn
+                finally:
+                    self._current_session = None
+                    self._current_conn = None
+        else:
+            assert self._current_session is not None
+            yield self._current_conn
+
     def configure_logging(
         self,
         *,
@@ -233,10 +254,7 @@ class Env:
             for module_name in path_parts[:max_idx]:
                 if module_name in self._module_log_level and record.levelno >= self._module_log_level[module_name]:
                     return True
-        if record.levelno >= self._default_log_level:
-            return True
-        else:
-            return False
+        return record.levelno >= self._default_log_level
 
     @property
     def console_logger(self) -> ConsoleLogger:
@@ -248,28 +266,14 @@ class Env:
 
         os.environ['TOKENIZERS_PARALLELISM'] = 'false'
 
+        config = Config.get()
+
         self._initialized = True
-        home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
-        assert self._home is None or self._home == home
-        self._home = home
-        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.toml')))
-        self._media_dir = self._home / 'media'
-        self._file_cache_dir = self._home / 'file_cache'
-        self._dataset_cache_dir = self._home / 'dataset_cache'
-        self._log_dir = self._home / 'logs'
-        self._tmp_dir = self._home / 'tmp'
-
-        if self._home.exists() and not self._home.is_dir():
-            raise RuntimeError(f'{self._home} is not a directory')
-
-        if not self._home.exists():
-            # we don't have our logger set up yet, so print to stdout
-            print(f'Creating a Pixeltable instance at: {self._home}')
-            self._home.mkdir()
-            # TODO (aaron-siegel) This is the existing behavior, but it seems scary. If something happens to
-            # self._home, it will cause the DB to be destroyed even if pgdata is in an alternate location.
-            # PROPOSAL: require `reinit_db` to be set explicitly to destroy the DB.
-            reinit_db = True
+        self._media_dir = Config.get().home / 'media'
+        self._file_cache_dir = Config.get().home / 'file_cache'
+        self._dataset_cache_dir = Config.get().home / 'dataset_cache'
+        self._log_dir = Config.get().home / 'logs'
+        self._tmp_dir = Config.get().home / 'tmp'
 
         if not self._media_dir.exists():
             self._media_dir.mkdir()
@@ -282,26 +286,24 @@ class Env:
         if not self._tmp_dir.exists():
             self._tmp_dir.mkdir()
 
-        # Read in the config
-        self._config = Config.from_file(self._config_file)
-        self._file_cache_size_g = self._config.get_float_value('file_cache_size_g')
+        self._file_cache_size_g = config.get_float_value('file_cache_size_g')
         if self._file_cache_size_g is None:
             raise excs.Error(
                 'pixeltable/file_cache_size_g is missing from configuration\n'
-                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {self._config_file},\n'
+                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {Config.get().config_file},\n'
                 'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
             )
-        self._pxt_api_key = self._config.get_string_value('api_key')
+        self._pxt_api_key = config.get_string_value('api_key')
 
         # Disable spurious warnings
         warnings.simplefilter('ignore', category=TqdmWarning)
-        if self._config.get_bool_value('hide_warnings'):
+        if config.get_bool_value('hide_warnings'):
             # Disable more warnings
             warnings.simplefilter('ignore', category=UserWarning)
             warnings.simplefilter('ignore', category=FutureWarning)
 
         # Set verbose level for user visible console messages
-        verbosity = map_level(self._config.get_int_value('verbosity'))
+        verbosity = map_level(config.get_int_value('verbosity'))
         stdout_handler = ConsoleOutputHandler(stream=stdout)
         stdout_handler.setLevel(verbosity)
         stdout_handler.addFilter(ConsoleMessageFilter())
@@ -339,7 +341,7 @@ class Env:
         self.clear_tmp_dir()
 
         self._db_name = os.environ.get('PIXELTABLE_DB', 'pixeltable')
-        self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(self._home / 'pgdata')))
+        self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(Config.get().home / 'pgdata')))
 
         # cleanup_mode=None will leave the postgres process running after Python exits
         # cleanup_mode='stop' will terminate the postgres process when Python exits
@@ -349,11 +351,11 @@ class Env:
         self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=cleanup_mode)
         self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
 
-        tz_name = self.config.get_string_value('time_zone')
+        tz_name = config.get_string_value('time_zone')
         if tz_name is not None:
             # Validate tzname
             if not isinstance(tz_name, str):
-                self._logger.error(f'Invalid time zone specified in configuration.')
+                self._logger.error('Invalid time zone specified in configuration.')
             else:
                 try:
                     _ = ZoneInfo(tz_name)
@@ -375,9 +377,9 @@ class Env:
         self._create_engine(time_zone_name=tz_name, echo=echo)
 
         if create_db:
-            from pixeltable.metadata import schema
+            from pixeltable import metadata
 
-            schema.base_metadata.create_all(self._sa_engine)
+            metadata.schema.base_metadata.create_all(self._sa_engine)
             metadata.create_system_info(self._sa_engine)
 
         self.console_logger.info(f'Connected to Pixeltable database at: {self.db_url}')
@@ -460,6 +462,8 @@ class Env:
             engine.dispose()
 
     def _upgrade_metadata(self) -> None:
+        from pixeltable import metadata
+
         metadata.upgrade_md(self._sa_engine)
 
     @property
@@ -467,7 +471,7 @@ class Env:
         if self._pxt_api_key is None:
             raise excs.Error(
                 'No API key is configured. Set the PIXELTABLE_API_KEY environment variable, or add an entry to '
-                f'config.toml as described here:\nhttps://pixeltable.github.io/pixeltable/config/'
+                'config.toml as described here:\nhttps://pixeltable.github.io/pixeltable/config/'
             )
         return self._pxt_api_key
 
@@ -486,14 +490,14 @@ class Env:
 
         init_kwargs: dict[str, str] = {}
         for param in cl.param_names:
-            arg = self._config.get_string_value(param, section=name)
+            arg = Config.get().get_string_value(param, section=name)
             if arg is not None and len(arg) > 0:
                 init_kwargs[param] = arg
             else:
                 raise excs.Error(
                     f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, or put `{param.lower()}` in '
-                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
+                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, '
+                    f'or put `{param.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
         cl.client_obj = cl.init_fn(**init_kwargs)
@@ -526,8 +530,6 @@ class Env:
         """Check for and start runtime services"""
         self._start_web_server()
         self.__register_packages()
-        if self.is_installed_package('spacy'):
-            self.__init_spacy()
 
     def __register_packages(self) -> None:
         """Declare optional packages that are utilized by some parts of the code."""
@@ -590,7 +592,8 @@ class Env:
             if not package_info.is_installed:
                 # Still not found.
                 raise excs.Error(
-                    f'This feature requires the `{package_name}` package. To install it, run: `pip install -U {package_info.library_name}`'
+                    f'This feature requires the `{package_name}` package. To install it, run: '
+                    f'`pip install -U {package_info.library_name}`'
                 )
 
         if min_version is None:
@@ -603,41 +606,12 @@ class Env:
 
         if min_version > package_info.version:
             raise excs.Error(
-                f'The installed version of package `{package_name}` is {".".join(str(v) for v in package_info.version)}, '
+                f'The installed version of package `{package_name}` is '
+                f'{".".join(str(v) for v in package_info.version)}, '
                 f'but version >={".".join(str(v) for v in min_version)} is required. '
                 f'To fix this, run: `pip install -U {package_info.library_name}`'
             )
 
-    def __init_spacy(self) -> None:
-        """
-        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
-        dependency, we install it programmatically here. This should cause no problems, since the model packages
-        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
-        """
-        import spacy
-        from spacy.cli.download import get_model_filename
-
-        spacy_model = 'en_core_web_sm'
-        spacy_model_version = '3.7.1'
-        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
-        url = f'{spacy.about.__download_url__}/{filename}'
-        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
-        # a problem, because the model have been installed on a previous attempt.
-        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
-        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
-        if ret.returncode != 0:
-            self._logger.warning(f'pip install failed for spaCy model: {filename}')
-        try:
-            self._logger.info(f'Loading spaCy model: {spacy_model}')
-            self._spacy_nlp = spacy.load(spacy_model)
-        except Exception as exc:
-            self._logger.warn(f'Failed to load spaCy model: {spacy_model}', exc_info=exc)
-            warnings.warn(
-                f"Failed to load spaCy model '{spacy_model}'. spaCy features will not be available.",
-                excs.PixeltableWarning,
-            )
-            self.__optional_packages['spacy'].is_installed = False
-
     def clear_tmp_dir(self) -> None:
         for path in glob.glob(f'{self._tmp_dir}/*'):
             if os.path.isdir(path):
@@ -660,11 +634,6 @@ class Env:
             self._resource_pool_info[pool_id] = info
         return info
 
-    @property
-    def home(self) -> Path:
-        assert self._home is not None
-        return self._home
-
     @property
     def media_dir(self) -> Path:
         assert self._media_dir is not None
@@ -693,9 +662,36 @@ class Env:
     @property
     def spacy_nlp(self) -> spacy.Language:
         Env.get().require_package('spacy')
+        if self._spacy_nlp is None:
+            self.__init_spacy()
         assert self._spacy_nlp is not None
         return self._spacy_nlp
 
+    def __init_spacy(self) -> None:
+        """
+        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
+        dependency, we install it programmatically here. This should cause no problems, since the model packages
+        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
+        """
+        import spacy
+        from spacy.cli.download import get_model_filename
+
+        spacy_model = 'en_core_web_sm'
+        spacy_model_version = '3.7.1'
+        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
+        url = f'{spacy.about.__download_url__}/{filename}'
+        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
+        # a problem, because the model might have been installed on a previous attempt.
+        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
+        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
+        if ret.returncode != 0:
+            self._logger.warning(f'pip install failed for spaCy model: {filename}')
+        self._logger.info(f'Loading spaCy model: {spacy_model}')
+        try:
+            self._spacy_nlp = spacy.load(spacy_model)
+        except Exception as exc:
+            raise excs.Error(f'Failed to load spaCy model: {spacy_model}') from exc
+
 
 def register_client(name: str) -> Callable:
     """Decorator that registers a third-party API client for use by Pixeltable.
@@ -723,7 +719,6 @@ def register_client(name: str) -> Callable:
     """
 
     def decorator(fn: Callable) -> None:
-        global _registered_clients
         sig = inspect.signature(fn)
         param_names = list(sig.parameters.keys())
         _registered_clients[name] = ApiClient(init_fn=fn, param_names=param_names)
@@ -731,73 +726,6 @@ def register_client(name: str) -> Callable:
     return decorator
 
 
-class Config:
-    """
-    The (global) Pixeltable configuration, as loaded from `config.toml`. Provides methods for retrieving
-    configuration values, which can be set in the config file or as environment variables.
-    """
-
-    __config: dict[str, Any]
-
-    @classmethod
-    def from_file(cls, path: Path) -> Config:
-        """
-        Loads configuration from the specified TOML file. If the file does not exist, it will be
-        created and populated with the default configuration.
-        """
-        if os.path.isfile(path):
-            with open(path, 'r') as stream:
-                try:
-                    config_dict = toml.load(stream)
-                except Exception as exc:
-                    raise excs.Error(f'Could not read config file: {str(path)}') from exc
-        else:
-            config_dict = cls.__create_default_config(path)
-            with open(path, 'w') as stream:
-                try:
-                    toml.dump(config_dict, stream)
-                except Exception as exc:
-                    raise excs.Error(f'Could not write config file: {str(path)}') from exc
-            logging.getLogger('pixeltable').info(f'Created default config file at: {str(path)}')
-        return cls(config_dict)
-
-    @classmethod
-    def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
-        free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
-        # Default cache size is 1/5 of free disk space
-        file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
-        return {'pixeltable': {'file_cache_size_g': round(file_cache_size_g, 1), 'hide_warnings': False}}
-
-    def __init__(self, config: dict[str, Any]) -> None:
-        self.__config = config
-
-    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
-        env_var = f'{section.upper()}_{key.upper()}'
-        if env_var in os.environ:
-            value = os.environ[env_var]
-        elif section in self.__config and key in self.__config[section]:
-            value = self.__config[section][key]
-        else:
-            return None
-
-        try:
-            return expected_type(value)  # type: ignore[call-arg]
-        except ValueError:
-            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
-
-    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
-        return self.get_value(key, str, section)
-
-    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
-        return self.get_value(key, int, section)
-
-    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
-        return self.get_value(key, float, section)
-
-    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
-        return self.get_value(key, bool, section)
-
-
 _registered_clients: dict[str, ApiClient] = {}
 
 
@@ -852,7 +780,8 @@ class RateLimitsInfo:
             # TODO: remove
             for info in self.resource_limits.values():
                 _logger.debug(
-                    f'Init {info.resource} rate limit: rem={info.remaining} reset={info.reset_at.strftime(TIME_FORMAT)} delta={(info.reset_at - now).total_seconds()}'
+                    f'Init {info.resource} rate limit: rem={info.remaining} '
+                    f'reset={info.reset_at.strftime(TIME_FORMAT)} delta={(info.reset_at - now).total_seconds()}'
                 )
         else:
             for k, v in kwargs.items():
@@ -886,5 +815,6 @@ class RateLimitInfo:
         self.reset_at = reset_at
         # TODO: remove
         _logger.debug(
-            f'Update {self.resource} rate limit: rem={self.remaining} reset={self.reset_at.strftime(TIME_FORMAT)} reset_delta={reset_delta.total_seconds()} recorded_delta={(self.reset_at - recorded_at).total_seconds()}'
+            f'Update {self.resource} rate limit: rem={self.remaining} reset={self.reset_at.strftime(TIME_FORMAT)} '
+            f'reset_delta={reset_delta.total_seconds()} recorded_delta={(self.reset_at - recorded_at).total_seconds()}'
         )

pixeltable/exec/aggregation_node.py

@@ -29,7 +29,7 @@ class AggregationNode(ExecNode):
 
     def __init__(
         self,
-        tbl: catalog.TableVersion,
+        tbl: catalog.TableVersionHandle,
         row_builder: exprs.RowBuilder,
         group_by: Optional[list[exprs.Expr]],
         agg_fn_calls: list[exprs.FunctionCall],
@@ -86,9 +86,10 @@ class AggregationNode(ExecNode):
                     self._reset_agg_state(0)
                 self._update_agg_state(row, 0)
                 prev_row = row
-        # emit the last group
-        self.row_builder.eval(prev_row, self.agg_fn_eval_ctx, profile=self.ctx.profile)
-        self.output_batch.add_row(prev_row)
+        if prev_row is not None:
+            # emit the last group
+            self.row_builder.eval(prev_row, self.agg_fn_eval_ctx, profile=self.ctx.profile)
+            self.output_batch.add_row(prev_row)
 
         self.output_batch.flush_imgs(None, self.stored_img_cols, self.flushed_img_slots)
         _logger.debug(f'AggregateNode: consumed {num_input_rows} rows, returning {len(self.output_batch.rows)} rows')

pixeltable/exec/cache_prefetch_node.py

@@ -40,7 +40,7 @@ class CachePrefetchNode(ExecNode):
     boto_client_lock: threading.Lock
 
     # execution state
-    batch_tbl_version: Optional[catalog.TableVersion]  # needed to construct output batches
+    batch_tbl_version: Optional[catalog.TableVersionHandle]  # needed to construct output batches
     num_returned_rows: int
 
     # ready_rows: rows that are ready to be returned, ordered by row idx;

pixeltable/exec/component_iteration_node.py

@@ -14,23 +14,25 @@ class ComponentIterationNode(ExecNode):
     Returns row batches of OUTPUT_BATCH_SIZE size.
     """
 
+    view: catalog.TableVersionHandle
+
     __OUTPUT_BATCH_SIZE = 1024
 
-    def __init__(self, view: catalog.TableVersion, input: ExecNode):
-        assert view.is_component_view()
+    def __init__(self, view: catalog.TableVersionHandle, input: ExecNode):
+        assert view.get().is_component_view
         super().__init__(input.row_builder, [], [], input)
         self.view = view
-        iterator_args = [view.iterator_args.copy()]
+        iterator_args = [view.get().iterator_args.copy()]
         self.row_builder.set_slot_idxs(iterator_args)
         self.iterator_args = iterator_args[0]
         assert isinstance(self.iterator_args, exprs.InlineDict)
         self.iterator_args_ctx = self.row_builder.create_eval_ctx([self.iterator_args])
-        self.iterator_output_schema, self.unstored_column_names = self.view.iterator_cls.output_schema(
+        self.iterator_output_schema, self.unstored_column_names = self.view.get().iterator_cls.output_schema(
             **self.iterator_args.to_kwargs()
         )
         self.iterator_output_fields = list(self.iterator_output_schema.keys())
         self.iterator_output_cols = {
-            field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields
+            field_name: self.view.get().cols_by_name[field_name] for field_name in self.iterator_output_fields
         }
         # referenced iterator output fields
         self.refd_output_slot_idxs = {
@@ -50,7 +52,7 @@ class ComponentIterationNode(ExecNode):
                 # specified and are not null. If any of them are null, then we skip this row (i.e., we emit 0
                 # output rows for this input row).
                 if self.__non_nullable_args_specified(iterator_args):
-                    iterator = self.view.iterator_cls(**iterator_args)
+                    iterator = self.view.get().iterator_cls(**iterator_args)
                     for pos, component_dict in enumerate(iterator):
                         output_row = output_batch.add_row()
                         input_row.copy(output_row)
@@ -67,7 +69,7 @@ class ComponentIterationNode(ExecNode):
         """
         Returns true if all non-nullable iterator arguments are not `None`.
         """
-        input_schema = self.view.iterator_cls.input_schema()
+        input_schema = self.view.get().iterator_cls.input_schema()
         for arg_name, arg_value in iterator_args.items():
             col_type = input_schema[arg_name]
             if arg_value is None and not col_type.nullable:
@@ -80,7 +82,9 @@ class ComponentIterationNode(ExecNode):
         # verify and copy component_dict fields to their respective slots in output_row
         for field_name, field_val in component_dict.items():
             if field_name not in self.iterator_output_fields:
-                raise excs.Error(f'Invalid field name {field_name} in output of {self.view.iterator_cls.__name__}')
+                raise excs.Error(
+                    f'Invalid field name {field_name} in output of {self.view.get().iterator_cls.__name__}'
+                )
             if field_name not in self.refd_output_slot_idxs:
                 # we can ignore this
                 continue
@@ -90,5 +94,5 @@ class ComponentIterationNode(ExecNode):
         if len(component_dict) != len(self.iterator_output_fields):
             missing_fields = set(self.refd_output_slot_idxs.keys()) - set(component_dict.keys())
             raise excs.Error(
-                f'Invalid output of {self.view.iterator_cls.__name__}: missing fields {", ".join(missing_fields)}'
+                f'Invalid output of {self.view.get().iterator_cls.__name__}: missing fields {", ".join(missing_fields)}'
             )

pixeltable/exec/data_row_batch.py

@@ -16,7 +16,7 @@ class DataRowBatch:
     Contains the metadata needed to initialize DataRows.
     """
 
-    tbl: Optional[catalog.TableVersion]
+    tbl: Optional[catalog.TableVersionHandle]
     row_builder: exprs.RowBuilder
     img_slot_idxs: list[int]
     media_slot_idxs: list[int]  # non-image media slots
@@ -25,7 +25,7 @@ class DataRowBatch:
 
     def __init__(
         self,
-        tbl: Optional[catalog.TableVersion],
+        tbl: Optional[catalog.TableVersionHandle],
         row_builder: exprs.RowBuilder,
         num_rows: Optional[int] = None,
         rows: Optional[list[exprs.DataRow]] = None,
@@ -91,7 +91,7 @@ class DataRowBatch:
             idx_range = slice(0, len(self.rows))
         for row in self.rows[idx_range]:
             for info in stored_img_info:
-                filepath = str(MediaStore.prepare_media_path(self.tbl.id, info.col.id, self.tbl.version))
+                filepath = str(MediaStore.prepare_media_path(self.tbl.id, info.col.id, self.tbl.get().version))
                 row.flush_img(info.slot_idx, filepath)
             for slot_idx in flushed_slot_idxs:
                 row.flush_img(slot_idx)

pixeltable/exec/exec_context.py

@@ -28,7 +28,3 @@ class ExecContext:
         self.pk_clause = pk_clause
         self.num_computed_exprs = num_computed_exprs
         self.ignore_errors = ignore_errors
-
-    def set_conn(self, conn: sql.engine.Connection) -> None:
-        self.conn = conn
-        self.row_builder.set_conn(conn)

pixeltable/exec/exec_node.py

@@ -6,7 +6,7 @@ import logging
 import sys
 from typing import AsyncIterator, Iterable, Iterator, Optional, TypeVar
 
-import pixeltable.exprs as exprs
+from pixeltable import exprs
 
 from .data_row_batch import DataRowBatch
 from .exec_context import ExecContext
@@ -31,6 +31,7 @@ class ExecNode(abc.ABC):
         input_exprs: Iterable[exprs.Expr],
         input: Optional[ExecNode] = None,
     ):
+        assert all(expr.is_valid for expr in output_exprs)
         self.output_exprs = output_exprs
         self.row_builder = row_builder
         self.input = input
@@ -65,7 +66,7 @@ class ExecNode(abc.ABC):
             # check if we are already in an event loop (eg, Jupyter's); if so, patch it to allow
             # multiple run_until_complete()
             running_loop = asyncio.get_running_loop()
-            import nest_asyncio  # type: ignore
+            import nest_asyncio  # type: ignore[import-untyped]
 
             nest_asyncio.apply()
             loop = running_loop

pixeltable/exec/expr_eval/schedulers.py

@@ -9,6 +9,7 @@ import time
 from typing import Awaitable, Collection, Optional
 
 from pixeltable import env, func
+from pixeltable.config import Config
 
 from .globals import Dispatcher, FnCallArgs, Scheduler
 
@@ -276,7 +277,7 @@ class RequestRateScheduler(Scheduler):
             _, endpoint, model = elems
             section = f'{endpoint}.rate_limits'
             key = model
-        requests_per_min = env.Env.get().config.get_int_value(key, section=section)
+        requests_per_min = Config.get().get_int_value(key, section=section)
         requests_per_min = requests_per_min or self.DEFAULT_RATE_LIMIT
         self.secs_per_request = 1 / (requests_per_min / 60)