pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/catalog/view.py

@@ -2,25 +2,28 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, List, Literal, Optional
+from typing import TYPE_CHECKING, Any, List, Literal
 from uuid import UUID
 
 import pixeltable.exceptions as excs
 import pixeltable.metadata.schema as md_schema
 import pixeltable.type_system as ts
 from pixeltable import catalog, exprs, func
-from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 
 from .column import Column
-from .globals import _POS_COLUMN_NAME, MediaValidation, UpdateStatus
+from .globals import _POS_COLUMN_NAME, MediaValidation
 from .table import Table
-from .table_version import TableVersion
+from .table_version import TableVersion, TableVersionKey, TableVersionMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .tbl_ops import CreateStoreTableOp, LoadViewOp, TableOp
+from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
+    from pixeltable.catalog.table import TableMetadata
     from pixeltable.globals import TableDataSource
+    from pixeltable.plan import SampleClause
 
 _logger = logging.getLogger('pixeltable')
 
@@ -37,21 +40,28 @@ class View(Table):
     def __init__(self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath, snapshot_only: bool):
         super().__init__(id, dir_id, name, tbl_version_path)
         self._snapshot_only = snapshot_only
+        if not snapshot_only:
+            self._tbl_version = tbl_version_path.tbl_version
+
+    def _display_name(self) -> str:
+        if self._tbl_version_path.is_replica():
+            return 'replica'
+        if self._tbl_version_path.is_snapshot():
+            return 'snapshot'
+        if self._tbl_version_path.is_view():
+            return 'view'
+        return 'table'
 
     @classmethod
-    def _display_name(cls) -> str:
-        return 'view'
-
-    @classmethod
-    def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, Optional[str]]]) -> dict[str, dict]:
+    def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, str | None]]) -> dict[str, dict]:
         """Returns a list of columns in the same format as the additional_columns parameter of View.create.
-        The source is the list of expressions from a select() statement on a DataFrame.
+        The source is the list of expressions from a select() statement on a Query.
         If the column is a ColumnRef, to a base table column, it is marked to not be stored.sy
         """
-        from pixeltable.dataframe import DataFrame
+        from pixeltable._query import Query
 
         r: dict[str, dict] = {}
-        exps, names = DataFrame._normalize_select_list([], select_list)
+        exps, names = Query._normalize_select_list([], select_list)
         for expr, name in zip(exps, names):
             stored = not isinstance(expr, exprs.ColumnRef)
             r[name] = {'value': expr, 'stored': stored}
@@ -63,16 +73,20 @@ class View(Table):
         dir_id: UUID,
         name: str,
         base: TableVersionPath,
-        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
+        select_list: list[tuple[exprs.Expr, str | None]] | None,
         additional_columns: dict[str, Any],
-        predicate: Optional['exprs.Expr'],
+        predicate: 'exprs.Expr' | None,
+        sample_clause: 'SampleClause' | None,
         is_snapshot: bool,
+        create_default_idxs: bool,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-        iterator_cls: Optional[type[ComponentIterator]],
-        iterator_args: Optional[dict],
-    ) -> View:
+        iterator_cls: type[ComponentIterator] | None,
+        iterator_args: dict | None,
+    ) -> tuple[TableVersionMd, list[TableOp] | None]:
+        from pixeltable.plan import SampleClause
+
         # Convert select_list to more additional_columns if present
         include_base_columns: bool = select_list is None
         select_list_columns: List[Column] = []
@@ -84,12 +98,25 @@ class View(Table):
         columns = select_list_columns + columns_from_additional_columns
         cls._verify_schema(columns)
 
-        # verify that filter can be evaluated in the context of the base
+        # verify that filters can be evaluated in the context of the base
         if predicate is not None:
             if not predicate.is_bound_by([base]):
-                raise excs.Error(f'Filter cannot be computed in the context of the base {base.tbl_name()}')
+                raise excs.Error(f'View filter cannot be computed in the context of the base table {base.tbl_name()!r}')
             # create a copy that we can modify and store
             predicate = predicate.copy()
+        if sample_clause is not None:
+            # make sure that the sample clause can be computed in the context of the base
+            if sample_clause.stratify_exprs is not None and not all(
+                stratify_expr.is_bound_by([base]) for stratify_expr in sample_clause.stratify_exprs
+            ):
+                raise excs.Error(
+                    f'View sample clause cannot be computed in the context of the base table {base.tbl_name()!r}'
+                )
+            # create a copy that we can modify and store
+            sc = sample_clause
+            sample_clause = SampleClause(
+                sc.version, sc.n, sc.n_per_stratum, sc.fraction, sc.seed, sc.stratify_exprs.copy()
+            )
 
         # same for value exprs
         for col in columns:
@@ -98,8 +125,8 @@ class View(Table):
             # make sure that the value can be computed in the context of the base
             if col.value_expr is not None and not col.value_expr.is_bound_by([base]):
                 raise excs.Error(
-                    f'Column {col.name}: value expression cannot be computed in the context of the '
-                    f'base {base.tbl_name()}'
+                    f'Column {col.name!r}: Value expression cannot be computed in the context of the '
+                    f'base table {base.tbl_name()!r}'
                 )
 
         if iterator_cls is not None:
@@ -126,18 +153,18 @@ class View(Table):
             sig = func.Signature(ts.InvalidType(), params)
 
             expr_args = {k: exprs.Expr.from_object(v) for k, v in bound_args.items()}
-            sig.validate_args(expr_args, context=f'in iterator {iterator_cls.__name__!r}')
+            sig.validate_args(expr_args, context=f'in iterator of type `{iterator_cls.__name__}`')
             literal_args = {k: v.val if isinstance(v, exprs.Literal) else v for k, v in expr_args.items()}
 
             # prepend pos and output_schema columns to cols:
             # a component view exposes the pos column of its rowid;
             # we create that column here, so it gets assigned a column id;
             # stored=False: it is not stored separately (it's already stored as part of the rowid)
-            iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), stored=False)]
+            iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), is_iterator_col=True, stored=False)]
             output_dict, unstored_cols = iterator_cls.output_schema(**literal_args)
             iterator_cols.extend(
                 [
-                    Column(col_name, col_type, stored=col_name not in unstored_cols)
+                    Column(col_name, col_type, is_iterator_col=True, stored=col_name not in unstored_cols)
                     for col_name, col_type in output_dict.items()
                 ]
             )
@@ -146,11 +173,10 @@ class View(Table):
             for col in columns:
                 if col.name in iterator_col_names:
                     raise excs.Error(
-                        f'Duplicate name: column {col.name} is already present in the iterator output schema'
+                        f'Duplicate name: column {col.name!r} is already present in the iterator output schema'
                     )
             columns = iterator_cols + columns
 
-        session = Env.get().session
         from pixeltable.exprs import InlineDict
 
         iterator_args_expr: exprs.Expr = InlineDict(iterator_args) if iterator_args is not None else None
@@ -160,6 +186,8 @@ class View(Table):
         # if this is a snapshot, we need to retarget all exprs to the snapshot tbl versions
         if is_snapshot:
             predicate = predicate.retarget(base_version_path) if predicate is not None else None
+            if sample_clause is not None:
+                exprs.Expr.retarget_list(sample_clause.stratify_exprs, base_version_path)
             iterator_args_expr = (
                 iterator_args_expr.retarget(base_version_path) if iterator_args_expr is not None else None
             )
@@ -171,51 +199,43 @@ class View(Table):
             is_snapshot=is_snapshot,
             include_base_columns=include_base_columns,
             predicate=predicate.as_dict() if predicate is not None else None,
+            sample_clause=sample_clause.as_dict() if sample_clause is not None else None,
             base_versions=base_version_path.as_md(),
             iterator_class_fqn=iterator_class_fqn,
             iterator_args=iterator_args_expr.as_dict() if iterator_args_expr is not None else None,
         )
 
-        id, tbl_version = TableVersion.create(
-            dir_id,
+        md = TableVersion.create_initial_md(
             name,
             columns,
             num_retained_versions,
             comment,
             media_validation=media_validation,
-            # base_path=base_version_path,
             view_md=view_md,
+            create_default_idxs=create_default_idxs,
         )
-        if tbl_version is None:
-            # this is purely a snapshot: we use the base's tbl version path
-            view = cls(id, dir_id, name, base_version_path, snapshot_only=True)
-            _logger.info(f'created snapshot {name}')
+        if md.tbl_md.is_pure_snapshot:
+            # this is purely a snapshot: no store table to create or load
+            return md, None
         else:
-            view = cls(
-                id,
-                dir_id,
-                name,
-                TableVersionPath(
-                    TableVersionHandle(tbl_version.id, tbl_version.effective_version), base=base_version_path
+            tbl_id = md.tbl_md.tbl_id
+            key = TableVersionKey(UUID(tbl_id), 0 if is_snapshot else None, None)
+            view_path = TableVersionPath(TableVersionHandle(key), base=base_version_path)
+            ops = [
+                TableOp(
+                    tbl_id=tbl_id, op_sn=0, num_ops=2, needs_xact=False, create_store_table_op=CreateStoreTableOp()
                 ),
-                snapshot_only=False,
-            )
-            _logger.info(f'Created view `{name}`, id={tbl_version.id}')
-
-            from pixeltable.plan import Planner
-
-            plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
-            num_rows, num_excs, _ = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
-            Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
-
-        session.commit()
-        return view
+                TableOp(
+                    tbl_id=tbl_id, op_sn=1, num_ops=2, needs_xact=True, load_view_op=LoadViewOp(view_path.as_dict())
+                ),
+            ]
+            return md, ops
 
     @classmethod
     def _verify_column(cls, col: Column) -> None:
         # make sure that columns are nullable or have a default
         if not col.col_type.nullable and not col.is_computed:
-            raise excs.Error(f'Column {col.name}: non-computed columns in views must be nullable')
+            raise excs.Error(f'Column {col.name!r}: Non-computed columns in views must be nullable')
         super()._verify_column(col)
 
     @classmethod
@@ -227,74 +247,98 @@ class View(Table):
         if tbl_version_path.is_snapshot():
             return tbl_version_path
         tbl_version = tbl_version_path.tbl_version.get()
-        if not tbl_version.is_snapshot:
-            # create and register snapshot version
-            tbl_version = tbl_version.create_snapshot_copy()
-            assert tbl_version.is_snapshot
+        assert not tbl_version.is_snapshot
 
         return TableVersionPath(
-            TableVersionHandle(tbl_version.id, tbl_version.effective_version),
+            TableVersionHandle(TableVersionKey(tbl_version.id, tbl_version.version, None)),
             base=cls._get_snapshot_path(tbl_version_path.base) if tbl_version_path.base is not None else None,
         )
 
-    def _drop(self) -> None:
-        if self._snapshot_only:
-            # there is not TableVersion to drop
-            self._check_is_dropped()
-            self.is_dropped = True
-            catalog.Catalog.get().delete_tbl_md(self._id)
-        else:
-            super()._drop()
+    def _is_named_pure_snapshot(self) -> bool:
+        """
+        Returns True if this is a named pure snapshot (i.e., a pure snapshot that is a separate schema object).
+        """
+        return self._id != self._tbl_version_path.tbl_id
 
-    def get_metadata(self) -> dict[str, Any]:
-        md = super().get_metadata()
+    def _is_anonymous_snapshot(self) -> bool:
+        """
+        Returns True if this is an unnamed snapshot (i.e., a snapshot that is not a separate schema object).
+        """
+        return self._snapshot_only and self._id == self._tbl_version_path.tbl_id
+
+    def _get_metadata(self) -> 'TableMetadata':
+        md = super()._get_metadata()
         md['is_view'] = True
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()
+        if self._is_anonymous_snapshot():
+            # Update name and path with version qualifiers.
+            md['name'] = f'{self._name}:{self._tbl_version_path.version()}'
+            md['path'] = f'{self._path()}:{self._tbl_version_path.version()}'
+        base_tbl_id = self._base_tbl_id
+        if base_tbl_id is not None:
+            base_tbl = self._get_base_table()
+            base_path = '<anonymous base table>' if base_tbl is None else base_tbl._path()
+            base_version = self._effective_base_versions[0]
+            md['base'] = base_path if base_version is None else f'{base_path}:{base_version}'
         return md
 
     def insert(
         self,
-        source: Optional[TableDataSource] = None,
+        source: TableDataSource | None = None,
         /,
         *,
-        source_format: Optional[Literal['csv', 'excel', 'parquet', 'json']] = None,
-        schema_overrides: Optional[dict[str, ts.ColumnType]] = None,
+        source_format: Literal['csv', 'excel', 'parquet', 'json'] | None = None,
+        schema_overrides: dict[str, ts.ColumnType] | None = None,
         on_error: Literal['abort', 'ignore'] = 'abort',
         print_stats: bool = False,
         **kwargs: Any,
     ) -> UpdateStatus:
-        raise excs.Error(f'{self._display_name()} {self._name!r}: cannot insert into view')
+        raise excs.Error(f'{self._display_str()}: Cannot insert into a {self._display_name()}.')
 
-    def delete(self, where: Optional[exprs.Expr] = None) -> UpdateStatus:
-        raise excs.Error(f'{self._display_name()} {self._name!r}: cannot delete from view')
+    def delete(self, where: exprs.Expr | None = None) -> UpdateStatus:
+        raise excs.Error(f'{self._display_str()}: Cannot delete from a {self._display_name()}.')
 
     @property
-    def _base_table(self) -> Optional['Table']:
-        # if this is a pure snapshot, our tbl_version_path only reflects the base (there is no TableVersion instance
-        # for the snapshot itself)
-        base_id = self._tbl_version.id if self._snapshot_only else self._tbl_version_path.base.tbl_version.id
-        return catalog.Catalog.get().get_table_by_id(base_id)
+    def _base_tbl_id(self) -> UUID | None:
+        if self._tbl_version_path.tbl_id != self._id:
+            # _tbl_version_path represents a different schema object from this one. This can only happen if this is a
+            # named pure snapshot.
+            return self._tbl_version_path.tbl_id
+        if self._tbl_version_path.base is None:
+            return None
+        return self._tbl_version_path.base.tbl_id
+
+    def _get_base_table(self) -> 'Table' | None:
+        """Returns None if there is no base table, or if the base table is hidden."""
+        base_tbl_id = self._base_tbl_id
+        if base_tbl_id is None:
+            return None
+        with catalog.Catalog.get().begin_xact(tbl_id=base_tbl_id, for_write=False):
+            return catalog.Catalog.get().get_table_by_id(base_tbl_id)
 
     @property
-    def _effective_base_versions(self) -> list[Optional[int]]:
+    def _effective_base_versions(self) -> list[int | None]:
         effective_versions = [tv.effective_version for tv in self._tbl_version_path.get_tbl_versions()]
-        if self._snapshot_only:
-            return effective_versions
+        if self._snapshot_only and not self._is_anonymous_snapshot():
+            return effective_versions  # Named pure snapshot
         else:
             return effective_versions[1:]
 
     def _table_descriptor(self) -> str:
-        display_name = 'Snapshot' if self._snapshot_only else 'View'
-        result = [f'{display_name} {self._path!r}']
+        result = [self._display_str()]
         bases_descrs: list[str] = []
-        for base, effective_version in zip(self._base_tables, self._effective_base_versions):
+        for base, effective_version in zip(self._get_base_tables(), self._effective_base_versions):
             if effective_version is None:
-                bases_descrs.append(f'{base._path!r}')
+                bases_descrs.append(f'{base._path()!r}')
             else:
-                base_descr = f'{base._path}:{effective_version}'
+                base_descr = f'{base._path()}:{effective_version}'
                 bases_descrs.append(f'{base_descr!r}')
-        result.append(f' (of {", ".join(bases_descrs)})')
-
-        if self._tbl_version.get().predicate is not None:
-            result.append(f'\nWhere: {self._tbl_version.get().predicate!s}')
+        if len(bases_descrs) > 0:
+            # bases_descrs can be empty in the case of a table-replica
+            result.append(f' (of {", ".join(bases_descrs)})')
+
+        if self._tbl_version_path.tbl_version.get().predicate is not None:
+            result.append(f'\nWhere: {self._tbl_version_path.tbl_version.get().predicate!s}')
+        if self._tbl_version_path.tbl_version.get().sample_clause is not None:
+            result.append(f'\nSample: {self._tbl_version.get().sample_clause!s}')
         return ''.join(result)

pixeltable/config.py

@@ -4,11 +4,11 @@ import logging
 import os
 import shutil
 from pathlib import Path
-from typing import Any, ClassVar, Optional, TypeVar
+from typing import Any, ClassVar, TypeVar
 
 import toml
 
-from pixeltable import exceptions as excs
+from pixeltable import env, exceptions as excs
 
 _logger = logging.getLogger('pixeltable')
 
@@ -21,23 +21,30 @@ class Config:
     configuration values, which can be set in the config file or as environment variables.
     """
 
-    __instance: ClassVar[Optional[Config]] = None
+    __instance: ClassVar[Config | None] = None
 
     __home: Path
     __config_file: Path
+    __config_overrides: dict[str, Any]
     __config_dict: dict[str, Any]
 
-    def __init__(self) -> None:
+    def __init__(self, config_overrides: dict[str, Any]) -> None:
         assert self.__instance is None, 'Config is a singleton; use Config.get() to access the instance'
 
-        self.__home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
+        for var in config_overrides:
+            if var not in KNOWN_CONFIG_OVERRIDES:
+                raise excs.Error(f'Unrecognized configuration variable: {var}')
+
+        self.__config_overrides = config_overrides
+
+        self.__home = Path(self.lookup_env('pixeltable', 'home', str(Path.home() / '.pixeltable')))
         if self.__home.exists() and not self.__home.is_dir():
-            raise RuntimeError(f'{self.__home} is not a directory')
+            raise excs.Error(f'Not a directory: {self.__home}')
         if not self.__home.exists():
             print(f'Creating a Pixeltable instance at: {self.__home}')
             self.__home.mkdir()
 
-        self.__config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self.__home / 'config.toml')))
+        self.__config_file = Path(self.lookup_env('pixeltable', 'config', str(self.__home / 'config.toml')))
 
         self.__config_dict: dict[str, Any]
         if os.path.isfile(self.__config_file):
@@ -46,6 +53,12 @@ class Config:
                     self.__config_dict = toml.load(stream)
                 except Exception as exc:
                     raise excs.Error(f'Could not read config file: {self.__config_file}') from exc
+            for section, section_dict in self.__config_dict.items():
+                if section not in KNOWN_CONFIG_OPTIONS:
+                    raise excs.Error(f'Unrecognized section {section!r} in config file: {self.__config_file}')
+                for key in section_dict:
+                    if key not in KNOWN_CONFIG_OPTIONS[section]:
+                        raise excs.Error(f"Unrecognized option '{section}.{key}' in config file: {self.__config_file}")
         else:
             self.__config_dict = self.__create_default_config(self.__config_file)
             with open(self.__config_file, 'w', encoding='utf-8') as stream:
@@ -65,10 +78,22 @@ class Config:
 
     @classmethod
     def get(cls) -> Config:
-        if cls.__instance is None:
-            cls.__instance = cls()
+        cls.init({})
         return cls.__instance
 
+    @classmethod
+    def init(cls, config_overrides: dict[str, Any], reinit: bool = False) -> None:
+        if reinit:
+            cls.__instance = None
+            for cl in env._registered_clients.values():
+                cl.client_obj = None
+        if cls.__instance is None:
+            cls.__instance = cls(config_overrides)
+        elif len(config_overrides) > 0:
+            raise excs.Error(
+                'Pixeltable has already been initialized; cannot specify new config values in the same session'
+            )
+
     @classmethod
     def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
         free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
@@ -76,28 +101,115 @@ class Config:
         file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
         return {'pixeltable': {'file_cache_size_g': round(file_cache_size_g, 1), 'hide_warnings': False}}
 
-    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+    def lookup_env(self, section: str, key: str, default: Any = None) -> Any:
+        override_var = f'{section}.{key}'
         env_var = f'{section.upper()}_{key.upper()}'
-        if env_var in os.environ:
-            value = os.environ[env_var]
-        elif section in self.__config_dict and key in self.__config_dict[section]:
-            value = self.__config_dict[section][key]
-        else:
-            return None
+        if override_var in self.__config_overrides:
+            return self.__config_overrides[override_var]
+        if env_var in os.environ and len(os.environ[env_var]) > 0:
+            return os.environ[env_var]
+        return default
+
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> T | None:
+        value: Any = self.lookup_env(section, key)  # Try to get from environment first
+        # Next try the config file
+        if value is None:
+            # Resolve nested section dicts
+            lookup_elems = [*section.split('.'), key]
+            value = self.__config_dict
+            for el in lookup_elems:
+                if isinstance(value, dict):
+                    if el not in value:
+                        return None
+                    value = value[el]
+                else:
+                    return None
+
+        if value is None:
+            return None  # Not specified
 
         try:
+            if expected_type is bool and isinstance(value, str):
+                if value.lower() not in ('true', 'false'):
+                    raise excs.Error(f"Invalid value for configuration parameter '{section}.{key}': {value}")
+                return value.lower() == 'true'  # type: ignore[return-value]
             return expected_type(value)  # type: ignore[call-arg]
-        except ValueError as exc:
-            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}') from exc
+        except (ValueError, TypeError) as exc:
+            raise excs.Error(f"Invalid value for configuration parameter '{section}.{key}': {value}") from exc
 
-    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
+    def get_string_value(self, key: str, section: str = 'pixeltable') -> str | None:
         return self.get_value(key, str, section)
 
-    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
+    def get_int_value(self, key: str, section: str = 'pixeltable') -> int | None:
         return self.get_value(key, int, section)
 
-    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
+    def get_float_value(self, key: str, section: str = 'pixeltable') -> float | None:
         return self.get_value(key, float, section)
 
-    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
+    def get_bool_value(self, key: str, section: str = 'pixeltable') -> bool | None:
         return self.get_value(key, bool, section)
+
+
+KNOWN_CONFIG_OPTIONS = {
+    'pixeltable': {
+        'home': 'Path to the Pixeltable home directory',
+        'config': 'Path to the Pixeltable config file',
+        'pgdata': 'Path to the Pixeltable postgres data directory',
+        'db': 'Postgres database name',
+        'file_cache_size_g': 'Size of the file cache in GB',
+        'time_zone': 'Default time zone for timestamps',
+        'hide_warnings': 'Hide warnings from the console',
+        'verbosity': 'Verbosity level for console output',
+        'api_key': 'API key for Pixeltable cloud',
+        'input_media_dest': 'Default destination URI for input media data',
+        'output_media_dest': 'Default destination URI for output (computed) media data',
+        'r2_profile': 'AWS config profile name used to access R2 storage',
+        's3_profile': 'AWS config profile name used to access S3 storage',
+        'b2_profile': 'AWS config profile name used to access Backblaze B2 storage',
+        'tigris_profile': 'AWS config profile name used to access Tigris object storage',
+    },
+    'anthropic': {'api_key': 'Anthropic API key'},
+    'azure': {'storage_account_name': 'Azure storage account name', 'storage_account_key': 'Azure storage account key'},
+    'bedrock': {'api_key': 'AWS Bedrock API key'},
+    'deepseek': {'api_key': 'Deepseek API key', 'rate_limit': 'Rate limit for Deepseek API requests'},
+    'fal': {'api_key': 'fal.ai API key', 'rate_limit': 'Rate limit for fal.ai API requests'},
+    'fireworks': {'api_key': 'Fireworks API key', 'rate_limit': 'Rate limit for Fireworks API requests'},
+    'gemini': {'api_key': 'Gemini API key', 'rate_limits': 'Per-model rate limits for Gemini API requests'},
+    'hf': {'auth_token': 'Hugging Face access token'},
+    'imagen': {'rate_limits': 'Per-model rate limits for Imagen API requests'},
+    'reve': {'api_key': 'Reve API key', 'rate_limit': 'Rate limit for Reve API requests (requests per minute)'},
+    'groq': {'api_key': 'Groq API key', 'rate_limit': 'Rate limit for Groq API requests'},
+    'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
+    'mistral': {'api_key': 'Mistral API key', 'rate_limit': 'Rate limit for Mistral API requests'},
+    'openai': {
+        'api_key': 'OpenAI API key',
+        'base_url': 'OpenAI API base URL',
+        'api_version': 'API version if using Azure OpenAI',
+        'rate_limits': 'Per-model rate limits for OpenAI API requests',
+        'max_connections': 'Maximum number of concurrent OpenAI API connections that can be established',
+        'max_keepalive_connections': 'Maximum number of keep-alive connections in the pool.'
+        ' Must not exceed max_connections.',
+    },
+    'openrouter': {
+        'api_key': 'OpenRouter API key',
+        'site_url': 'Optional URL for your application (for OpenRouter analytics)',
+        'app_name': 'Optional name for your application (for OpenRouter analytics)',
+        'rate_limit': 'Rate limit for OpenRouter API requests',
+    },
+    'replicate': {'api_token': 'Replicate API token'},
+    'together': {
+        'api_key': 'Together API key',
+        'rate_limits': 'Per-model category rate limits for Together API requests',
+    },
+    'twelvelabs': {'api_key': 'TwelveLabs API key', 'rate_limit': 'Rate limit for TwelveLabs API requests'},
+    'veo': {'rate_limits': 'Per-model rate limits for Veo API requests'},
+    'voyage': {'api_key': 'Voyage AI API key', 'rate_limit': 'Rate limit for Voyage AI API requests'},
+    'pypi': {'api_key': 'PyPI API key (for internal use only)'},
+}
+
+
+KNOWN_CONFIG_OVERRIDES = {
+    f'{section}.{key}': info
+    for section, section_dict in KNOWN_CONFIG_OPTIONS.items()
+    for key, info in section_dict.items()
+}