pixeltable 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

pixeltable/catalog/table_version_handle.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Optional
+from uuid import UUID
+
+from .table_version import TableVersion
+
+if TYPE_CHECKING:
+    pass
+
+_logger = logging.getLogger('pixeltable')
+
+
+class TableVersionHandle:
+    """
+    Indirection mechanism for TableVersion instances, which get resolved against the catalog at runtime.
+    """
+
+    id: UUID
+    effective_version: Optional[int]
+    _tbl_version: Optional[TableVersion]
+
+    def __init__(self, tbl_id: UUID, effective_version: Optional[int], tbl_version: Optional[TableVersion] = None):
+        self.id = tbl_id
+        self.effective_version = effective_version
+        self._tbl_version = tbl_version
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, TableVersionHandle):
+            return False
+        return self.id == other.id and self.effective_version == other.effective_version
+
+    @classmethod
+    def create(cls, tbl_version: TableVersion) -> TableVersionHandle:
+        return cls(tbl_version.id, tbl_version.effective_version, tbl_version)
+
+    def get(self) -> TableVersion:
+        from .catalog import Catalog
+
+        if self._tbl_version is None:
+            self._tbl_version = Catalog.get().get_tbl_version(self.id, self.effective_version)
+        return self._tbl_version
+
+    def as_dict(self) -> dict:
+        return {'id': str(self.id), 'effective_version': self.effective_version}
+
+    @classmethod
+    def from_dict(cls, d: dict) -> TableVersionHandle:
+        return cls(UUID(d['id']), d['effective_version'])

pixeltable/catalog/table_version_path.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional, Union
+from typing import Optional
 from uuid import UUID
 
-import pixeltable as pxt
-from pixeltable import exprs
+from pixeltable.metadata import schema
 
 from .column import Column
-from .table_version import TableVersion
+from .table_version_handle import TableVersionHandle
 
 _logger = logging.getLogger('pixeltable')
 
@@ -25,22 +24,40 @@ class TableVersionPath:
     table/view.
     """
 
-    def __init__(self, tbl_version: TableVersion, base: Optional[TableVersionPath] = None):
+    tbl_version: TableVersionHandle
+    base: Optional[TableVersionPath]
+
+    def __init__(self, tbl_version: TableVersionHandle, base: Optional[TableVersionPath] = None):
         assert tbl_version is not None
         self.tbl_version = tbl_version
         self.base = base
 
+    @classmethod
+    def from_md(cls, path: schema.TableVersionPath) -> TableVersionPath:
+        assert len(path) > 0
+        result: Optional[TableVersionPath] = None
+        for tbl_id_str, effective_version in path[::-1]:
+            tbl_id = UUID(tbl_id_str)
+            result = TableVersionPath(TableVersionHandle(tbl_id, effective_version), base=result)
+        return result
+
+    def as_md(self) -> schema.TableVersionPath:
+        result = [(self.tbl_version.id.hex, self.tbl_version.effective_version)]
+        if self.base is not None:
+            result.extend(self.base.as_md())
+        return result
+
     def tbl_id(self) -> UUID:
         """Return the id of the table/view that this path represents"""
         return self.tbl_version.id
 
     def version(self) -> int:
         """Return the version of the table/view that this path represents"""
-        return self.tbl_version.version
+        return self.tbl_version.get().version
 
     def tbl_name(self) -> str:
         """Return the name of the table/view that this path represents"""
-        return self.tbl_version.name
+        return self.tbl_version.get().name
 
     def path_len(self) -> int:
         """Return the length of the path"""
@@ -48,32 +65,32 @@ class TableVersionPath:
 
     def is_snapshot(self) -> bool:
         """Return True if this is a path of snapshot versions"""
-        if not self.tbl_version.is_snapshot:
+        if not self.tbl_version.get().is_snapshot:
             return False
         return self.base.is_snapshot() if self.base is not None else True
 
     def is_view(self) -> bool:
-        return self.tbl_version.is_view()
+        return self.tbl_version.get().is_view
 
     def is_component_view(self) -> bool:
-        return self.tbl_version.is_component_view()
+        return self.tbl_version.get().is_component_view
 
     def is_insertable(self) -> bool:
-        return self.tbl_version.is_insertable()
+        return self.tbl_version.get().is_insertable()
 
-    def get_tbl_versions(self) -> list[TableVersion]:
+    def get_tbl_versions(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""
         if self.base is None:
             return [self.tbl_version]
         return [self.tbl_version] + self.base.get_tbl_versions()
 
-    def get_bases(self) -> list[TableVersion]:
+    def get_bases(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""
         if self.base is None:
             return []
         return self.base.get_tbl_versions()
 
-    def find_tbl_version(self, id: UUID) -> Optional[TableVersion]:
+    def find_tbl_version(self, id: UUID) -> Optional[TableVersionHandle]:
         """Return the matching TableVersion in the chain of TableVersions, starting with this one"""
         if self.tbl_version.id == id:
             return self.tbl_version
@@ -81,24 +98,13 @@ class TableVersionPath:
             return None
         return self.base.find_tbl_version(id)
 
-    def get_column_ref(self, col_name: str) -> exprs.ColumnRef:
-        """Return a ColumnRef for the given column name."""
-        from pixeltable.exprs import ColumnRef
-
-        if col_name not in self.tbl_version.cols_by_name:
-            if self.base is None or not self.tbl_version.include_base_columns:
-                raise AttributeError(f'Column {col_name} unknown')
-            return self.base.get_column_ref(col_name)
-        col = self.tbl_version.cols_by_name[col_name]
-        return ColumnRef(col)
-
     def columns(self) -> list[Column]:
         """Return all user columns visible in this tbl version path, including columns from bases"""
-        result = list(self.tbl_version.cols_by_name.values())
-        if self.base is not None and self.tbl_version.include_base_columns:
+        result = list(self.tbl_version.get().cols_by_name.values())
+        if self.base is not None and self.tbl_version.get().include_base_columns:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
-            result.extend(c for c in base_cols if c.name not in self.tbl_version.cols_by_name)
+            result.extend(c for c in base_cols if c.name not in self.tbl_version.get().cols_by_name)
         return result
 
     def cols_by_name(self) -> dict[str, Column]:
@@ -111,23 +117,33 @@ class TableVersionPath:
         cols = self.columns()
         return {col.id: col for col in cols}
 
-    def get_column(self, name: str, include_bases: bool = True) -> Optional[Column]:
+    def get_column(self, name: str, include_bases: Optional[bool] = None) -> Optional[Column]:
         """Return the column with the given name, or None if not found"""
-        col = self.tbl_version.cols_by_name.get(name)
+        col = self.tbl_version.get().cols_by_name.get(name)
         if col is not None:
             return col
-        elif self.base is not None and include_bases:
+        elif self.base is not None and (include_bases or self.tbl_version.get().include_base_columns):
             return self.base.get_column(name)
         else:
             return None
 
+    def get_column_by_id(self, tbl_id: UUID, col_id: int) -> Optional[Column]:
+        """Return the column for the given tbl/col id"""
+        if self.tbl_version.id == tbl_id:
+            assert col_id in self.tbl_version.get().cols_by_id
+            return self.tbl_version.get().cols_by_id[col_id]
+        elif self.base is not None:
+            return self.base.get_column_by_id(tbl_id, col_id)
+        else:
+            return None
+
     def has_column(self, col: Column, include_bases: bool = True) -> bool:
         """Return True if this table has the given column."""
         assert col.tbl is not None
         if (
             col.tbl.id == self.tbl_version.id
             and col.tbl.effective_version == self.tbl_version.effective_version
-            and col.id in self.tbl_version.cols_by_id
+            and col.id in self.tbl_version.get().cols_by_id
         ):
             # the column is visible in this table version
             return True
@@ -144,6 +160,6 @@ class TableVersionPath:
 
     @classmethod
     def from_dict(cls, d: dict) -> TableVersionPath:
-        tbl_version = TableVersion.from_dict(d['tbl_version'])
+        tbl_version = TableVersionHandle.from_dict(d['tbl_version'])
         base = TableVersionPath.from_dict(d['base']) if d['base'] is not None else None
         return cls(tbl_version, base)

pixeltable/catalog/view.py

@@ -5,8 +5,6 @@ import logging
 from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional
 from uuid import UUID
 
-import sqlalchemy.orm as orm
-
 import pixeltable.exceptions as excs
 import pixeltable.metadata.schema as md_schema
 import pixeltable.type_system as ts
@@ -14,11 +12,11 @@ from pixeltable import catalog, exprs, func
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 
-from .catalog import Catalog
 from .column import Column
 from .globals import _POS_COLUMN_NAME, MediaValidation, UpdateStatus
 from .table import Table
 from .table_version import TableVersion
+from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 
 if TYPE_CHECKING:
@@ -36,19 +34,8 @@ class View(Table):
     is simply a reference to a specific set of base versions.
     """
 
-    def __init__(
-        self,
-        id: UUID,
-        dir_id: UUID,
-        name: str,
-        tbl_version_path: TableVersionPath,
-        base_id: UUID,
-        snapshot_only: bool,
-        include_base_columns: bool,
-    ):
+    def __init__(self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath, snapshot_only: bool):
         super().__init__(id, dir_id, name, tbl_version_path)
-        assert base_id in catalog.Catalog.get().tbl_dependents
-        self._base_id = base_id  # keep a reference to the base Table ID, so that we can keep track of its dependents
         self._snapshot_only = snapshot_only
 
     @classmethod
@@ -162,87 +149,66 @@ class View(Table):
                     )
             columns = iterator_cols + columns
 
-        with orm.Session(Env.get().engine, future=True) as session:
-            from pixeltable.exprs import InlineDict
+        session = Env.get().session
+        from pixeltable.exprs import InlineDict
 
-            iterator_args_expr: exprs.Expr = InlineDict(iterator_args) if iterator_args is not None else None
-            iterator_class_fqn = (
-                f'{iterator_cls.__module__}.{iterator_cls.__name__}' if iterator_cls is not None else None
-            )
-            base_version_path = cls._get_snapshot_path(base) if is_snapshot else base
-            base_versions = [
-                (tbl_version.id.hex, tbl_version.version if is_snapshot or tbl_version.is_snapshot else None)
-                for tbl_version in base_version_path.get_tbl_versions()
-            ]
+        iterator_args_expr: exprs.Expr = InlineDict(iterator_args) if iterator_args is not None else None
+        iterator_class_fqn = f'{iterator_cls.__module__}.{iterator_cls.__name__}' if iterator_cls is not None else None
+        base_version_path = cls._get_snapshot_path(base) if is_snapshot else base
 
-            # if this is a snapshot, we need to retarget all exprs to the snapshot tbl versions
-            if is_snapshot:
-                predicate = predicate.retarget(base_version_path) if predicate is not None else None
-                iterator_args_expr = (
-                    iterator_args_expr.retarget(base_version_path) if iterator_args_expr is not None else None
-                )
-                for col in columns:
-                    if col.value_expr is not None:
-                        col.set_value_expr(col.value_expr.retarget(base_version_path))
-
-            view_md = md_schema.ViewMd(
-                is_snapshot=is_snapshot,
-                include_base_columns=include_base_columns,
-                predicate=predicate.as_dict() if predicate is not None else None,
-                base_versions=base_versions,
-                iterator_class_fqn=iterator_class_fqn,
-                iterator_args=iterator_args_expr.as_dict() if iterator_args_expr is not None else None,
+        # if this is a snapshot, we need to retarget all exprs to the snapshot tbl versions
+        if is_snapshot:
+            predicate = predicate.retarget(base_version_path) if predicate is not None else None
+            iterator_args_expr = (
+                iterator_args_expr.retarget(base_version_path) if iterator_args_expr is not None else None
             )
+            for col in columns:
+                if col.value_expr is not None:
+                    col.set_value_expr(col.value_expr.retarget(base_version_path))
+
+        view_md = md_schema.ViewMd(
+            is_snapshot=is_snapshot,
+            include_base_columns=include_base_columns,
+            predicate=predicate.as_dict() if predicate is not None else None,
+            base_versions=base_version_path.as_md(),
+            iterator_class_fqn=iterator_class_fqn,
+            iterator_args=iterator_args_expr.as_dict() if iterator_args_expr is not None else None,
+        )
 
-            id, tbl_version = TableVersion.create(
-                session,
+        id, tbl_version = TableVersion.create(
+            dir_id,
+            name,
+            columns,
+            num_retained_versions,
+            comment,
+            media_validation=media_validation,
+            # base_path=base_version_path,
+            view_md=view_md,
+        )
+        if tbl_version is None:
+            # this is purely a snapshot: we use the base's tbl version path
+            view = cls(id, dir_id, name, base_version_path, snapshot_only=True)
+            _logger.info(f'created snapshot {name}')
+        else:
+            view = cls(
+                id,
                 dir_id,
                 name,
-                columns,
-                num_retained_versions,
-                comment,
-                media_validation=media_validation,
-                base_path=base_version_path,
-                view_md=view_md,
+                TableVersionPath(
+                    TableVersionHandle(tbl_version.id, tbl_version.effective_version), base=base_version_path
+                ),
+                snapshot_only=False,
             )
-            if tbl_version is None:
-                # this is purely a snapshot: we use the base's tbl version path
-                view = cls(
-                    id,
-                    dir_id,
-                    name,
-                    base_version_path,
-                    base.tbl_id(),
-                    snapshot_only=True,
-                    include_base_columns=include_base_columns,
-                )
-                _logger.info(f'created snapshot {name}')
-            else:
-                view = cls(
-                    id,
-                    dir_id,
-                    name,
-                    TableVersionPath(tbl_version, base=base_version_path),
-                    base.tbl_id(),
-                    snapshot_only=False,
-                    include_base_columns=include_base_columns,
-                )
-                _logger.info(f'Created view `{name}`, id={tbl_version.id}')
+            _logger.info(f'Created view `{name}`, id={tbl_version.id}')
 
-                from pixeltable.plan import Planner
+            from pixeltable.plan import Planner
 
-                plan, num_values_per_row = Planner.create_view_load_plan(view._tbl_version_path)
-                num_rows, num_excs, cols_with_excs = tbl_version.store_tbl.insert_rows(
-                    plan, session.connection(), v_min=tbl_version.version
-                )
-                Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
+            plan, num_values_per_row = Planner.create_view_load_plan(view._tbl_version_path)
+            num_rows, num_excs, cols_with_excs = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+            Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
 
-            session.commit()
-            cat = Catalog.get()
-            cat.tbl_dependents[view._id] = []
-            cat.tbl_dependents[base.tbl_id()].append(view)
-            cat.tbls[view._id] = view
-            return view
+        session.commit()
+        return view
 
     @classmethod
     def _verify_column(cls, col: Column) -> None:
@@ -259,35 +225,29 @@ class View(Table):
         """
         if tbl_version_path.is_snapshot():
             return tbl_version_path
-        tbl_version = tbl_version_path.tbl_version
+        tbl_version = tbl_version_path.tbl_version.get()
         if not tbl_version.is_snapshot:
             # create and register snapshot version
             tbl_version = tbl_version.create_snapshot_copy()
             assert tbl_version.is_snapshot
 
         return TableVersionPath(
-            tbl_version,
+            TableVersionHandle(tbl_version.id, tbl_version.effective_version),
             base=cls._get_snapshot_path(tbl_version_path.base) if tbl_version_path.base is not None else None,
         )
 
     def _drop(self) -> None:
         cat = catalog.Catalog.get()
-        # verify all dependents are deleted by now
-        for dep in cat.tbl_dependents[self._id]:
-            assert dep._is_dropped
         if self._snapshot_only:
             # there is not TableVersion to drop
             self._check_is_dropped()
             self.is_dropped = True
-            with Env.get().engine.begin() as conn:
-                TableVersion.delete_md(self._id, conn)
+            TableVersion.delete_md(self._id)
             # update catalog
             cat = catalog.Catalog.get()
-            del cat.tbls[self._id]
+            cat.remove_tbl(self._id)
         else:
             super()._drop()
-        cat.tbl_dependents[self._base_id].remove(self)
-        del cat.tbl_dependents[self._id]
 
     def get_metadata(self) -> dict[str, Any]:
         md = super().get_metadata()

pixeltable/config.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Any, ClassVar, Optional, TypeVar
+
+import toml
+
+from pixeltable import exceptions as excs
+
+_logger = logging.getLogger('pixeltable')
+
+T = TypeVar('T')
+
+
+class Config:
+    """
+    The (global) Pixeltable configuration, as loaded from `config.toml`. Provides methods for retrieving
+    configuration values, which can be set in the config file or as environment variables.
+    """
+
+    __instance: ClassVar[Optional[Config]] = None
+
+    __home: Path
+    __config_file: Path
+    __config_dict: dict[str, Any]
+
+    def __init__(self) -> None:
+        assert self.__instance is None, 'Config is a singleton; use Config.get() to access the instance'
+
+        self.__home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
+        if self.__home.exists() and not self.__home.is_dir():
+            raise RuntimeError(f'{self.__home} is not a directory')
+        if not self.__home.exists():
+            print(f'Creating a Pixeltable instance at: {self.__home}')
+            self.__home.mkdir()
+
+        self.__config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self.__home / 'config.toml')))
+
+        self.__config_dict: dict[str, Any]
+        if os.path.isfile(self.__config_file):
+            with open(self.__config_file, 'r', encoding='utf-8') as stream:
+                try:
+                    self.__config_dict = toml.load(stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not read config file: {self.__config_file}') from exc
+        else:
+            self.__config_dict = self.__create_default_config(self.__config_file)
+            with open(self.__config_file, 'w', encoding='utf-8') as stream:
+                try:
+                    toml.dump(self.__config_dict, stream)
+                except Exception as exc:
+                    raise excs.Error(f'Could not write config file: {self.__config_file}') from exc
+            _logger.info(f'Created default config file at: {self.__config_file}')
+
+    @property
+    def home(self) -> Path:
+        return self.__home
+
+    @property
+    def config_file(self) -> Path:
+        return self.__config_file
+
+    @classmethod
+    def get(cls) -> Config:
+        if cls.__instance is None:
+            cls.__instance = cls()
+        return cls.__instance
+
+    @classmethod
+    def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
+        free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
+        # Default cache size is 1/5 of free disk space
+        file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
+        return {'pixeltable': {'file_cache_size_g': round(file_cache_size_g, 1), 'hide_warnings': False}}
+
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+        env_var = f'{section.upper()}_{key.upper()}'
+        if env_var in os.environ:
+            value = os.environ[env_var]
+        elif section in self.__config_dict and key in self.__config_dict[section]:
+            value = self.__config_dict[section][key]
+        else:
+            return None
+
+        try:
+            return expected_type(value)  # type: ignore[call-arg]
+        except ValueError as exc:
+            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}') from exc
+
+    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
+        return self.get_value(key, str, section)
+
+    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
+        return self.get_value(key, int, section)
+
+    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
+        return self.get_value(key, float, section)
+
+    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
+        return self.get_value(key, bool, section)