pixeltable 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

pixeltable/utils/formatter.py

@@ -4,7 +4,7 @@ import io
 import json
 import logging
 import mimetypes
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import av
 import numpy as np
@@ -39,7 +39,7 @@ class Formatter:
         self.__num_cols = num_cols
         self.__http_address = http_address
 
-    def get_pandas_formatter(self, col_type: ts.ColumnType) -> Optional[Callable]:
+    def get_pandas_formatter(self, col_type: ts.ColumnType) -> Callable | None:
         if col_type.is_string_type():
             return self.format_string
         if col_type.is_float_type():
@@ -184,7 +184,7 @@ class Formatter:
         """
 
     @classmethod
-    def extract_first_video_frame(cls, file_path: str) -> Optional[Image.Image]:
+    def extract_first_video_frame(cls, file_path: str) -> Image.Image | None:
         with av.open(file_path) as container:
             try:
                 img = next(container.decode(video=0)).to_image()
@@ -224,9 +224,7 @@ class Formatter:
         """
 
     @classmethod
-    def make_document_thumbnail(
-        cls, file_path: str, max_width: int = 320, max_height: int = 320
-    ) -> Optional[Image.Image]:
+    def make_document_thumbnail(cls, file_path: str, max_width: int = 320, max_height: int = 320) -> Image.Image | None:
         """
         Returns a thumbnail image of a document.
         """

pixeltable/utils/gcs_store.py

@@ -5,7 +5,7 @@ import re
 import urllib.parse
 import uuid
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterator, Optional
+from typing import TYPE_CHECKING, Any, Iterator
 
 from google.api_core.exceptions import GoogleAPIError
 from google.cloud import storage  # type: ignore[attr-defined]
@@ -81,7 +81,7 @@ class GCSStore(ObjectStoreBase):
         """Return the prefix from the base URI."""
         return self.__prefix_name
 
-    def validate(self, error_col_name: str) -> Optional[str]:
+    def validate(self, error_col_name: str) -> str | None:
         """
         Checks if the URI exists.
 
@@ -99,7 +99,7 @@ class GCSStore(ObjectStoreBase):
             self.handle_gcs_error(e, self.bucket_name, f'validate bucket {error_col_name}')
         return None
 
-    def _prepare_uri_raw(self, tbl_id: uuid.UUID, col_id: int, tbl_version: int, ext: Optional[str] = None) -> str:
+    def _prepare_uri_raw(self, tbl_id: uuid.UUID, col_id: int, tbl_version: int, ext: str | None = None) -> str:
         """
         Construct a new, unique URI for a persisted media file.
         """
@@ -107,12 +107,12 @@ class GCSStore(ObjectStoreBase):
         parent = f'{self.__base_uri}{prefix}'
         return f'{parent}/{filename}'
 
-    def _prepare_uri(self, col: Column, ext: Optional[str] = None) -> str:
+    def _prepare_uri(self, col: Column, ext: str | None = None) -> str:
         """
         Construct a new, unique URI for a persisted media file.
         """
-        assert col.tbl is not None, 'Column must be associated with a table'
-        return self._prepare_uri_raw(col.tbl.id, col.id, col.tbl.version, ext=ext)
+        assert col.get_tbl() is not None, 'Column must be associated with a table'
+        return self._prepare_uri_raw(col.get_tbl().id, col.id, col.get_tbl().version, ext=ext)
 
     def copy_local_file(self, col: Column, src_path: Path) -> str:
         """Copy a local file, and return its new URL"""
@@ -142,7 +142,7 @@ class GCSStore(ObjectStoreBase):
             self.handle_gcs_error(e, self.bucket_name, f'download file {src_path}')
             raise
 
-    def _get_filtered_objects(self, bucket: Any, tbl_id: uuid.UUID, tbl_version: Optional[int] = None) -> Iterator:
+    def _get_filtered_objects(self, bucket: Any, tbl_id: uuid.UUID, tbl_version: int | None = None) -> Iterator:
         """Private method to get filtered objects for a table, optionally filtered by version.
 
         Args:
@@ -168,7 +168,7 @@ class GCSStore(ObjectStoreBase):
 
         return blob_iterator
 
-    def count(self, tbl_id: uuid.UUID, tbl_version: Optional[int] = None) -> int:
+    def count(self, tbl_id: uuid.UUID, tbl_version: int | None = None) -> int:
         """Count the number of files belonging to tbl_id. If tbl_version is not None,
         count only those files belonging to the specified tbl_version.
 
@@ -193,7 +193,7 @@ class GCSStore(ObjectStoreBase):
             self.handle_gcs_error(e, self.bucket_name, f'setup iterator {self.prefix}')
             raise
 
-    def delete(self, tbl_id: uuid.UUID, tbl_version: Optional[int] = None) -> int:
+    def delete(self, tbl_id: uuid.UUID, tbl_version: int | None = None) -> int:
         """Delete all files belonging to tbl_id. If tbl_version is not None, delete
         only those files belonging to the specified tbl_version.
 

pixeltable/utils/local_store.py

@@ -10,7 +10,7 @@ import urllib.request
 import uuid
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 import PIL.Image
@@ -35,7 +35,7 @@ class LocalStore(ObjectStoreBase):
 
     __base_dir: Path
 
-    soa: Optional[StorageObjectAddress]
+    soa: StorageObjectAddress | None
 
     def __init__(self, location: Path | StorageObjectAddress):
         if isinstance(location, Path):
@@ -69,7 +69,7 @@ class LocalStore(ObjectStoreBase):
             raise excs.Error(f'{error_col_name}`destination` must be a valid path. Error: {e}') from None
 
     @staticmethod
-    def file_url_to_path(url: str) -> Optional[Path]:
+    def file_url_to_path(url: str) -> Path | None:
         """Convert a file:// URI to a Path object with support for Windows UNC paths."""
         assert isinstance(url, str), type(url)
         parsed = urllib.parse.urlparse(url)
@@ -90,7 +90,7 @@ class LocalStore(ObjectStoreBase):
         return Path(path_str)
 
     @classmethod
-    def _save_binary_media_file(cls, file_data: bytes, dest_path: Path, format: Optional[str]) -> Path:
+    def _save_binary_media_file(cls, file_data: bytes, dest_path: Path, format: str | None) -> Path:
         """Save binary data to a file in a LocalStore. format is ignored for binary data."""
         assert isinstance(file_data, bytes)
         with open(dest_path, 'wb') as f:
@@ -100,7 +100,7 @@ class LocalStore(ObjectStoreBase):
         return dest_path
 
     @classmethod
-    def _save_pil_image_file(cls, image: PIL.Image.Image, dest_path: Path, format: Optional[str]) -> Path:
+    def _save_pil_image_file(cls, image: PIL.Image.Image, dest_path: Path, format: str | None) -> Path:
         """Save a PIL Image to a file in a LocalStore with the specified format."""
         if dest_path.suffix != f'.{format}':
             dest_path = dest_path.with_name(f'{dest_path.name}.{format}')
@@ -111,7 +111,7 @@ class LocalStore(ObjectStoreBase):
             os.fsync(f.fileno())  # Forces OS to write to physical storage
         return dest_path
 
-    def _prepare_path_raw(self, tbl_id: UUID, col_id: int, tbl_version: int, ext: Optional[str] = None) -> Path:
+    def _prepare_path_raw(self, tbl_id: UUID, col_id: int, tbl_version: int, ext: str | None = None) -> Path:
         """
         Construct a new, unique Path name in the __base_dir for a persisted file.
         Create the parent directory for the new Path if it does not already exist.
@@ -121,19 +121,19 @@ class LocalStore(ObjectStoreBase):
         parent.mkdir(parents=True, exist_ok=True)
         return parent / filename
 
-    def _prepare_path(self, col: Column, ext: Optional[str] = None) -> Path:
+    def _prepare_path(self, col: Column, ext: str | None = None) -> Path:
         """
         Construct a new, unique Path name in the __base_dir for a persisted file.
         Create the parent directory for the new Path if it does not already exist.
         """
-        assert col.tbl is not None, 'Column must be associated with a table'
-        return self._prepare_path_raw(col.tbl.id, col.id, col.tbl.version, ext)
+        assert col.get_tbl() is not None, 'Column must be associated with a table'
+        return self._prepare_path_raw(col.get_tbl().id, col.id, col.get_tbl().version, ext)
 
     def contains_path(self, file_path: Path) -> bool:
         """Return True if the given path refers to a file managed by this LocalStore, else False."""
         return str(file_path).startswith(str(self.__base_dir))
 
-    def resolve_url(self, file_url: Optional[str]) -> Optional[Path]:
+    def resolve_url(self, file_url: str | None) -> Path | None:
         """Return path if the given url refers to a file managed by this LocalStore, else None.
 
         Args:
@@ -168,7 +168,7 @@ class LocalStore(ObjectStoreBase):
         _logger.debug(f'Media Storage: copied {src_path} to {new_file_url}')
         return new_file_url
 
-    def save_media_object(self, data: bytes | PIL.Image.Image, col: Column, format: Optional[str]) -> tuple[Path, str]:
+    def save_media_object(self, data: bytes | PIL.Image.Image, col: Column, format: str | None) -> tuple[Path, str]:
         """Save a data object to a file in a LocalStore
         Returns:
             dest_path: Path to the saved file
@@ -185,7 +185,7 @@ class LocalStore(ObjectStoreBase):
         new_file_url = urllib.parse.urljoin('file:', urllib.request.pathname2url(str(dest_path)))
         return dest_path, new_file_url
 
-    def delete(self, tbl_id: UUID, tbl_version: Optional[int] = None) -> Optional[int]:
+    def delete(self, tbl_id: UUID, tbl_version: int | None = None) -> int | None:
         """Delete all files belonging to tbl_id. If tbl_version is not None, delete
         only those files belonging to the specified tbl_version.
 
@@ -209,7 +209,7 @@ class LocalStore(ObjectStoreBase):
                 os.remove(p)
             return len(paths)
 
-    def count(self, tbl_id: Optional[UUID], tbl_version: Optional[int] = None) -> int:
+    def count(self, tbl_id: UUID | None, tbl_version: int | None = None) -> int:
         """
         Return number of files for given tbl_id.
         """
@@ -277,7 +277,7 @@ class TempStore:
         return env.Env.get().tmp_dir
 
     @classmethod
-    def count(cls, tbl_id: Optional[UUID] = None, tbl_version: Optional[int] = None) -> int:
+    def count(cls, tbl_id: UUID | None = None, tbl_version: int | None = None) -> int:
         return LocalStore(cls._tmp_dir()).count(tbl_id, tbl_version)
 
     @classmethod
@@ -285,11 +285,11 @@ class TempStore:
         return LocalStore(cls._tmp_dir()).contains_path(file_path)
 
     @classmethod
-    def resolve_url(cls, file_url: Optional[str]) -> Optional[Path]:
+    def resolve_url(cls, file_url: str | None) -> Path | None:
         return LocalStore(cls._tmp_dir()).resolve_url(file_url)
 
     @classmethod
-    def save_media_object(cls, data: bytes | PIL.Image.Image, col: Column, format: Optional[str]) -> tuple[Path, str]:
+    def save_media_object(cls, data: bytes | PIL.Image.Image, col: Column, format: str | None) -> tuple[Path, str]:
         return LocalStore(cls._tmp_dir()).save_media_object(data, col, format)
 
     @classmethod
@@ -302,7 +302,7 @@ class TempStore:
         _logger.debug(f'Media Storage: deleted {file_path}')
 
     @classmethod
-    def create_path(cls, tbl_id: Optional[UUID] = None, extension: str = '') -> Path:
+    def create_path(cls, tbl_id: UUID | None = None, extension: str = '') -> Path:
         """Return a new, unique Path located in the temporary store.
         If tbl_id is provided, the path name will be similar to a LocalStore path based on the tbl_id.
         If tbl_id is None, a random UUID will be used to create the path."""

pixeltable/utils/object_stores.py

@@ -7,7 +7,7 @@ import urllib.parse
 import urllib.request
 import uuid
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, NamedTuple, Optional
+from typing import TYPE_CHECKING, NamedTuple
 from uuid import UUID
 
 from pixeltable import env, exceptions as excs
@@ -44,7 +44,7 @@ class StorageObjectAddress(NamedTuple):
     key: str = ''  # Key parsed from the source (prefix + object_name)
     prefix: str = ''  # Prefix (within the bucket) parsed from the source
     object_name: str = ''  # Object name parsed from the source (if requested and applicable)
-    path: Optional[Path] = None
+    path: Path | None = None
 
     @property
     def has_object(self) -> bool:
@@ -56,11 +56,11 @@ class StorageObjectAddress(NamedTuple):
 
     @property
     def is_azure_scheme(self) -> bool:
-        return self.scheme in ['wasb', 'wasbs', 'abfs', 'abfss']
+        return self.scheme in ('wasb', 'wasbs', 'abfs', 'abfss')
 
     @property
     def has_valid_storage_target(self) -> bool:
-        return self.storage_target in [
+        return self.storage_target in (
             StorageTarget.LOCAL_STORE,
             StorageTarget.S3_STORE,
             StorageTarget.R2_STORE,
@@ -68,7 +68,7 @@ class StorageObjectAddress(NamedTuple):
             StorageTarget.GCS_STORE,
             StorageTarget.AZURE_STORE,
             StorageTarget.HTTP_STORE,
-        ]
+        )
 
     @property
     def prefix_free_uri(self) -> str:
@@ -120,9 +120,7 @@ class ObjectPath:
         return tbl_id.hex
 
     @classmethod
-    def create_prefix_raw(
-        cls, tbl_id: UUID, col_id: int, tbl_version: int, ext: Optional[str] = None
-    ) -> tuple[str, str]:
+    def create_prefix_raw(cls, tbl_id: UUID, col_id: int, tbl_version: int, ext: str | None = None) -> tuple[str, str]:
         """Construct a unique unix-style prefix and filename for a persisted file.
         The results are derived from table, col, and version specs.
         Returns:
@@ -202,7 +200,7 @@ class ObjectPath:
             container = parsed.netloc
             key = parsed.path.lstrip('/')
 
-        elif scheme in ['wasb', 'wasbs', 'abfs', 'abfss']:
+        elif scheme in ('wasb', 'wasbs', 'abfs', 'abfss'):
             # Azure-specific URI schemes
             # wasb[s]://container@account.blob.core.windows.net/<optional prefix>/<optional object>
             # abfs[s]://container@account.dfs.core.windows.net/<optional prefix>/<optional object>
@@ -216,7 +214,7 @@ class ObjectPath:
                 raise ValueError(f'Invalid Azure URI format: {src_addr}')
             key = parsed.path.lstrip('/')
 
-        elif scheme in ['http', 'https']:
+        elif scheme in ('http', 'https'):
             # Standard HTTP(S) URL format
             # https://account.blob.core.windows.net/container/<optional path>/<optional object>
             # https://account.r2.cloudflarestorage.com/container/<optional path>/<optional object>
@@ -253,7 +251,7 @@ class ObjectPath:
         return r
 
     @classmethod
-    def parse_object_storage_addr(cls, src_addr: str, may_contain_object_name: bool) -> StorageObjectAddress:
+    def parse_object_storage_addr(cls, src_addr: str, allow_obj_name: bool) -> StorageObjectAddress:
         """
         Parses a cloud storage URI into its scheme, bucket, prefix, and object name.
 
@@ -273,14 +271,14 @@ class ObjectPath:
             https://raw.github.com/pixeltable/pixeltable/main/docs/resources/images/000000000030.jpg
         """
         soa = cls.parse_object_storage_addr1(src_addr)
-        prefix, object_name = cls.separate_prefix_object(soa.key, may_contain_object_name)
+        prefix, object_name = cls.separate_prefix_object(soa.key, allow_obj_name)
         assert not object_name.endswith('/')
         r = soa._replace(prefix=prefix, object_name=object_name)
         return r
 
 
 class ObjectStoreBase:
-    def validate(self, error_col_name: str) -> Optional[str]:
+    def validate(self, error_prefix: str) -> str | None:
         """Check the store configuration. Returns base URI if store is accessible.
 
         Args:
@@ -303,7 +301,7 @@ class ObjectStoreBase:
         """
         raise AssertionError
 
-    def move_local_file(self, col: Column, src_path: Path) -> Optional[str]:
+    def move_local_file(self, col: Column, src_path: Path) -> str | None:
         """Move a file associated with a Column to the store, returning the file's URL within the destination.
 
         Args:
@@ -324,7 +322,7 @@ class ObjectStoreBase:
         """
         raise AssertionError
 
-    def count(self, tbl_id: UUID, tbl_version: Optional[int] = None) -> int:
+    def count(self, tbl_id: UUID, tbl_version: int | None = None) -> int:
         """Return the number of objects in the store associated with the given tbl_id
 
         Args:
@@ -336,7 +334,7 @@ class ObjectStoreBase:
         """
         raise AssertionError
 
-    def delete(self, tbl_id: UUID, tbl_version: Optional[int] = None) -> Optional[int]:
+    def delete(self, tbl_id: UUID, tbl_version: int | None = None) -> int | None:
         """Delete objects in the destination for a given table ID, table version.
 
         Args:
@@ -360,28 +358,15 @@ class ObjectStoreBase:
 
 class ObjectOps:
     @classmethod
-    def get_store(cls, dest: Optional[str], may_contain_object_name: bool, col_name: Optional[str] = None) -> Any:
+    def get_store(cls, dest: str | None, allow_obj_name: bool, col_name: str | None = None) -> ObjectStoreBase:
         from pixeltable.env import Env
         from pixeltable.utils.local_store import LocalStore
 
-        soa = (
-            Env.get().object_soa
-            if dest is None
-            else ObjectPath.parse_object_storage_addr(dest, may_contain_object_name=may_contain_object_name)
-        )
+        dest = dest or str(Env.get().media_dir)  # Use local media dir as fallback
+        soa = ObjectPath.parse_object_storage_addr(dest, allow_obj_name=allow_obj_name)
         if soa.storage_target == StorageTarget.LOCAL_STORE:
             return LocalStore(soa)
-        if soa.storage_target == StorageTarget.S3_STORE and soa.scheme == 's3':
-            env.Env.get().require_package('boto3')
-            from pixeltable.utils.s3_store import S3Store
-
-            return S3Store(soa)
-        if soa.storage_target == StorageTarget.R2_STORE:
-            env.Env.get().require_package('boto3')
-            from pixeltable.utils.s3_store import S3Store
-
-            return S3Store(soa)
-        if soa.storage_target == StorageTarget.B2_STORE:
+        if soa.storage_target in (StorageTarget.S3_STORE, StorageTarget.R2_STORE, StorageTarget.B2_STORE):
             env.Env.get().require_package('boto3')
             from pixeltable.utils.s3_store import S3Store
 
@@ -391,6 +376,11 @@ class ObjectOps:
             from pixeltable.utils.gcs_store import GCSStore
 
             return GCSStore(soa)
+        if soa.storage_target == StorageTarget.AZURE_STORE:
+            env.Env.get().require_package('azure.storage.blob')
+            from pixeltable.utils.azure_store import AzureBlobStore
+
+            return AzureBlobStore(soa)
         if soa.storage_target == StorageTarget.HTTP_STORE and soa.is_http_readable:
             return HTTPStore(soa)
         error_col_name = f'Column {col_name!r}: ' if col_name is not None else ''
@@ -399,7 +389,7 @@ class ObjectOps:
         )
 
     @classmethod
-    def validate_destination(cls, dest: str | Path | None, col_name: Optional[str]) -> str:
+    def validate_destination(cls, dest: str | Path | None, col_name: str | None = None) -> str:
         """Convert a Column destination parameter to a URI, else raise errors.
         Args:
             dest: The requested destination
@@ -407,19 +397,19 @@ class ObjectOps:
         Returns:
             URI of destination, or raises an error
         """
-        error_col_name = f'Column {col_name!r}: ' if col_name is not None else ''
+        error_col_str = f'column {col_name!r}' if col_name is not None else ''
 
         # General checks on any destination
         if isinstance(dest, Path):
             dest = str(dest)
         if dest is not None and not isinstance(dest, str):
-            raise excs.Error(f'{error_col_name}`destination` must be a string or path, got {dest!r}')
+            raise excs.Error(f'{error_col_str}: `destination` must be a string or path; got {dest!r}')
 
         # Specific checks for storage backends
         store = cls.get_store(dest, False, col_name)
-        dest2 = store.validate(error_col_name)
+        dest2 = store.validate(error_col_str)
         if dest2 is None:
-            raise excs.Error(f'{error_col_name}`destination` must be a supported destination, got {dest!r}')
+            raise excs.Error(f'{error_col_str}: `destination` must be a supported destination; got {dest!r}')
         return dest2
 
     @classmethod
@@ -427,7 +417,7 @@ class ObjectOps:
         """Copy an object from a URL to a local Path. Thread safe.
         Raises an exception if the download fails or the scheme is not supported
         """
-        soa = ObjectPath.parse_object_storage_addr(src_uri, may_contain_object_name=True)
+        soa = ObjectPath.parse_object_storage_addr(src_uri, allow_obj_name=True)
         store = cls.get_store(src_uri, True)
         store.copy_object_to_local_file(soa.object_name, dest_path)
 
@@ -466,7 +456,7 @@ class ObjectOps:
         return store.copy_local_file(col, src_path)
 
     @classmethod
-    def delete(cls, dest: Optional[str], tbl_id: UUID, tbl_version: Optional[int] = None) -> Optional[int]:
+    def delete(cls, dest: str | None, tbl_id: UUID, tbl_version: int | None = None) -> int | None:
         """Delete objects in the destination for a given table ID, table version.
         Returns:
             Number of objects deleted or None
@@ -475,13 +465,39 @@ class ObjectOps:
         return store.delete(tbl_id, tbl_version)
 
     @classmethod
-    def count(cls, dest: Optional[str], tbl_id: UUID, tbl_version: Optional[int] = None) -> int:
-        """Return the count of objects in the destination for a given table ID"""
+    def count(
+        cls,
+        tbl_id: UUID,
+        tbl_version: int | None = None,
+        dest: str | None = None,
+        default_input_dest: bool = False,
+        default_output_dest: bool = False,
+    ) -> int:
+        """
+        Return the count of objects in the destination for a given table ID.
+
+        At most one of dest, default_input, default_output may be specified. If none are specified, the fallback is the
+        local media directory.
+
+        Args:
+            tbl_id: Table ID for which to count objects
+            tbl_version: If specified, only counts objects for a specific table version
+            dest: The destination to count objects in
+            default_input_dest: If `True`, use the default input media destination
+            default_output_dest: If `True`, use the default output media destination
+        """
+        assert sum((dest is not None, default_input_dest, default_output_dest)) <= 1, (
+            'At most one of dest, default_input, default_output may be specified'
+        )
+        if default_input_dest:
+            dest = env.Env.get().default_input_media_dest
+        if default_output_dest:
+            dest = env.Env.get().default_output_media_dest
         store = cls.get_store(dest, False)
         return store.count(tbl_id, tbl_version)
 
     @classmethod
-    def list_objects(cls, dest: Optional[str], return_uri: bool, n_max: int = 10) -> list[str]:
+    def list_objects(cls, dest: str | None, return_uri: bool, n_max: int = 10) -> list[str]:
         """Return a list of objects found in the specified destination bucket.
         The dest specification string must not contain an object name.
         Each returned object includes the full set of prefixes.