pixeltable 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

pixeltable/exec/expr_eval/globals.py

@@ -4,7 +4,7 @@ import abc
 import asyncio
 from dataclasses import dataclass
 from types import TracebackType
-from typing import Any, Iterable, Optional, Protocol
+from typing import Any, Iterable, Protocol
 
 import numpy as np
 
@@ -18,11 +18,11 @@ class FnCallArgs:
     fn_call: exprs.FunctionCall
     rows: list[exprs.DataRow]
     # single call
-    args: Optional[list[Any]] = None
-    kwargs: Optional[dict[str, Any]] = None
+    args: list[Any] | None = None
+    kwargs: dict[str, Any] | None = None
     # batch call
-    batch_args: Optional[list[list[Optional[Any]]]] = None
-    batch_kwargs: Optional[dict[str, list[Optional[Any]]]] = None
+    batch_args: list[list[Any | None]] | None = None
+    batch_kwargs: dict[str, list[Any | None]] | None = None
 
     @property
     def pxt_fn(self) -> func.CallableFunction:
@@ -56,7 +56,7 @@ class Scheduler(abc.ABC):
         request: FnCallArgs
         num_retries: int
         exec_ctx: ExecCtx
-        retry_after: Optional[float] = None  # time.monotonic()
+        retry_after: float | None = None  # time.monotonic()
 
         def __lt__(self, other: Scheduler.QueueItem) -> bool:
             # prioritize by number of retries (more retries = higher priority)

pixeltable/exec/expr_eval/row_buffer.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional
 
 import numpy as np
 
@@ -14,7 +13,7 @@ class RowBuffer:
     """Fixed-length circular buffer of DataRows; knows how to maintain input order"""
 
     size: int
-    row_pos_map: Optional[dict[int, int]]  # id(row) -> position of row in output; None if not maintaining order
+    row_pos_map: dict[int, int] | None  # id(row) -> position of row in output; None if not maintaining order
     num_rows: int  # number of rows in the buffer
     num_ready: int  # number of consecutive non-None rows at head
     buffer: np.ndarray  # of object

pixeltable/exec/expr_eval/schedulers.py

@@ -7,7 +7,7 @@ import logging
 import re
 import sys
 import time
-from typing import Any, Awaitable, Collection, Optional
+from typing import Any, Awaitable, Collection
 
 from pixeltable import env, func
 from pixeltable.config import Config
@@ -35,7 +35,7 @@ class RateLimitsScheduler(Scheduler):
     get_request_resources_param_names: list[str]  # names of parameters of RateLimitsInfo.get_request_resources()
 
     # scheduling-related state
-    pool_info: Optional[env.RateLimitsInfo]
+    pool_info: env.RateLimitsInfo | None
     est_usage: dict[str, int]  # value per resource; accumulated estimates since the last util. report
 
     num_in_flight: int  # unfinished tasks
@@ -77,7 +77,7 @@ class RateLimitsScheduler(Scheduler):
         self.est_usage = dict.fromkeys(self._resources, 0)
 
     async def _main_loop(self) -> None:
-        item: Optional[RateLimitsScheduler.QueueItem] = None
+        item: RateLimitsScheduler.QueueItem | None = None
         while True:
             if item is None:
                 item = await self.queue.get()
@@ -102,8 +102,8 @@ class RateLimitsScheduler(Scheduler):
             request_resources = self._get_request_resources(item.request)
             limits_info = self._check_resource_limits(request_resources)
             aws: list[Awaitable[None]] = []
-            completed_aw: Optional[asyncio.Task] = None
-            wait_for_reset: Optional[asyncio.Task] = None
+            completed_aw: asyncio.Task | None = None
+            wait_for_reset: asyncio.Task | None = None
             if limits_info is not None:
                 # limits_info's resource is depleted, wait for capacity to free up
 
@@ -167,7 +167,7 @@ class RateLimitsScheduler(Scheduler):
             constant_kwargs, batch_kwargs = request.pxt_fn.create_batch_kwargs(batch_kwargs)
             return self.pool_info.get_request_resources(**constant_kwargs, **batch_kwargs)
 
-    def _check_resource_limits(self, request_resources: dict[str, int]) -> Optional[env.RateLimitInfo]:
+    def _check_resource_limits(self, request_resources: dict[str, int]) -> env.RateLimitInfo | None:
         """Returns the most depleted resource, relative to its limit, or None if all resources are within limits"""
         candidates: list[tuple[env.RateLimitInfo, float]] = []  # (info, relative remaining)
         for resource, usage in request_resources.items():
@@ -405,7 +405,7 @@ class RequestRateScheduler(Scheduler):
             if is_task:
                 self.num_in_flight -= 1
 
-    def _is_rate_limit_error(self, exc: Exception) -> tuple[bool, Optional[float]]:
+    def _is_rate_limit_error(self, exc: Exception) -> tuple[bool, float | None]:
         """Returns True if the exception indicates a rate limit error, and the retry delay in seconds."""
         from http import HTTPStatus
 
@@ -413,7 +413,7 @@ class RequestRateScheduler(Scheduler):
         # We look for attributes that contain status codes, instead of checking the type of the exception,
         # in order to handle a wider variety of exception classes.
         is_rate_limit_error = False
-        retry_delay: Optional[float] = None
+        retry_delay: float | None = None
 
         # requests.HTTPError/httpx.HTTPStatusError
         if (
@@ -443,7 +443,7 @@ class RequestRateScheduler(Scheduler):
 
         return False, None
 
-    def _extract_retry_delay_from_headers(self, headers: Optional[Any]) -> Optional[float]:
+    def _extract_retry_delay_from_headers(self, headers: Any | None) -> float | None:
         """Extract retry delay from HTTP headers."""
         if headers is None:
             return None
@@ -489,7 +489,7 @@ class RequestRateScheduler(Scheduler):
 
         return None
 
-    def _extract_retry_delay_from_message(self, msg: str) -> Optional[float]:
+    def _extract_retry_delay_from_message(self, msg: str) -> float | None:
         msg_lower = msg.lower()
         for pattern in self.RETRY_AFTER_PATTERNS:
             match = re.search(pattern, msg_lower)
@@ -500,7 +500,7 @@ class RequestRateScheduler(Scheduler):
                     continue
         return None
 
-    def _compute_retry_delay(self, num_retries: int, retry_after: Optional[float] = None) -> float:
+    def _compute_retry_delay(self, num_retries: int, retry_after: float | None = None) -> float:
         """
         Calculate exponential backoff delay for rate limit errors.
 

pixeltable/exec/in_memory_data_node.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, AsyncIterator, Optional
+from typing import Any, AsyncIterator
 
 from pixeltable import catalog, exprs
 from pixeltable.utils.local_store import TempStore
@@ -23,7 +23,7 @@ class InMemoryDataNode(ExecNode):
 
     input_rows: list[dict[str, Any]]
     start_row_id: int
-    output_batch: Optional[DataRowBatch]
+    output_batch: DataRowBatch | None
 
     # output_exprs is declared in the superclass, but we redeclare it here with a more specific type
     output_exprs: list[exprs.ColumnRef]

pixeltable/exec/object_store_save_node.py

@@ -6,7 +6,7 @@ import logging
 from collections import defaultdict, deque
 from concurrent import futures
 from pathlib import Path
-from typing import AsyncIterator, Iterator, NamedTuple, Optional
+from typing import AsyncIterator, Iterator, NamedTuple
 
 from pixeltable import exprs
 from pixeltable.utils.object_stores import ObjectOps, ObjectPath, StorageTarget
@@ -44,11 +44,11 @@ class ObjectStoreSaveNode(ExecNode):
         """Specify the source and destination for a WorkItem"""
 
         src_path: str  # source of the file to be processed
-        destination: Optional[str]  # destination URI for the file to be processed
+        destination: str  # destination URI for the file to be processed
 
     class WorkItem(NamedTuple):
         src_path: Path
-        destination: Optional[str]
+        destination: str | None
         info: exprs.ColumnSlotIdx  # column info for the file being processed
         destination_count: int = 1  # number of unique destinations for this file
 
@@ -60,7 +60,7 @@ class ObjectStoreSaveNode(ExecNode):
 
     # ready_rows: rows that are ready to be returned, ordered by row idx;
     # the implied row idx of ready_rows[0] is num_returned_rows
-    ready_rows: deque[Optional[exprs.DataRow]]
+    ready_rows: deque[exprs.DataRow | None]
 
     in_flight_rows: dict[int, ObjectStoreSaveNode.RowState]  # rows with in-flight work; id(row) -> RowState
     in_flight_requests: dict[
@@ -71,12 +71,12 @@ class ObjectStoreSaveNode(ExecNode):
     ]  # WorkDesignator -> [(row, info)]
 
     input_finished: bool
-    row_idx: Iterator[Optional[int]]
+    row_idx: Iterator[int | None]
 
     @dataclasses.dataclass
     class RowState:
         row: exprs.DataRow
-        idx: Optional[int]  # position in input stream; None if we don't retain input order
+        idx: int | None  # position in input stream; None if we don't retain input order
         num_missing: int  # number of references to media files in this row
         delete_destinations: list[Path]  # paths to delete after all copies are complete
 
@@ -99,7 +99,7 @@ class ObjectStoreSaveNode(ExecNode):
     def queued_work(self) -> int:
         return len(self.in_flight_requests)
 
-    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> Optional[DataRowBatch]:
+    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> DataRowBatch | None:
         """Get the next batch of input rows, or None if there are no more rows"""
         try:
             input_batch = await anext(input_iter)
@@ -148,7 +148,7 @@ class ObjectStoreSaveNode(ExecNode):
             sum(int(row is not None) for row in itertools.islice(self.ready_rows, self.BATCH_SIZE)) == self.BATCH_SIZE
         )
 
-    def __add_ready_row(self, row: exprs.DataRow, row_idx: Optional[int]) -> None:
+    def __add_ready_row(self, row: exprs.DataRow, row_idx: int | None) -> None:
         if row_idx is None:
             self.ready_rows.append(row)
         else:
@@ -209,14 +209,11 @@ class ObjectStoreSaveNode(ExecNode):
             assert col.col_type.is_media_type()
 
             destination = info.col.destination
-            soa = None if destination is None else ObjectPath.parse_object_storage_addr(destination, False)
-            if (
-                soa is not None
-                and soa.storage_target == StorageTarget.LOCAL_STORE
-                and LocalStore(soa).resolve_url(url) is not None
-            ):
-                # A local non-default destination was specified, and the url already points there
-                continue
+            if destination is not None:
+                soa = ObjectPath.parse_object_storage_addr(destination, False)
+                if soa.storage_target == StorageTarget.LOCAL_STORE and LocalStore(soa).resolve_url(url) is not None:
+                    # A local non-default destination was specified, and the url already points there
+                    continue
 
             src_path = LocalStore.file_url_to_path(url)
             if src_path is None:
@@ -283,7 +280,7 @@ class ObjectStoreSaveNode(ExecNode):
             )
             _logger.debug(f'submitted {work_item}')
 
-    def __persist_media_file(self, work_item: WorkItem) -> tuple[Optional[str], Optional[Exception]]:
+    def __persist_media_file(self, work_item: WorkItem) -> tuple[str | None, Exception | None]:
         """Move data from the TempStore to another location"""
         src_path = work_item.src_path
         col = work_item.info.col

pixeltable/exec/sql_node.py

@@ -2,7 +2,7 @@ import datetime
 import logging
 import warnings
 from decimal import Decimal
-from typing import TYPE_CHECKING, AsyncIterator, Iterable, NamedTuple, Optional, Sequence
+from typing import TYPE_CHECKING, AsyncIterator, Iterable, NamedTuple, Sequence
 from uuid import UUID
 
 import sqlalchemy as sql
@@ -22,13 +22,13 @@ _logger = logging.getLogger('pixeltable')
 
 class OrderByItem(NamedTuple):
     expr: exprs.Expr
-    asc: Optional[bool]
+    asc: bool | None
 
 
 OrderByClause = list[OrderByItem]
 
 
-def combine_order_by_clauses(clauses: Iterable[OrderByClause]) -> Optional[OrderByClause]:
+def combine_order_by_clauses(clauses: Iterable[OrderByClause]) -> OrderByClause | None:
     """Returns a clause that's compatible with 'clauses', or None if that doesn't exist.
     Two clauses are compatible if for each of their respective items c1[i] and c2[i]
     a) the exprs are identical and
@@ -81,15 +81,15 @@ class SqlNode(ExecNode):
         set_pk: if True, sets the primary for each DataRow
     """
 
-    tbl: Optional[catalog.TableVersionPath]
+    tbl: catalog.TableVersionPath | None
     select_list: exprs.ExprSet
     columns: list[catalog.Column]  # for which columns to populate DataRow.cell_vals/cell_md
     cell_md_refs: list[exprs.ColumnPropertyRef]  # of ColumnRefs which also need DataRow.slot_cellmd for evaluation
     set_pk: bool
     num_pk_cols: int
-    py_filter: Optional[exprs.Expr]  # a predicate that can only be run in Python
-    py_filter_eval_ctx: Optional[exprs.RowBuilder.EvalCtx]
-    cte: Optional[sql.CTE]
+    py_filter: exprs.Expr | None  # a predicate that can only be run in Python
+    py_filter_eval_ctx: exprs.RowBuilder.EvalCtx | None
+    cte: sql.CTE | None
     sql_elements: exprs.SqlElementCache
 
     # execution state
@@ -99,15 +99,15 @@ class SqlNode(ExecNode):
     result_cursor: sql.engine.CursorResult | None
 
     # where_clause/-_element: allow subclass to set one or the other (but not both)
-    where_clause: Optional[exprs.Expr]
-    where_clause_element: Optional[sql.ColumnElement]
+    where_clause: exprs.Expr | None
+    where_clause_element: sql.ColumnElement | None
 
     order_by_clause: OrderByClause
-    limit: Optional[int]
+    limit: int | None
 
     def __init__(
         self,
-        tbl: Optional[catalog.TableVersionPath],
+        tbl: catalog.TableVersionPath | None,
         row_builder: exprs.RowBuilder,
         select_list: Iterable[exprs.Expr],
         columns: list[catalog.Column],
@@ -216,7 +216,7 @@ class SqlNode(ExecNode):
     def _ordering_tbl_ids(self) -> set[UUID]:
         return exprs.Expr.all_tbl_ids(e for e, _ in self.order_by_clause)
 
-    def to_cte(self, keep_pk: bool = False) -> Optional[tuple[sql.CTE, exprs.ExprDict[sql.ColumnElement]]]:
+    def to_cte(self, keep_pk: bool = False) -> tuple[sql.CTE, exprs.ExprDict[sql.ColumnElement]] | None:
         """
         Creates a CTE that materializes the output of this node plus a mapping from select list expr to output column.
         keep_pk: if True, the PK columns are included in the CTE Select statement
@@ -245,8 +245,8 @@ class SqlNode(ExecNode):
         cls,
         tbl: catalog.TableVersionPath,
         stmt: sql.Select,
-        refd_tbl_ids: Optional[set[UUID]] = None,
-        exact_version_only: Optional[set[UUID]] = None,
+        refd_tbl_ids: set[UUID] | None = None,
+        exact_version_only: set[UUID] | None = None,
     ) -> sql.Select:
         """Add From clause to stmt for tables/views referenced by materialized_exprs
         Args:
@@ -270,7 +270,7 @@ class SqlNode(ExecNode):
                 joined_tbls.append(t)
 
         first = True
-        prev_tv: Optional[catalog.TableVersion] = None
+        prev_tv: catalog.TableVersion | None = None
         for t in joined_tbls[::-1]:
             tv = t.get()
             # _logger.debug(f'create_from_clause: tbl_id={tv.id} {id(tv.store_tbl.sa_tbl)}')
@@ -347,7 +347,7 @@ class SqlNode(ExecNode):
                 pass
 
         output_batch = DataRowBatch(self.row_builder)
-        output_row: Optional[exprs.DataRow] = None
+        output_row: exprs.DataRow | None = None
         num_rows_returned = 0
         is_using_cockroachdb = Env.get().is_using_cockroachdb
         tzinfo = Env.get().default_time_zone
@@ -450,7 +450,7 @@ class SqlScanNode(SqlNode):
         columns: list[catalog.Column],
         cell_md_col_refs: list[exprs.ColumnRef] | None = None,
         set_pk: bool = False,
-        exact_version_only: Optional[list[catalog.TableVersionHandle]] = None,
+        exact_version_only: list[catalog.TableVersionHandle] | None = None,
     ):
         sql_elements = exprs.SqlElementCache()
         super().__init__(
@@ -528,17 +528,17 @@ class SqlAggregationNode(SqlNode):
         limit: max number of rows to return: None = no limit
     """
 
-    group_by_items: Optional[list[exprs.Expr]]
-    input_cte: Optional[sql.CTE]
+    group_by_items: list[exprs.Expr] | None
+    input_cte: sql.CTE | None
 
     def __init__(
         self,
         row_builder: exprs.RowBuilder,
         input: SqlNode,
         select_list: Iterable[exprs.Expr],
-        group_by_items: Optional[list[exprs.Expr]] = None,
-        limit: Optional[int] = None,
-        exact_version_only: Optional[list[catalog.TableVersion]] = None,
+        group_by_items: list[exprs.Expr] | None = None,
+        limit: int | None = None,
+        exact_version_only: list[catalog.TableVersion] | None = None,
     ):
         assert len(input.cell_md_refs) == 0  # there's no aggregation over json or arrays in SQL
         self.input_cte, input_col_map = input.to_cte()
@@ -617,9 +617,9 @@ class SqlSampleNode(SqlNode):
         stratify_exprs: Analyzer processed list of expressions to stratify by.
     """
 
-    input_cte: Optional[sql.CTE]
+    input_cte: sql.CTE | None
     pk_count: int
-    stratify_exprs: Optional[list[exprs.Expr]]
+    stratify_exprs: list[exprs.Expr] | None
     sample_clause: 'SampleClause'
 
     def __init__(
@@ -648,7 +648,6 @@ class SqlSampleNode(SqlNode):
         )
         self.stratify_exprs = stratify_exprs
         self.sample_clause = sample_clause
-        assert isinstance(self.sample_clause.seed, int)
 
     @classmethod
     def key_sql_expr(cls, seed: sql.ColumnElement, sql_cols: Iterable[sql.ColumnElement]) -> sql.ColumnElement:
@@ -667,7 +666,9 @@ class SqlSampleNode(SqlNode):
         """Create an expression for randomly ordering rows with a given seed"""
         rowid_cols = [*cte.c[-self.pk_count : -1]]  # exclude the version column
         assert len(rowid_cols) > 0
-        return self.key_sql_expr(sql.literal_column(str(self.sample_clause.seed)), rowid_cols)
+        # If seed is not set in the sample clause, use the random seed given by the execution context
+        seed = self.sample_clause.seed if self.sample_clause.seed is not None else self.ctx.random_seed
+        return self.key_sql_expr(sql.literal_column(str(seed)), rowid_cols)
 
     def _create_stmt(self) -> sql.Select:
         from pixeltable.plan import SampleClause
@@ -691,7 +692,7 @@ class SqlSampleNode(SqlNode):
 
             return self._create_stmt_stratified_n(self.sample_clause.n, self.sample_clause.n_per_stratum)
 
-    def _create_stmt_stratified_n(self, n: Optional[int], n_per_stratum: Optional[int]) -> sql.Select:
+    def _create_stmt_stratified_n(self, n: int | None, n_per_stratum: int | None) -> sql.Select:
         """Create a Select stmt that returns n samples across all strata or n_per_stratum samples per stratum"""
 
         sql_strata_exprs = [self.sql_elements.get(e) for e in self.stratify_exprs]

pixeltable/exprs/arithmetic_expr.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -58,7 +58,7 @@ class ArithmeticExpr(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [*super()._id_attrs(), ('operator', self.operator.value)]
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
         assert self.col_type.is_int_type() or self.col_type.is_float_type() or self.col_type.is_json_type()
         left = sql_elements.get(self._op1)
         right = sql_elements.get(self._op2)
@@ -118,7 +118,7 @@ class ArithmeticExpr(Expr):
 
         data_row[self.slot_idx] = self.eval_nullable(op1_val, op2_val)
 
-    def eval_nullable(self, op1_val: Optional[float], op2_val: Optional[float]) -> Optional[float]:
+    def eval_nullable(self, op1_val: float | None, op2_val: float | None) -> float | None:
         """
         Return the result of evaluating the expression on two nullable int/float operands,
         None is interpreted as SQL NULL
@@ -144,7 +144,7 @@ class ArithmeticExpr(Expr):
         elif self.operator == ArithmeticOperator.FLOORDIV:
             return op1_val // op2_val
 
-    def as_literal(self) -> Optional[Literal]:
+    def as_literal(self) -> Literal | None:
         op1_lit = self._op1.as_literal()
         if op1_lit is None:
             return None

pixeltable/exprs/array_slice.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -43,7 +43,7 @@ class ArraySlice(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [*super()._id_attrs(), ('index', self.index)]
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         return None
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/exprs/column_property_ref.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import enum
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -34,7 +34,7 @@ class ColumnPropertyRef(Expr):
         self.prop = prop
         self.id = self._create_id()
 
-    def default_column_name(self) -> Optional[str]:
+    def default_column_name(self) -> str | None:
         return str(self).replace('.', '_')
 
     def _equals(self, other: ColumnPropertyRef) -> bool:
@@ -55,7 +55,7 @@ class ColumnPropertyRef(Expr):
     def is_cellmd_prop(self) -> bool:
         return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG, self.Property.CELLMD)
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
         if not self.col_ref.col_handle.get().is_stored:
             return None
         col = self.col_ref.col_handle.get()