pixeltable 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl

pixeltable/functions/math.py

@@ -1,3 +1,15 @@
+"""
+Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs) for mathematical operations.
+
+Example:
+```python
+import pixeltable as pxt
+
+t = pxt.get_table(...)
+t.select(t.float_col.floor()).collect()
+```
+"""
+
 import builtins
 import math
 from typing import Optional
@@ -10,6 +22,11 @@ from pixeltable.utils.code import local_public_names
 
 @pxt.udf(is_method=True)
 def abs(self: float) -> float:
+    """
+    Return the absolute value of the given number.
+
+    Equivalent to Python [`builtins.abs()`](https://docs.python.org/3/library/functions.html#abs).
+    """
     return builtins.abs(self)
 
 
@@ -20,6 +37,14 @@ def _(self: sql.ColumnElement) -> sql.ColumnElement:
 
 @pxt.udf(is_method=True)
 def ceil(self: float) -> float:
+    """
+    Return the ceiling of the given number.
+
+    Equivalent to Python [`float(math.ceil(self))`](https://docs.python.org/3/library/math.html#math.ceil) if `self`
+    is finite, or `self` itself if `self` is infinite. (This is slightly different from the default behavior of
+    `math.ceil(self)`, which always returns an `int` and raises an error if `self` is infinite. The behavior in
+    Pixeltable generalizes the Python operator and is chosen to align with the SQL standard.)
+    """
     # This ensures the same behavior as SQL
     if math.isfinite(self):
         return float(math.ceil(self))
@@ -34,6 +59,14 @@ def _(self: sql.ColumnElement) -> sql.ColumnElement:
 
 @pxt.udf(is_method=True)
 def floor(self: float) -> float:
+    """
+    Return the ceiling of the given number.
+
+    Equivalent to Python [`float(math.floor(self))`](https://docs.python.org/3/library/math.html#math.ceil) if `self`
+    is finite, or `self` itself if `self` is infinite. (This is slightly different from the default behavior of
+    `math.floor(self)`, which always returns an `int` and raises an error if `self` is infinite. The behavior of
+    Pixeltable generalizes the Python operator and is chosen to align with the SQL standard.)
+    """
     # This ensures the same behavior as SQL
     if math.isfinite(self):
         return float(math.floor(self))
@@ -48,6 +81,13 @@ def _(self: sql.ColumnElement) -> sql.ColumnElement:
 
 @pxt.udf(is_method=True)
 def round(self: float, digits: Optional[int] = None) -> float:
+    """
+    Round a number to a given precision in decimal digits.
+
+    Equivalent to Python [`builtins.round(self, digits or 0)`](https://docs.python.org/3/library/functions.html#round).
+    Note that if `digits` is not specified, the behavior matches `builtins.round(self, 0)` rather than
+    `builtins.round(self)`; this ensures that the return type is always `float` (as in SQL) rather than `int`.
+    """
     # Set digits explicitly to 0 to guarantee a return type of float; this ensures the same behavior as SQL
     return builtins.round(self, digits or 0)
 
@@ -60,6 +100,69 @@ def _(self: sql.ColumnElement, digits: Optional[sql.ColumnElement] = None) -> sq
         return sql.func.round(sql.cast(self, sql.Numeric), sql.cast(digits, sql.Integer))
 
 
+@pxt.udf(is_method=True)
+def pow(self: int, other: int) -> float:
+    """
+    Raise `self` to the power of `other`.
+
+    Equivalent to Python [`self ** other`](https://docs.python.org/3/library/functions.html#pow).
+    """
+    return self**other
+
+
+@pow.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return sql.func.pow(self, other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_and(self: int, other: int) -> int:
+    """
+    Bitwise AND of two integers.
+
+    Equivalent to Python
+    [`self & other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self & other
+
+
+@bitwise_and.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_and(other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_or(self: int, other: int) -> int:
+    """
+    Bitwise OR of two integers.
+
+    Equivalent to Python
+    [`self | other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self | other
+
+
+@bitwise_or.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_or(other)
+
+
+@pxt.udf(is_method=True)
+def bitwise_xor(self: int, other: int) -> int:
+    """
+    Bitwise XOR of two integers.
+
+    Equivalent to Python
+    [`self ^ other`](https://docs.python.org/3/library/stdtypes.html#bitwise-operations-on-integer-types).
+    """
+    return self ^ other
+
+
+@bitwise_xor.to_sql
+def _(self: sql.ColumnElement, other: sql.ColumnElement) -> sql.ColumnElement:
+    return self.bitwise_xor(other)
+
+
 __all__ = local_public_names(__name__)
 
 

pixeltable/functions/string.py

@@ -5,10 +5,9 @@ It closely follows the Pandas `pandas.Series.str` API.
 Example:
 ```python
 import pixeltable as pxt
-from pixeltable.functions import string as pxt_str
 
 t = pxt.get_table(...)
-t.select(pxt_str.capitalize(t.str_col)).collect()
+t.select(t.str_col.capitalize()).collect()
 ```
 """
 

pixeltable/functions/video.py

@@ -4,10 +4,10 @@ Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
 Example:
 ```python
 import pixeltable as pxt
-from pixeltable.functions import video as pxt_video
+import pixeltable.functions as pxtf
 
 t = pxt.get_table(...)
-t.select(pxt_video.extract_audio(t.video_col)).collect()
+t.select(pxtf.video.extract_audio(t.video_col)).collect()
 ```
 """
 

pixeltable/globals.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 import os
-import urllib.parse
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, Optional, Union
 
@@ -372,6 +371,31 @@ def create_snapshot(
     )
 
 
+def create_replica(destination: str, source: Union[str, catalog.Table]) -> Optional[catalog.Table]:
+    """
+    Create a replica of a table. Can be used either to create a remote replica of a local table, or to create a local
+    replica of a remote table. A given table can have at most one replica per Pixeltable instance.
+
+    Args:
+        destination: Path where the replica will be created. Can be either a local path such as `'my_dir.my_table'`, or
+            a remote URI such as `'pxt://username/mydir.my_table'`.
+        source: Path to the source table, or (if the source table is a local table) a handle to the source table.
+    """
+    remote_dest = destination.startswith('pxt://')
+    remote_source = isinstance(source, str) and source.startswith('pxt://')
+    if remote_dest == remote_source:
+        raise excs.Error('Exactly one of `destination` or `source` must be a remote URI.')
+
+    if remote_dest:
+        if isinstance(source, str):
+            source = get_table(source)
+        share.push_replica(destination, source)
+        return None
+    else:
+        assert isinstance(source, str)
+        return share.pull_replica(destination, source)
+
+
 def get_table(path: str) -> catalog.Table:
     """Get a handle to an existing table, view, or snapshot.
 
@@ -470,7 +494,7 @@ def drop_table(
         # if we're dropping a table by handle, we first need to get the current path, then drop the S lock on
         # the Table record, and then get X locks in the correct order (first containing directory, then table)
         with Env.get().begin_xact():
-            tbl_path = table._path()
+            tbl_path = table._path
     else:
         assert isinstance(table, str)
         tbl_path = table
@@ -627,13 +651,6 @@ def _extract_paths(
     return result
 
 
-def publish_snapshot(dest_uri: str, table: catalog.Table) -> None:
-    parsed_uri = urllib.parse.urlparse(dest_uri)
-    if parsed_uri.scheme != 'pxt':
-        raise excs.Error(f'Invalid Pixeltable URI (does not start with pxt://): {dest_uri}')
-    share.publish_snapshot(dest_uri, table)
-
-
 def list_dirs(path: str = '', recursive: bool = True) -> list[str]:
     """List the directories in a directory.
 

pixeltable/io/hf_datasets.py

@@ -31,8 +31,8 @@ _hf_to_pxt: dict[str, ts.ColumnType] = {
     'timestamp[s]': ts.TimestampType(nullable=True),
     'timestamp[ms]': ts.TimestampType(nullable=True),  # HF dataset iterator converts timestamps to datetime.datetime
     'timestamp[us]': ts.TimestampType(nullable=True),
-    'date32': ts.StringType(nullable=True),  # date32 is not supported in pixeltable, use string
-    'date64': ts.StringType(nullable=True),  # date64 is not supported in pixeltable, use string
+    'date32': ts.DateType(nullable=True),
+    'date64': ts.DateType(nullable=True),
 }
 
 

pixeltable/io/pandas.py

@@ -9,6 +9,7 @@ from pandas.api.types import is_datetime64_any_dtype, is_extension_array_dtype
 import pixeltable as pxt
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
+from pixeltable.env import Env
 
 
 def import_pandas(
@@ -209,14 +210,25 @@ def _df_row_to_pxt_row(
             nval = bool(val)
         elif pxt_type.is_string_type():
             nval = str(val)
+        elif pxt_type.is_date_type():
+            if pd.isnull(val):
+                # pandas has the bespoke 'NaT' valud for a missing timestamp
+                # This is not supported by postgres, and must be converted to None
+                nval = None
+            else:
+                nval = pd.Timestamp(val).date()
         elif pxt_type.is_timestamp_type():
             if pd.isnull(val):
-                # pandas has the bespoke 'NaT' type for a missing timestamp; postgres is very
-                # much not-ok with it. (But if we convert it to None and then load out the
-                # table contents as a pandas DataFrame, it will correctly restore the 'NaT'!)
+                # pandas has the bespoke 'NaT' value for a missing timestamp
+                # This is not supported by postgres, and must be converted to None
                 nval = None
             else:
-                nval = pd.Timestamp(val).to_pydatetime()
+                tval = pd.Timestamp(val)
+                # pandas supports tz-aware and naive timestamps.
+                if tval.tz is None:
+                    nval = pd.Timestamp(tval).tz_localize(tz=Env.get().default_time_zone)
+                else:
+                    nval = tval.astimezone(Env.get().default_time_zone)
         else:
             nval = val
         pxt_row[pxt_name] = nval

pixeltable/io/parquet.py

@@ -112,11 +112,11 @@ def export_parquet(
                         length = len(val)
                     elif col_type.is_string_type():
                         length = len(val)
-                    elif col_type.is_video_type():
+                    elif col_type.is_video_type() or col_type.is_audio_type():
                         if data_row.file_paths is not None and data_row.file_paths[e.slot_idx] is not None:
                             val = data_row.file_paths[e.slot_idx]
                         else:
-                            raise excs.Error(f'unknown video type {type(val)}')
+                            raise excs.Error(f'unknown audio/video type {type(val)}')
                         length = len(val)
                     elif col_type.is_json_type():
                         val = json.dumps(val)
@@ -127,6 +127,8 @@ def export_parquet(
                         length = 8
                     elif col_type.is_bool_type():
                         length = 1
+                    elif col_type.is_date_type():
+                        length = 4
                     elif col_type.is_timestamp_type():
                         val = val.astimezone(datetime.timezone.utc)
                         length = 8

pixeltable/metadata/__init__.py

@@ -16,7 +16,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 34
+VERSION = 35
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/converters/convert_34.py

@@ -0,0 +1,21 @@
+from typing import Any, Optional
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=34)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, substitution_fn=__substitute_md)
+
+
+def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+    if isinstance(v, dict) and '_classname' in v and v['_classname'] == 'ColumnRef':
+        # Add reference_tbl to ColumnRef; for historical metadata it is always equal to tbl
+        assert 'reference_tbl' not in v
+        v['reference_tbl'] = None
+        return k, v
+
+    return None

pixeltable/metadata/notes.py

@@ -2,6 +2,7 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    35: 'Track reference_tbl in ColumnRef',
     34: 'Set default value for is_pk field in column metadata to False',
     33: 'Add is_replica field to table metadata',
     32: 'Add the lock_dummy BIGINT column to the dirs table',

pixeltable/plan.py

@@ -635,8 +635,8 @@ class Planner:
                 raise excs.Error(f'Join predicate {join_clause.join_predicate} not expressible in SQL')
 
     @classmethod
-    def _verify_ordering(cls, analyzer: Analyzer, verify_agg: bool) -> None:
-        """Verify that the various ordering requirements don't conflict"""
+    def _create_combined_ordering(cls, analyzer: Analyzer, verify_agg: bool) -> Optional[OrderByClause]:
+        """Verify that the various ordering requirements don't conflict and return a combined ordering"""
         ob_clauses: list[OrderByClause] = [analyzer.order_by_clause.copy()]
 
         if verify_agg:
@@ -652,8 +652,11 @@ class Planner:
                     OrderByItem(e, True) for e in fn_call.get_agg_order_by()
                 ]
                 ob_clauses.append(ordering)
-        if len(ob_clauses) <= 1:
-            return
+
+        if len(ob_clauses) == 0:
+            return None
+        elif len(ob_clauses) == 1:
+            return ob_clauses[0]
 
         combined_ordering = ob_clauses[0]
         for ordering in ob_clauses[1:]:
@@ -664,6 +667,7 @@ class Planner:
                     f'{print_order_by_clause(combined_ordering)} vs {print_order_by_clause(ordering)}'
                 )
             combined_ordering = combined
+        return combined_ordering
 
     @classmethod
     def _is_contained_in(cls, l1: Iterable[exprs.Expr], l2: Iterable[exprs.Expr]) -> bool:
@@ -761,7 +765,7 @@ class Planner:
             analyzer.window_fn_calls
         )
         ctx = exec.ExecContext(row_builder)
-        cls._verify_ordering(analyzer, verify_agg=is_python_agg)
+        combined_ordering = cls._create_combined_ordering(analyzer, verify_agg=is_python_agg)
         cls._verify_join_clauses(analyzer)
 
         # materialized with SQL table scans (ie, single-table SELECT statements):
@@ -859,6 +863,9 @@ class Planner:
                     row_builder, input=plan, select_list=analyzer.select_list, group_by_items=analyzer.group_by_clause
                 )
             else:
+                input_sql_node = plan.get_node(exec.SqlNode)
+                assert combined_ordering is not None
+                input_sql_node.set_order_by(combined_ordering)
                 plan = exec.AggregationNode(
                     tbl.tbl_version,
                     row_builder,

pixeltable/share/__init__.py

@@ -1,3 +1,3 @@
 # ruff: noqa: F401
 
-from .publish import publish_snapshot
+from .publish import pull_replica, push_replica