pyspiral 0.2.5__cp310-abi3-macosx_11_0_arm64.whl → 0.4.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/core/{spec → table/spec}/__init__.pyi

@@ -14,6 +14,24 @@ class ColumnGroup:
     @staticmethod
     def from_str(path: str) -> ColumnGroup: ...
 
+class KeySpaceMetadata:
+    def __init__(
+        self,
+        *,
+        manifest_handle: ManifestHandle | None,
+        last_modified_at: int,
+    ): ...
+
+    manifest_handle: ManifestHandle | None
+    last_modified_at: int
+
+    def asof(self, asof: int) -> KeySpaceMetadata:
+        """Returns the metadata as of a given timestamp. Currently just filtering versioned schemas."""
+        ...
+
+    def apply_wal(self, wal: WriteAheadLog) -> KeySpaceMetadata:
+        """Applies the given WAL to the metadata."""
+
 class ColumnGroupMetadata:
     def __init__(
         self,
@@ -44,57 +62,36 @@ class ColumnGroupMetadata:
     def apply_wal(self, wal: WriteAheadLog) -> ColumnGroupMetadata:
         """Applies the given WAL to the metadata."""
 
-    def __bytes__(self):
-        """Serializes the ColumnGroupMetadata to a protobuf buffer."""
-
-    @staticmethod
-    def from_proto(buffer: bytes) -> ColumnGroupMetadata:
-        """Deserializes a ColumnGroupMetadata from a protobuf buffer."""
-        ...
-
 class LogEntry:
     ts: int
-    operation: KeySpaceWriteOp | FragmentSetWriteOp | ConfigurationOp | SchemaEvolutionOp | SchemaBreakOp
+    operation: (
+        KeySpaceWriteOp
+        | ColumnGroupWriteOp
+        | SchemaEvolutionOp
+        | SchemaBreakOp
+        | KeySpaceCompactOp
+        | ColumnGroupCompactOp
+    )
 
     def column_group(self) -> ColumnGroup | None:
         """Returns the column group of the entry if it is associated with one."""
 
-    def replace_timestamp(self, ts: int) -> LogEntry:
-        """Returns a copy of the entry with the timestamp replaced."""
-
-    @staticmethod
-    def schema_break(*, column_group: ColumnGroup, removed_column_names: list[str]) -> LogEntry: ...
-    @staticmethod
-    def schema_evolution(*, column_group: ColumnGroup, new_schema: Schema) -> LogEntry: ...
-    @staticmethod
-    def ks_write(
-        *,
-        ks_id: str,
-        manifest_handle: ManifestHandle,
-    ) -> LogEntry: ...
-    @staticmethod
-    def fs_write(
-        *,
-        column_group: ColumnGroup,
-        fs_id: str,
-        fs_level: FragmentLevel,
-        manifest_handle: ManifestHandle,
-        key_span: KeySpan,
-        key_extent: KeyExtent,
-        column_ids: list[str],
-    ) -> LogEntry: ...
-
 class FileFormat:
     def __init__(self, value: int): ...
 
     Parquet: FileFormat
     Protobuf: FileFormat
     BinaryArray: FileFormat
+    Vortex: FileFormat
 
     def __int__(self) -> int:
         """Returns the protobuf enum int value."""
         ...
 
+    def __str__(self) -> str:
+        """Returns the string representation of the file format."""
+        ...
+
 class FragmentLevel:
     L0: FragmentLevel
     L1: FragmentLevel
@@ -104,15 +101,9 @@ class FragmentLevel:
         ...
 
 class Key:
-    def __init__(self, key: bytes): ...
-
     key: bytes
 
-    def __add__(self, other: Key) -> Key:
-        """Concatenates two keys.
-
-        TODO(ngates): remove this function. It should not be necessary to concatenate keys."""
-
+    def __init__(self, key: bytes): ...
     def __bytes__(self): ...
     def step(self) -> Key:
         """Returns the next key in the key space."""
@@ -121,8 +112,6 @@ class Key:
     def min() -> Key: ...
     @staticmethod
     def max() -> Key: ...
-    @staticmethod
-    def from_array_tuple(array_tuple: tuple[pa.Array]) -> Key: ...
 
 class KeyExtent:
     """An inclusive range of keys."""
@@ -132,9 +121,6 @@ class KeyExtent:
     min: Key
     max: Key
 
-    def to_range(self) -> KeyRange:
-        """Turn this inclusive key extent into an exclusive key range."""
-
     def union(self, key_extent: KeyExtent) -> KeyExtent: ...
     def __or__(self, other: KeyExtent) -> KeyExtent: ...
     def intersection(self, key_extent: KeyExtent) -> KeyExtent | None: ...
@@ -142,30 +128,6 @@ class KeyExtent:
     def contains(self, item: Key) -> bool: ...
     def __contains__(self, item: Key) -> bool: ...
 
-class KeyRange:
-    """A right-exclusive range of keys."""
-
-    def __init__(self, *, begin: Key, end: Key): ...
-
-    begin: Key
-    end: Key
-
-    def union(self, other: KeyRange) -> KeyRange: ...
-    def __or__(self, other: KeyRange) -> KeyRange: ...
-    def intersection(self, key_extent: KeyRange) -> KeyRange | None: ...
-    def __and__(self, other: KeyRange) -> KeyRange | None: ...
-    def contains(self, item: Key) -> bool: ...
-    def __contains__(self, item: Key) -> bool: ...
-    def is_disjoint(self, key_range: KeyRange) -> bool:
-        return self.end <= key_range.begin or self.begin >= key_range.end
-
-    @staticmethod
-    def beginning_with(begin: Key) -> KeyRange: ...
-    @staticmethod
-    def ending_with(end: Key) -> KeyRange: ...
-    @staticmethod
-    def full() -> KeyRange: ...
-
 class KeySpan:
     """An exclusive range of keys as indexed by their position in a key space."""
 
@@ -179,26 +141,26 @@ class KeySpan:
     def union(self, other: KeySpan) -> KeySpan: ...
     def __or__(self, other: KeySpan) -> KeySpan: ...
 
-class KeyMap:
-    """Displacement map."""
-
 class ManifestHandle:
-    def __init__(self, id: str, format: FileFormat, file_size: int, spfs_format_metadata: bytes | None): ...
-
     id: str
     format: FileFormat
     file_size: int
-    spfs_format_metadata: bytes | None
 
 class Schema:
     def to_arrow(self) -> pa.Schema:
         """Returns the Arrow schema."""
         ...
-
     @staticmethod
     def from_arrow(arrow: pa.Schema) -> Schema:
         """Creates a Schema from an Arrow schema."""
         ...
+    def __len__(self):
+        """Returns the number of columns in the schema."""
+        ...
+    @property
+    def names(self) -> list[str]:
+        """Returns the names of the columns in the schema."""
+        ...
 
 class VersionedSchema:
     ts: int
@@ -209,24 +171,28 @@ class KeySpaceWriteOp:
     ks_id: str
     manifest_handle: ManifestHandle
 
-class FragmentSetWriteOp:
+class ColumnGroupWriteOp:
     column_group: ColumnGroup
-    fs_id: str
-    fs_level: FragmentLevel
+    level: FragmentLevel
     manifest_handle: ManifestHandle
     key_span: KeySpan
     key_extent: KeyExtent
     column_ids: list[str]
 
-class ConfigurationOp:
-    column_group: ColumnGroup
-
 class SchemaEvolutionOp:
     column_group: ColumnGroup
 
 class SchemaBreakOp:
     column_group: ColumnGroup
 
+class KeySpaceCompactOp:
+    ks_ids: list[str]
+    moved_ks_ids: list[str]
+
+class ColumnGroupCompactOp:
+    column_group: ColumnGroup
+    fragment_ids: list[int]
+
 class WriteAheadLog:
     def __init__(
         self,
@@ -237,7 +203,6 @@ class WriteAheadLog:
 
     entries: list[LogEntry]
     truncated_up_to: int
-    ks_manifest_handle: ManifestHandle | None
 
     @property
     def last_modified_at(self) -> int:
@@ -247,11 +212,3 @@ class WriteAheadLog:
         self, asof: int | None = None, since: int | None = None, column_group: ColumnGroup | None = None
     ) -> WriteAheadLog:
         """Filters the WAL to entries by the given parameters."""
-
-    def __bytes__(self):
-        """Serializes the ColumnGroupMetadata to a protobuf buffer."""
-
-    @staticmethod
-    def from_proto(buffer: bytes) -> WriteAheadLog:
-        """Deserializes a WriteAheadLog from a protobuf buffer."""
-        ...

spiral/datetime_.py

@@ -0,0 +1,27 @@
+import warnings
+from datetime import datetime, timedelta, timezone, tzinfo
+
+_THE_EPOCH = datetime.fromtimestamp(0, tz=timezone.utc)
+
+
+def local_tz() -> tzinfo:
+    """Determine this machine's local timezone."""
+    tz = datetime.now().astimezone().tzinfo
+    if tz is None:
+        raise ValueError("Could not determine this machine's local timezone.")
+    return tz
+
+
+def timestamp_micros(instant: datetime) -> int:
+    """The number of microseconds between the epoch and the given instant."""
+    if instant.tzinfo is None:
+        warnings.warn("assuming timezone-naive datetime is local time", stacklevel=2)
+        instant = instant.replace(tzinfo=local_tz())
+    return (instant - _THE_EPOCH) // timedelta(microseconds=1)
+
+
+def from_timestamp_micros(ts: int) -> datetime:
+    """Convert a timestamp in microseconds to a datetime."""
+    if ts < 0:
+        raise ValueError("Timestamp must be non-negative")
+    return _THE_EPOCH + timedelta(microseconds=ts)

spiral/expressions/__init__.py

@@ -1,20 +1,25 @@
 import builtins
 import functools
 import operator
+import warnings
 from typing import Any
 
 import pyarrow as pa
 
-from spiral import _lib, arrow
+from spiral import _lib, arrow_
 
 from . import http as http
 from . import io as io
 from . import list_ as list
+from . import mp4 as mp4
+from . import png as png
+from . import qoi as qoi
 from . import refs as refs
 from . import str_ as str
 from . import struct as struct
+from . import text as text
 from . import tiff as tiff
-from .base import Expr, ExprLike
+from .base import Expr, ExprLike, NativeExpr
 
 __all__ = [
     "Expr",
@@ -42,6 +47,7 @@ __all__ = [
     "not_",
     "or_",
     "pack",
+    "keyed",
     "ref",
     "refs",
     "scalar",
@@ -50,8 +56,11 @@ __all__ = [
     "struct",
     "subtract",
     "tiff",
-    "var",
     "xor",
+    "png",
+    "qoi",
+    "mp4",
+    "text",
 ]
 
 # Inline some of the struct expressions since they're so common
@@ -67,13 +76,15 @@ def lift(expr: ExprLike) -> Expr:
     # Convert an ExprLike into an Expr.
     if isinstance(expr, Expr):
         return expr
+    if isinstance(expr, NativeExpr):
+        return Expr(expr)
 
     if isinstance(expr, dict):
         # NOTE: we assume this is a struct expression. We could be smarter and be context aware to determine if
         # this is in fact a struct scalar, but the user can always create one of those manually.
 
         # First we un-nest any dot-separated field names
-        expr: dict = arrow.nest_structs(expr)
+        expr: dict = arrow_.nest_structs(expr)
 
         return pack({k: lift(v) for k, v in expr.items()})
 
@@ -88,18 +99,30 @@ def lift(expr: ExprLike) -> Expr:
 
     # If the value is struct-like, we un-nest any dot-separated field names
     if isinstance(expr, pa.StructArray | pa.StructScalar):
-        return lift(arrow.nest_structs(expr))
+        # TODO(marko): Figure out what to do with nullable struct arrays when unpacking them.
+        #   We need to merge struct validity into the child validity?
+        if isinstance(expr, pa.StructArray) and expr.null_count != 0:
+            # raise ValueError("lift: cannot lift a struct array with nulls.")
+            warnings.warn("found a struct array with nulls", stacklevel=2)
+        if isinstance(expr, pa.StructScalar) and not expr.is_valid():
+            # raise ValueError("lift: cannot lift a struct scalar with nulls.")
+            warnings.warn("found a struct scalar with nulls", stacklevel=2)
+        return lift(arrow_.nest_structs(expr))
 
     if isinstance(expr, pa.Array):
-        return Expr(_lib.spql.expr.array_lit(expr))
+        return Expr(_lib.expr.array_lit(expr))
 
     # Otherwise, assume it's a scalar.
     return scalar(expr)
 
 
-def var(name: builtins.str) -> Expr:
-    """Create a variable expression."""
-    return Expr(_lib.spql.expr.var(name))
+def key(name: builtins.str) -> Expr:
+    """Create a variable expression referencing a key column.
+
+    Args:
+        name: variable name
+    """
+    return Expr(_lib.expr.keyed(name))
 
 
 def keyed(name: builtins.str, dtype: pa.DataType) -> Expr:
@@ -112,20 +135,21 @@ def keyed(name: builtins.str, dtype: pa.DataType) -> Expr:
         name: variable name
         dtype: must match dtype of the column in the key table.
     """
-    return Expr(_lib.spql.expr.keyed(f"#{name}", dtype))
+    return Expr(_lib.expr.keyed(name, dtype))
 
 
 def scalar(value: Any) -> Expr:
     """Create a scalar expression."""
     if not isinstance(value, pa.Scalar):
         value = pa.scalar(value)
-    return Expr(_lib.spql.expr.scalar(value))
+    # TODO(marko): Use Vortex scalar instead of passing as array.
+    return Expr(_lib.expr.scalar(pa.array([value.as_py()], type=value.type)))
 
 
 def cast(expr: ExprLike, dtype: pa.DataType) -> Expr:
     """Cast an expression into another PyArrow DataType."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.cast(expr.__expr__, dtype))
+    return Expr(_lib.expr.cast(expr.__expr__, dtype))
 
 
 def and_(expr: ExprLike, *exprs: ExprLike) -> Expr:
@@ -182,19 +206,18 @@ def negate(expr: ExprLike) -> Expr:
 def not_(expr: ExprLike) -> Expr:
     """Negate the given expression."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("not", expr.__expr__))
+    return Expr(_lib.expr.not_(expr.__expr__))
 
 
 def is_null(expr: ExprLike) -> Expr:
     """Check if the given expression is null."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("is_null", expr.__expr__))
+    return Expr(_lib.expr.is_null(expr.__expr__))
 
 
 def is_not_null(expr: ExprLike) -> Expr:
-    """Check if the given expression is null."""
-    expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("is_not_null", expr.__expr__))
+    """Check if the given expression is not null."""
+    return not_(is_null(expr))
 
 
 def add(lhs: ExprLike, rhs: ExprLike) -> Expr:

spiral/expressions/base.py

@@ -5,7 +5,7 @@ import pyarrow as pa
 
 from spiral import _lib
 
-NativeExpr: TypeAlias = _lib.spql.expr.Expr
+NativeExpr: TypeAlias = _lib.expr.Expr
 
 
 class Expr:
@@ -92,13 +92,13 @@ class Expr:
         return self._binary("mod", other)
 
     def __neg__(self):
-        return Expr(_lib.spql.expr.unary("neg", self.__expr__))
+        return Expr(_lib.expr.unary("neg", self.__expr__))
 
     def in_(self, other: "ExprLike") -> "Expr":
         from spiral import expressions as se
 
         other = se.lift(other)
-        return Expr(_lib.spql.expr.list.contains(other.__expr__, self.__expr__))
+        return Expr(_lib.expr.list.contains(other.__expr__, self.__expr__))
 
     def contains(self, other: "ExprLike") -> "Expr":
         from spiral import expressions as se
@@ -107,7 +107,7 @@ class Expr:
 
     def cast(self, dtype: pa.DataType) -> "Expr":
         """Cast the expression result to a different data type."""
-        return Expr(_lib.spql.expr.cast(self.__expr__, dtype))
+        return Expr(_lib.expr.cast(self.__expr__, dtype))
 
     def select(self, *paths: str, exclude: list[str] = None) -> "Expr":
         """Select fields from a struct-like expression.
@@ -142,7 +142,7 @@ class Expr:
         from spiral import expressions as se
 
         rhs = se.lift(rhs)
-        return Expr(_lib.spql.expr.binary(op, self.__expr__, rhs.__expr__))
+        return Expr(_lib.expr.binary(op, self.__expr__, rhs.__expr__))
 
 
 ScalarLike: TypeAlias = bool | int | float | str | list | datetime.datetime | None

spiral/expressions/list_.py

@@ -29,7 +29,7 @@ def element_at(expr: ExprLike, index: ExprLike) -> Expr:
 
     expr = lift(expr)
     index = lift(index)
-    return Expr(_lib.spql.expr.list.element_at(expr.__expr__, index.__expr__))
+    return Expr(_lib.expr.list.element_at(expr.__expr__, index.__expr__))
 
 
 def of(*expr: ExprLike) -> Expr:

spiral/expressions/mp4.py

@@ -0,0 +1,62 @@
+import pyarrow as pa
+
+from spiral.expressions.base import Expr, ExprLike
+
+_MP4_RES_DTYPE: pa.DataType = pa.struct(
+    [
+        pa.field("pixels", pa.large_binary()),
+        pa.field("height", pa.uint32()),
+        pa.field("width", pa.uint32()),
+        pa.field("frames", pa.uint32()),
+    ]
+)
+
+
+# TODO(marko): Support optional range and crop.
+#   IMPORTANT: Frames is currently broken and defaults to full.
+def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str):
+    """
+    Read referenced cell in a `MP4` format. Requires `ffmpeg`.
+
+    Args:
+        expr: The referenced `Mp4` bytes.
+            A str is assumed to be the `se.keyed` expression.
+        frames: The range of frames to read. Each element must be a list of two uint32,
+            frame start and frame end, or null / empty list to read all frames.
+            A str is assumed to be the `se.keyed` expression.
+        crop: The crop of the frames to read. Each element must be a list of four uint32,
+            x, y, width, height or null / empty list to read full frames.
+            A str is assumed to be the `se.keyed` expression.
+
+    Returns:
+        An array where each element is a decoded cropped video with fields:
+            pixels: RGB8 bytes, frames * width * height * 3.
+            width: Width of the image with type `pa.uint32()`.
+            height: Height of the image with type `pa.uint32()`.
+            frames: Number of frames with type `pa.uint32()`.
+    """
+    from spiral import _lib
+    from spiral.expressions import keyed, lift
+
+    if isinstance(expr, str):
+        expr = keyed(
+            expr,
+            pa.struct([("__ref__", pa.struct([("id", pa.string()), ("begin", pa.uint64()), ("end", pa.uint64())]))]),
+        )
+    if isinstance(frames, str):
+        frames = keyed(frames, pa.list_(pa.uint32()))
+    if isinstance(crop, str):
+        crop = keyed(crop, pa.list_(pa.uint32()))
+
+    expr = lift(expr)
+    frames = lift(frames)
+    crop = lift(crop)
+
+    return Expr(
+        _lib.expr.video.read(
+            expr.__expr__,
+            frames.__expr__,
+            crop.__expr__,
+            format="mp4",
+        )
+    )

spiral/expressions/png.py

@@ -0,0 +1,18 @@
+from spiral.expressions.base import Expr, ExprLike
+
+
+def encode(expr: ExprLike) -> Expr:
+    """Encode the given expression as a PNG image.
+
+    Args:
+        expr: The expression to encode.
+            Expects a struct with `pixels`, `width`, `height`, `channels`, `channel_bit_depth` fields.
+
+    Returns:
+        The encoded PNG images.
+    """
+    from spiral import _lib
+    from spiral.expressions import lift
+
+    expr = lift(expr)
+    return Expr(_lib.expr.img.encode(expr.__expr__, format="png"))

spiral/expressions/qoi.py

@@ -0,0 +1,18 @@
+from spiral.expressions.base import Expr, ExprLike
+
+
+def encode(expr: ExprLike) -> Expr:
+    """Encode the given expression as a QOI image.
+
+    Args:
+        expr: The expression to encode.
+            Expects a struct with `pixels`, `width`, `height`, `channels`, `channel_bit_depth` fields.
+
+    Returns:
+        The encoded QOI images.
+    """
+    from spiral import _lib
+    from spiral.expressions import lift
+
+    expr = lift(expr)
+    return Expr(_lib.expr.img.encode(expr.__expr__, format="qoi"))

spiral/expressions/refs.py

@@ -1,10 +1,7 @@
-from typing import TYPE_CHECKING
+import pyarrow as pa
 
 from spiral.expressions.base import Expr, ExprLike
 
-if TYPE_CHECKING:
-    from spiral import Table
-
 
 def ref(expr: ExprLike, field: str | None = None) -> Expr:
     """Store binary values as references. This expression can only be used on write.
@@ -22,23 +19,40 @@ def ref(expr: ExprLike, field: str | None = None) -> Expr:
     from spiral.expressions import lift
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.ref(expr.__expr__, field))
+    return Expr(_lib.expr.refs.ref(expr.__expr__, field))
 
 
-def deref(expr: ExprLike, field: str | None = None, table: "Table" = None) -> Expr:
+def deref(expr: ExprLike | str, field: str | None = None) -> Expr:
     """De-reference referenced values.
 
     See `ref` for more information on Spiral's reference values. This expression is used to de-reference referenced
     column back into their original form, e.g. binary.
 
     Args:
-        expr: The expression to de-reference.
+        expr: The expression to de-reference. A str is assumed to be the `se.keyed` expression.
         field: If the expr evaluates into struct, the field name of that struct that should be de-referenced.
             If `None`, the expr must evaluate into a reference type.
-        table (optional): The table to de-reference from, if not available in input expression.
     """
     from spiral import _lib
+    from spiral.expressions import keyed, lift
+
+    if isinstance(expr, str):
+        expr = keyed(
+            expr,
+            pa.struct([("__ref__", pa.struct([("id", pa.string()), ("begin", pa.uint64()), ("end", pa.uint64())]))]),
+        )
+
+    expr = lift(expr)
+    return Expr(_lib.expr.refs.deref(expr.__expr__, field=field))
+
+
+def nbytes(expr: ExprLike) -> Expr:
+    """Return the number of bytes in a reference.
+
+    Args:
+        expr: The ref expression to get the number of bytes from.
+    """
     from spiral.expressions import lift
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.deref(expr.__expr__, field, table._table if table is not None else None))
+    return expr["__ref__"]["end"] - expr["__ref__"]["begin"]

spiral/expressions/struct.py

@@ -12,10 +12,10 @@ def getitem(expr: ExprLike, field: str) -> Expr:
     from spiral import expressions as se
 
     expr = se.lift(expr)
-    return Expr(_lib.spql.expr.struct.getitem(expr.__expr__, field))
+    return Expr(_lib.expr.struct.getitem(expr.__expr__, field))
 
 
-def pack(fields: dict[str, ExprLike]) -> Expr:
+def pack(fields: dict[str, ExprLike], *, nullable: bool = False) -> Expr:
     """Assemble a new struct from the given named fields.
 
     Args:
@@ -23,7 +23,9 @@ def pack(fields: dict[str, ExprLike]) -> Expr:
     """
     from spiral import expressions as se
 
-    return Expr(_lib.spql.expr.struct.pack(list(fields.keys()), [se.lift(expr).__expr__ for expr in fields.values()]))
+    return Expr(
+        _lib.expr.struct.pack(list(fields.keys()), [se.lift(expr).__expr__ for expr in fields.values()], nullable)
+    )
 
 
 def merge(*structs: "ExprLike") -> Expr:
@@ -40,7 +42,7 @@ def merge(*structs: "ExprLike") -> Expr:
 
     if len(structs) == 1:
         return se.lift(structs[0])
-    return Expr(_lib.spql.expr.struct.merge([se.lift(struct).__expr__ for struct in structs]))
+    return Expr(_lib.expr.struct.merge([se.lift(struct).__expr__ for struct in structs]))
 
 
 def select(expr: ExprLike, names: list[str] = None, exclude: list[str] = None) -> Expr:
@@ -54,4 +56,4 @@ def select(expr: ExprLike, names: list[str] = None, exclude: list[str] = None) -
     from spiral import expressions as se
 
     expr = se.lift(expr)
-    return Expr(_lib.spql.expr.struct.select(expr.__expr__, names, exclude))
+    return Expr(_lib.expr.struct.select(expr.__expr__, names, exclude))