pyspiral 0.3.1__cp310-abi3-macosx_11_0_arm64.whl → 0.4.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/core/{spec → table/spec}/__init__.pyi

@@ -14,6 +14,24 @@ class ColumnGroup:
     @staticmethod
     def from_str(path: str) -> ColumnGroup: ...
 
+class KeySpaceMetadata:
+    def __init__(
+        self,
+        *,
+        manifest_handle: ManifestHandle | None,
+        last_modified_at: int,
+    ): ...
+
+    manifest_handle: ManifestHandle | None
+    last_modified_at: int
+
+    def asof(self, asof: int) -> KeySpaceMetadata:
+        """Returns the metadata as of a given timestamp. Currently just filtering versioned schemas."""
+        ...
+
+    def apply_wal(self, wal: WriteAheadLog) -> KeySpaceMetadata:
+        """Applies the given WAL to the metadata."""
+
 class ColumnGroupMetadata:
     def __init__(
         self,
@@ -44,17 +62,16 @@ class ColumnGroupMetadata:
     def apply_wal(self, wal: WriteAheadLog) -> ColumnGroupMetadata:
         """Applies the given WAL to the metadata."""
 
-    def __bytes__(self):
-        """Serializes the ColumnGroupMetadata to a protobuf buffer."""
-
-    @staticmethod
-    def from_proto(buffer: bytes) -> ColumnGroupMetadata:
-        """Deserializes a ColumnGroupMetadata from a protobuf buffer."""
-        ...
-
 class LogEntry:
     ts: int
-    operation: KeySpaceWriteOp | FragmentSetWriteOp | ConfigurationOp | SchemaEvolutionOp | SchemaBreakOp
+    operation: (
+        KeySpaceWriteOp
+        | ColumnGroupWriteOp
+        | SchemaEvolutionOp
+        | SchemaBreakOp
+        | KeySpaceCompactOp
+        | ColumnGroupCompactOp
+    )
 
     def column_group(self) -> ColumnGroup | None:
         """Returns the column group of the entry if it is associated with one."""
@@ -71,6 +88,10 @@ class FileFormat:
         """Returns the protobuf enum int value."""
         ...
 
+    def __str__(self) -> str:
+        """Returns the string representation of the file format."""
+        ...
+
 class FragmentLevel:
     L0: FragmentLevel
     L1: FragmentLevel
@@ -80,15 +101,9 @@ class FragmentLevel:
         ...
 
 class Key:
-    def __init__(self, key: bytes): ...
-
     key: bytes
 
-    def __add__(self, other: Key) -> Key:
-        """Concatenates two keys.
-
-        TODO(ngates): remove this function. It should not be necessary to concatenate keys."""
-
+    def __init__(self, key: bytes): ...
     def __bytes__(self): ...
     def step(self) -> Key:
         """Returns the next key in the key space."""
@@ -97,8 +112,6 @@ class Key:
     def min() -> Key: ...
     @staticmethod
     def max() -> Key: ...
-    @staticmethod
-    def from_array_tuple(array_tuple: tuple[pa.Array]) -> Key: ...
 
 class KeyExtent:
     """An inclusive range of keys."""
@@ -108,9 +121,6 @@ class KeyExtent:
     min: Key
     max: Key
 
-    def to_range(self) -> KeyRange:
-        """Turn this inclusive key extent into an exclusive key range."""
-
     def union(self, key_extent: KeyExtent) -> KeyExtent: ...
     def __or__(self, other: KeyExtent) -> KeyExtent: ...
     def intersection(self, key_extent: KeyExtent) -> KeyExtent | None: ...
@@ -118,30 +128,6 @@ class KeyExtent:
     def contains(self, item: Key) -> bool: ...
     def __contains__(self, item: Key) -> bool: ...
 
-class KeyRange:
-    """A right-exclusive range of keys."""
-
-    def __init__(self, *, begin: Key, end: Key): ...
-
-    begin: Key
-    end: Key
-
-    def union(self, other: KeyRange) -> KeyRange: ...
-    def __or__(self, other: KeyRange) -> KeyRange: ...
-    def intersection(self, key_extent: KeyRange) -> KeyRange | None: ...
-    def __and__(self, other: KeyRange) -> KeyRange | None: ...
-    def contains(self, item: Key) -> bool: ...
-    def __contains__(self, item: Key) -> bool: ...
-    def is_disjoint(self, key_range: KeyRange) -> bool:
-        return self.end <= key_range.begin or self.begin >= key_range.end
-
-    @staticmethod
-    def beginning_with(begin: Key) -> KeyRange: ...
-    @staticmethod
-    def ending_with(end: Key) -> KeyRange: ...
-    @staticmethod
-    def full() -> KeyRange: ...
-
 class KeySpan:
     """An exclusive range of keys as indexed by their position in a key space."""
 
@@ -155,16 +141,10 @@ class KeySpan:
     def union(self, other: KeySpan) -> KeySpan: ...
     def __or__(self, other: KeySpan) -> KeySpan: ...
 
-class KeyMap:
-    """Displacement map."""
-
 class ManifestHandle:
-    def __init__(self, id: str, format: FileFormat, file_size: int, spfs_format_metadata: bytes | None): ...
-
     id: str
     format: FileFormat
     file_size: int
-    spfs_format_metadata: bytes | None
 
 class Schema:
     def to_arrow(self) -> pa.Schema:
@@ -191,24 +171,28 @@ class KeySpaceWriteOp:
     ks_id: str
     manifest_handle: ManifestHandle
 
-class FragmentSetWriteOp:
+class ColumnGroupWriteOp:
     column_group: ColumnGroup
-    fs_id: str
-    fs_level: FragmentLevel
+    level: FragmentLevel
     manifest_handle: ManifestHandle
     key_span: KeySpan
     key_extent: KeyExtent
     column_ids: list[str]
 
-class ConfigurationOp:
-    column_group: ColumnGroup
-
 class SchemaEvolutionOp:
     column_group: ColumnGroup
 
 class SchemaBreakOp:
     column_group: ColumnGroup
 
+class KeySpaceCompactOp:
+    ks_ids: list[str]
+    moved_ks_ids: list[str]
+
+class ColumnGroupCompactOp:
+    column_group: ColumnGroup
+    fragment_ids: list[int]
+
 class WriteAheadLog:
     def __init__(
         self,
@@ -219,7 +203,6 @@ class WriteAheadLog:
 
     entries: list[LogEntry]
     truncated_up_to: int
-    ks_manifest_handle: ManifestHandle | None
 
     @property
     def last_modified_at(self) -> int:
@@ -229,11 +212,3 @@ class WriteAheadLog:
         self, asof: int | None = None, since: int | None = None, column_group: ColumnGroup | None = None
     ) -> WriteAheadLog:
         """Filters the WAL to entries by the given parameters."""
-
-    def __bytes__(self):
-        """Serializes the ColumnGroupMetadata to a protobuf buffer."""
-
-    @staticmethod
-    def from_proto(buffer: bytes) -> WriteAheadLog:
-        """Deserializes a WriteAheadLog from a protobuf buffer."""
-        ...

spiral/datetime_.py

@@ -0,0 +1,27 @@
+import warnings
+from datetime import datetime, timedelta, timezone, tzinfo
+
+_THE_EPOCH = datetime.fromtimestamp(0, tz=timezone.utc)
+
+
+def local_tz() -> tzinfo:
+    """Determine this machine's local timezone."""
+    tz = datetime.now().astimezone().tzinfo
+    if tz is None:
+        raise ValueError("Could not determine this machine's local timezone.")
+    return tz
+
+
+def timestamp_micros(instant: datetime) -> int:
+    """The number of microseconds between the epoch and the given instant."""
+    if instant.tzinfo is None:
+        warnings.warn("assuming timezone-naive datetime is local time", stacklevel=2)
+        instant = instant.replace(tzinfo=local_tz())
+    return (instant - _THE_EPOCH) // timedelta(microseconds=1)
+
+
+def from_timestamp_micros(ts: int) -> datetime:
+    """Convert a timestamp in microseconds to a datetime."""
+    if ts < 0:
+        raise ValueError("Timestamp must be non-negative")
+    return _THE_EPOCH + timedelta(microseconds=ts)

spiral/expressions/__init__.py

@@ -1,11 +1,12 @@
 import builtins
 import functools
 import operator
+import warnings
 from typing import Any
 
 import pyarrow as pa
 
-from spiral import _lib, arrow
+from spiral import _lib, arrow_
 
 from . import http as http
 from . import io as io
@@ -16,8 +17,9 @@ from . import qoi as qoi
 from . import refs as refs
 from . import str_ as str
 from . import struct as struct
+from . import text as text
 from . import tiff as tiff
-from .base import Expr, ExprLike
+from .base import Expr, ExprLike, NativeExpr
 
 __all__ = [
     "Expr",
@@ -54,11 +56,11 @@ __all__ = [
     "struct",
     "subtract",
     "tiff",
-    "var",
     "xor",
     "png",
     "qoi",
     "mp4",
+    "text",
 ]
 
 # Inline some of the struct expressions since they're so common
@@ -74,13 +76,15 @@ def lift(expr: ExprLike) -> Expr:
     # Convert an ExprLike into an Expr.
     if isinstance(expr, Expr):
         return expr
+    if isinstance(expr, NativeExpr):
+        return Expr(expr)
 
     if isinstance(expr, dict):
         # NOTE: we assume this is a struct expression. We could be smarter and be context aware to determine if
         # this is in fact a struct scalar, but the user can always create one of those manually.
 
         # First we un-nest any dot-separated field names
-        expr: dict = arrow.nest_structs(expr)
+        expr: dict = arrow_.nest_structs(expr)
 
         return pack({k: lift(v) for k, v in expr.items()})
 
@@ -95,14 +99,18 @@ def lift(expr: ExprLike) -> Expr:
 
     # If the value is struct-like, we un-nest any dot-separated field names
     if isinstance(expr, pa.StructArray | pa.StructScalar):
+        # TODO(marko): Figure out what to do with nullable struct arrays when unpacking them.
+        #   We need to merge struct validity into the child validity?
         if isinstance(expr, pa.StructArray) and expr.null_count != 0:
-            raise ValueError("lift: cannot lift a struct array with nulls.")
-        if isinstance(expr, pa.StructArray) and not expr.is_valid():
-            raise ValueError("lift: cannot lift a struct scalar with nulls.")
-        return lift(arrow.nest_structs(expr))
+            # raise ValueError("lift: cannot lift a struct array with nulls.")
+            warnings.warn("found a struct array with nulls", stacklevel=2)
+        if isinstance(expr, pa.StructScalar) and not expr.is_valid():
+            # raise ValueError("lift: cannot lift a struct scalar with nulls.")
+            warnings.warn("found a struct scalar with nulls", stacklevel=2)
+        return lift(arrow_.nest_structs(expr))
 
     if isinstance(expr, pa.Array):
-        return Expr(_lib.spql.expr.array_lit(expr))
+        return Expr(_lib.expr.array_lit(expr))
 
     # Otherwise, assume it's a scalar.
     return scalar(expr)
@@ -114,7 +122,7 @@ def key(name: builtins.str) -> Expr:
     Args:
         name: variable name
     """
-    return Expr(_lib.spql.expr.keyed(name))
+    return Expr(_lib.expr.keyed(name))
 
 
 def keyed(name: builtins.str, dtype: pa.DataType) -> Expr:
@@ -127,20 +135,21 @@ def keyed(name: builtins.str, dtype: pa.DataType) -> Expr:
         name: variable name
         dtype: must match dtype of the column in the key table.
     """
-    return Expr(_lib.spql.expr.keyed(name, dtype))
+    return Expr(_lib.expr.keyed(name, dtype))
 
 
 def scalar(value: Any) -> Expr:
     """Create a scalar expression."""
     if not isinstance(value, pa.Scalar):
         value = pa.scalar(value)
-    return Expr(_lib.spql.expr.scalar(value))
+    # TODO(marko): Use Vortex scalar instead of passing as array.
+    return Expr(_lib.expr.scalar(pa.array([value.as_py()], type=value.type)))
 
 
 def cast(expr: ExprLike, dtype: pa.DataType) -> Expr:
     """Cast an expression into another PyArrow DataType."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.cast(expr.__expr__, dtype))
+    return Expr(_lib.expr.cast(expr.__expr__, dtype))
 
 
 def and_(expr: ExprLike, *exprs: ExprLike) -> Expr:
@@ -197,19 +206,18 @@ def negate(expr: ExprLike) -> Expr:
 def not_(expr: ExprLike) -> Expr:
     """Negate the given expression."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("not", expr.__expr__))
+    return Expr(_lib.expr.not_(expr.__expr__))
 
 
 def is_null(expr: ExprLike) -> Expr:
     """Check if the given expression is null."""
     expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("is_null", expr.__expr__))
+    return Expr(_lib.expr.is_null(expr.__expr__))
 
 
 def is_not_null(expr: ExprLike) -> Expr:
-    """Check if the given expression is null."""
-    expr = lift(expr)
-    return Expr(_lib.spql.expr.unary("is_not_null", expr.__expr__))
+    """Check if the given expression is not null."""
+    return not_(is_null(expr))
 
 
 def add(lhs: ExprLike, rhs: ExprLike) -> Expr:

spiral/expressions/base.py

@@ -5,7 +5,7 @@ import pyarrow as pa
 
 from spiral import _lib
 
-NativeExpr: TypeAlias = _lib.spql.expr.Expr
+NativeExpr: TypeAlias = _lib.expr.Expr
 
 
 class Expr:
@@ -92,13 +92,13 @@ class Expr:
         return self._binary("mod", other)
 
     def __neg__(self):
-        return Expr(_lib.spql.expr.unary("neg", self.__expr__))
+        return Expr(_lib.expr.unary("neg", self.__expr__))
 
     def in_(self, other: "ExprLike") -> "Expr":
         from spiral import expressions as se
 
         other = se.lift(other)
-        return Expr(_lib.spql.expr.list.contains(other.__expr__, self.__expr__))
+        return Expr(_lib.expr.list.contains(other.__expr__, self.__expr__))
 
     def contains(self, other: "ExprLike") -> "Expr":
         from spiral import expressions as se
@@ -107,7 +107,7 @@ class Expr:
 
     def cast(self, dtype: pa.DataType) -> "Expr":
         """Cast the expression result to a different data type."""
-        return Expr(_lib.spql.expr.cast(self.__expr__, dtype))
+        return Expr(_lib.expr.cast(self.__expr__, dtype))
 
     def select(self, *paths: str, exclude: list[str] = None) -> "Expr":
         """Select fields from a struct-like expression.
@@ -142,7 +142,7 @@ class Expr:
         from spiral import expressions as se
 
         rhs = se.lift(rhs)
-        return Expr(_lib.spql.expr.binary(op, self.__expr__, rhs.__expr__))
+        return Expr(_lib.expr.binary(op, self.__expr__, rhs.__expr__))
 
 
 ScalarLike: TypeAlias = bool | int | float | str | list | datetime.datetime | None

spiral/expressions/list_.py

@@ -29,7 +29,7 @@ def element_at(expr: ExprLike, index: ExprLike) -> Expr:
 
     expr = lift(expr)
     index = lift(index)
-    return Expr(_lib.spql.expr.list.element_at(expr.__expr__, index.__expr__))
+    return Expr(_lib.expr.list.element_at(expr.__expr__, index.__expr__))
 
 
 def of(*expr: ExprLike) -> Expr:

spiral/expressions/mp4.py

@@ -1,12 +1,7 @@
-from typing import TYPE_CHECKING
-
 import pyarrow as pa
 
 from spiral.expressions.base import Expr, ExprLike
 
-if TYPE_CHECKING:
-    from spiral import Table
-
 _MP4_RES_DTYPE: pa.DataType = pa.struct(
     [
         pa.field("pixels", pa.large_binary()),
@@ -19,7 +14,7 @@ _MP4_RES_DTYPE: pa.DataType = pa.struct(
 
 # TODO(marko): Support optional range and crop.
 #   IMPORTANT: Frames is currently broken and defaults to full.
-def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str, *, table: "Table" = None):
+def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str):
     """
     Read referenced cell in a `MP4` format. Requires `ffmpeg`.
 
@@ -32,7 +27,6 @@ def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str, *,
         crop: The crop of the frames to read. Each element must be a list of four uint32,
             x, y, width, height or null / empty list to read full frames.
             A str is assumed to be the `se.keyed` expression.
-        table (optional): The table to de-reference from, if not available in input expression.
 
     Returns:
         An array where each element is a decoded cropped video with fields:
@@ -59,11 +53,10 @@ def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str, *,
     crop = lift(crop)
 
     return Expr(
-        _lib.spql.expr.video.read(
+        _lib.expr.video.read(
             expr.__expr__,
             frames.__expr__,
             crop.__expr__,
             format="mp4",
-            table=table._table if table is not None else None,
         )
     )

spiral/expressions/png.py

@@ -15,4 +15,4 @@ def encode(expr: ExprLike) -> Expr:
     from spiral.expressions import lift
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.img.encode(expr.__expr__, format="png"))
+    return Expr(_lib.expr.img.encode(expr.__expr__, format="png"))

spiral/expressions/qoi.py

@@ -15,4 +15,4 @@ def encode(expr: ExprLike) -> Expr:
     from spiral.expressions import lift
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.img.encode(expr.__expr__, format="qoi"))
+    return Expr(_lib.expr.img.encode(expr.__expr__, format="qoi"))

spiral/expressions/refs.py

@@ -1,12 +1,7 @@
-from typing import TYPE_CHECKING
-
 import pyarrow as pa
 
 from spiral.expressions.base import Expr, ExprLike
 
-if TYPE_CHECKING:
-    from spiral import Table
-
 
 def ref(expr: ExprLike, field: str | None = None) -> Expr:
     """Store binary values as references. This expression can only be used on write.
@@ -24,10 +19,10 @@ def ref(expr: ExprLike, field: str | None = None) -> Expr:
     from spiral.expressions import lift
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.ref(expr.__expr__, field))
+    return Expr(_lib.expr.refs.ref(expr.__expr__, field))
 
 
-def deref(expr: ExprLike | str, field: str | None = None, *, table: "Table" = None) -> Expr:
+def deref(expr: ExprLike | str, field: str | None = None) -> Expr:
     """De-reference referenced values.
 
     See `ref` for more information on Spiral's reference values. This expression is used to de-reference referenced
@@ -37,7 +32,6 @@ def deref(expr: ExprLike | str, field: str | None = None, *, table: "Table" = No
         expr: The expression to de-reference. A str is assumed to be the `se.keyed` expression.
         field: If the expr evaluates into struct, the field name of that struct that should be de-referenced.
             If `None`, the expr must evaluate into a reference type.
-        table (optional): The table to de-reference from, if not available in input expression.
     """
     from spiral import _lib
     from spiral.expressions import keyed, lift
@@ -49,7 +43,7 @@ def deref(expr: ExprLike | str, field: str | None = None, *, table: "Table" = No
         )
 
     expr = lift(expr)
-    return Expr(_lib.spql.expr.deref(expr.__expr__, field=field, table=table._table if table is not None else None))
+    return Expr(_lib.expr.refs.deref(expr.__expr__, field=field))
 
 
 def nbytes(expr: ExprLike) -> Expr:

spiral/expressions/struct.py

@@ -12,10 +12,10 @@ def getitem(expr: ExprLike, field: str) -> Expr:
     from spiral import expressions as se
 
     expr = se.lift(expr)
-    return Expr(_lib.spql.expr.struct.getitem(expr.__expr__, field))
+    return Expr(_lib.expr.struct.getitem(expr.__expr__, field))
 
 
-def pack(fields: dict[str, ExprLike]) -> Expr:
+def pack(fields: dict[str, ExprLike], *, nullable: bool = False) -> Expr:
     """Assemble a new struct from the given named fields.
 
     Args:
@@ -23,7 +23,9 @@ def pack(fields: dict[str, ExprLike]) -> Expr:
     """
     from spiral import expressions as se
 
-    return Expr(_lib.spql.expr.struct.pack(list(fields.keys()), [se.lift(expr).__expr__ for expr in fields.values()]))
+    return Expr(
+        _lib.expr.struct.pack(list(fields.keys()), [se.lift(expr).__expr__ for expr in fields.values()], nullable)
+    )
 
 
 def merge(*structs: "ExprLike") -> Expr:
@@ -40,7 +42,7 @@ def merge(*structs: "ExprLike") -> Expr:
 
     if len(structs) == 1:
         return se.lift(structs[0])
-    return Expr(_lib.spql.expr.struct.merge([se.lift(struct).__expr__ for struct in structs]))
+    return Expr(_lib.expr.struct.merge([se.lift(struct).__expr__ for struct in structs]))
 
 
 def select(expr: ExprLike, names: list[str] = None, exclude: list[str] = None) -> Expr:
@@ -54,4 +56,4 @@ def select(expr: ExprLike, names: list[str] = None, exclude: list[str] = None) -
     from spiral import expressions as se
 
     expr = se.lift(expr)
-    return Expr(_lib.spql.expr.struct.select(expr.__expr__, names, exclude))
+    return Expr(_lib.expr.struct.select(expr.__expr__, names, exclude))

spiral/expressions/text.py

@@ -0,0 +1,62 @@
+from spiral.expressions.base import Expr, ExprLike
+
+
+def field(expr: ExprLike, field_name: str | None = None, tokenizer: str | None = None) -> Expr:
+    """Configure a column for text indexing.
+
+    Args:
+        expr: An input column. The expression must either evaluate to a UTF-8,
+            or, if a `field_name` is provided, to a struct with a field of that name.
+        field_name: If provided, the expression must evaluate to a struct with a field of that name.
+            The given field will be indexed.
+        tokenizer: If provided, the text will be tokenized using the given tokenizer.
+
+    Returns:
+        An expression that can be used to construct a text index.
+    """
+    from spiral import _lib
+    from spiral.expressions import getitem, lift, merge, pack
+
+    expr = lift(expr)
+    if field_name is None:
+        return Expr(_lib.expr.text.field(expr.__expr__, tokenizer))
+
+    child = _lib.expr.text.field(getitem(expr, field_name).__expr__)
+    return merge(
+        expr,
+        pack({field_name: child}),
+    )
+
+
+def find(expr: ExprLike, term: str) -> Expr:
+    """Search for a term in the text.
+
+    Args:
+        expr: An index field.
+        term: The term to search for.
+
+    Returns:
+        An expression that can be used in ranking for text search.
+    """
+    from spiral import _lib
+    from spiral.expressions import lift
+
+    expr = lift(expr)
+    return Expr(_lib.expr.text.find(expr.__expr__, term))
+
+
+def boost(expr: ExprLike, factor: float) -> Expr:
+    """Boost the relevance of a ranking expression.
+
+    Args:
+        expr: Rank by expression.
+        factor: The factor by which to boost the relevance.
+
+    Returns:
+        An expression that can be used in ranking for text search.
+    """
+    from spiral import _lib
+    from spiral.expressions import lift
+
+    expr = lift(expr)
+    return Expr(_lib.expr.text.boost(expr.__expr__, factor))

spiral/expressions/udf.py

@@ -25,7 +25,7 @@ class UDF(BaseUDF):
     """A User-Defined Function (UDF)."""
 
     def __init__(self, name: str):
-        super().__init__(_lib.spql.expr.udf.create(name, return_type=self.return_type, invoke=self.invoke))
+        super().__init__(_lib.expr.udf.create(name, return_type=self.return_type, invoke=self.invoke))
 
     @abc.abstractmethod
     def invoke(self, *input_args: pa.Array) -> pa.Array: ...
@@ -35,10 +35,10 @@ class RefUDF(BaseUDF):
     """A UDF over a single ref cell, and therefore can access the file object."""
 
     def __init__(self, name: str):
-        super().__init__(_lib.spql.expr.udf.create(name, return_type=self.return_type, invoke=self.invoke, scope="ref"))
+        super().__init__(_lib.expr.udf.create(name, return_type=self.return_type, invoke=self.invoke, scope="ref"))
 
     @abc.abstractmethod
-    def invoke(self, fp, *input_args: pa.Array) -> pa.Array:
+    def invoke(self, fp: _lib.FileObject, *input_args: pa.Array) -> pa.Array:
         """Invoke the UDF with the given arguments.
 
         NOTE: The first argument is always the ref cell. All array input args will be sliced to the appropriate row.

spiral/iceberg/__init__.py

@@ -0,0 +1,3 @@
+from spiral.iceberg.client import Iceberg
+
+__all__ = ["Iceberg"]

spiral/iceberg/client.py

@@ -0,0 +1,33 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pyiceberg.catalog import Catalog
+
+    from spiral.client import Spiral
+
+
+class Iceberg:
+    """
+    Apache Iceberg is a powerful open-source table format designed for high-performance data lakes.
+    Iceberg brings reliability, scalability, and advanced features like time travel, schema evolution,
+    and ACID transactions to your warehouse.
+    """
+
+    def __init__(self, spiral: "Spiral", *, project_id: str | None = None):
+        self._spiral = spiral
+        self._project_id = project_id
+
+        self._api = self._spiral.config.api
+
+    def catalog(self) -> "Catalog":
+        """Open the Iceberg catalog."""
+        from pyiceberg.catalog import load_catalog
+
+        return load_catalog(
+            "default",
+            **{
+                "type": "rest",
+                "uri": self._spiral.config.spiraldb.uri + "/iceberg",
+                "token": self._spiral.config.authn.token().expose_secret(),
+            },
+        )

spiral/indexes/__init__.py

@@ -0,0 +1,5 @@
+from spiral.indexes.client import Indexes
+from spiral.indexes.index import TextIndex
+from spiral.indexes.scan import SearchScan
+
+__all__ = ["Indexes", "SearchScan", "TextIndex"]