datachain 0.7.6__py3-none-any.whl → 0.7.8__py3-none-any.whl

datachain/func/__init__.py

@@ -1,4 +1,4 @@
-from sqlalchemy import literal
+from sqlalchemy import case, literal
 
 from . import array, path, random, string
 from .aggregate import (
@@ -17,6 +17,7 @@ from .aggregate import (
 )
 from .array import cosine_distance, euclidean_distance, length, sip_hash_64
 from .conditional import greatest, least
+from .numeric import bit_and, bit_or, bit_xor, int_hash_64
 from .random import rand
 from .window import window
 
@@ -24,6 +25,10 @@ __all__ = [
     "any_value",
     "array",
     "avg",
+    "bit_and",
+    "bit_or",
+    "bit_xor",
+    "case",
     "collect",
     "concat",
     "cosine_distance",
@@ -32,6 +37,7 @@ __all__ = [
     "euclidean_distance",
     "first",
     "greatest",
+    "int_hash_64",
     "least",
     "length",
     "literal",

datachain/func/func.py

@@ -2,11 +2,15 @@ import inspect
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from sqlalchemy import BindParameter, ColumnElement, desc
+from sqlalchemy import BindParameter, Case, ColumnElement, Integer, cast, desc
+from sqlalchemy.ext.hybrid import Comparator
+from sqlalchemy.sql import func as sa_func
 
 from datachain.lib.convert.python_to_sql import python_to_sql
+from datachain.lib.convert.sql_to_python import sql_to_python
 from datachain.lib.utils import DataChainColumnError, DataChainParamsError
 from datachain.query.schema import Column, ColumnMeta
+from datachain.sql.functions import numeric
 
 from .base import Function
 
@@ -71,7 +75,7 @@ class Func(Function):
         return (
             [
                 col
-                if isinstance(col, (Func, BindParameter))
+                if isinstance(col, (Func, BindParameter, Case, Comparator))
                 else ColumnMeta.to_db_name(
                     col.name if isinstance(col, ColumnElement) else col
                 )
@@ -96,94 +100,232 @@ class Func(Function):
         return list[col_type] if self.is_array else col_type  # type: ignore[valid-type]
 
     def __add__(self, other: Union[ColT, float]) -> "Func":
-        return math_add(self, other)
+        if isinstance(other, (int, float)):
+            return Func("add", lambda a: a + other, [self])
+        return Func("add", lambda a1, a2: a1 + a2, [self, other])
 
     def __radd__(self, other: Union[ColT, float]) -> "Func":
-        return math_add(other, self)
+        if isinstance(other, (int, float)):
+            return Func("add", lambda a: other + a, [self])
+        return Func("add", lambda a1, a2: a1 + a2, [other, self])
 
     def __sub__(self, other: Union[ColT, float]) -> "Func":
-        return math_sub(self, other)
+        if isinstance(other, (int, float)):
+            return Func("sub", lambda a: a - other, [self])
+        return Func("sub", lambda a1, a2: a1 - a2, [self, other])
 
     def __rsub__(self, other: Union[ColT, float]) -> "Func":
-        return math_sub(other, self)
+        if isinstance(other, (int, float)):
+            return Func("sub", lambda a: other - a, [self])
+        return Func("sub", lambda a1, a2: a1 - a2, [other, self])
 
     def __mul__(self, other: Union[ColT, float]) -> "Func":
-        return math_mul(self, other)
+        if isinstance(other, (int, float)):
+            return Func("mul", lambda a: a * other, [self])
+        return Func("mul", lambda a1, a2: a1 * a2, [self, other])
 
     def __rmul__(self, other: Union[ColT, float]) -> "Func":
-        return math_mul(other, self)
+        if isinstance(other, (int, float)):
+            return Func("mul", lambda a: other * a, [self])
+        return Func("mul", lambda a1, a2: a1 * a2, [other, self])
 
     def __truediv__(self, other: Union[ColT, float]) -> "Func":
-        return math_truediv(self, other)
+        if isinstance(other, (int, float)):
+            return Func("div", lambda a: _truediv(a, other), [self], result_type=float)
+        return Func(
+            "div", lambda a1, a2: _truediv(a1, a2), [self, other], result_type=float
+        )
 
     def __rtruediv__(self, other: Union[ColT, float]) -> "Func":
-        return math_truediv(other, self)
+        if isinstance(other, (int, float)):
+            return Func("div", lambda a: _truediv(other, a), [self], result_type=float)
+        return Func(
+            "div", lambda a1, a2: _truediv(a1, a2), [other, self], result_type=float
+        )
 
     def __floordiv__(self, other: Union[ColT, float]) -> "Func":
-        return math_floordiv(self, other)
+        if isinstance(other, (int, float)):
+            return Func(
+                "floordiv", lambda a: _floordiv(a, other), [self], result_type=int
+            )
+        return Func(
+            "floordiv", lambda a1, a2: _floordiv(a1, a2), [self, other], result_type=int
+        )
 
     def __rfloordiv__(self, other: Union[ColT, float]) -> "Func":
-        return math_floordiv(other, self)
+        if isinstance(other, (int, float)):
+            return Func(
+                "floordiv", lambda a: _floordiv(other, a), [self], result_type=int
+            )
+        return Func(
+            "floordiv", lambda a1, a2: _floordiv(a1, a2), [other, self], result_type=int
+        )
 
     def __mod__(self, other: Union[ColT, float]) -> "Func":
-        return math_mod(self, other)
+        if isinstance(other, (int, float)):
+            return Func("mod", lambda a: a % other, [self], result_type=int)
+        return Func("mod", lambda a1, a2: a1 % a2, [self, other], result_type=int)
 
     def __rmod__(self, other: Union[ColT, float]) -> "Func":
-        return math_mod(other, self)
-
-    def __pow__(self, other: Union[ColT, float]) -> "Func":
-        return math_pow(self, other)
-
-    def __rpow__(self, other: Union[ColT, float]) -> "Func":
-        return math_pow(other, self)
-
-    def __lshift__(self, other: Union[ColT, float]) -> "Func":
-        return math_lshift(self, other)
-
-    def __rlshift__(self, other: Union[ColT, float]) -> "Func":
-        return math_lshift(other, self)
-
-    def __rshift__(self, other: Union[ColT, float]) -> "Func":
-        return math_rshift(self, other)
-
-    def __rrshift__(self, other: Union[ColT, float]) -> "Func":
-        return math_rshift(other, self)
+        if isinstance(other, (int, float)):
+            return Func("mod", lambda a: other % a, [self], result_type=int)
+        return Func("mod", lambda a1, a2: a1 % a2, [other, self], result_type=int)
 
     def __and__(self, other: Union[ColT, float]) -> "Func":
-        return math_and(self, other)
+        if isinstance(other, (int, float)):
+            return Func(
+                "and", lambda a: numeric.bit_and(a, other), [self], result_type=int
+            )
+        return Func(
+            "and",
+            lambda a1, a2: numeric.bit_and(a1, a2),
+            [self, other],
+            result_type=int,
+        )
 
     def __rand__(self, other: Union[ColT, float]) -> "Func":
-        return math_and(other, self)
+        if isinstance(other, (int, float)):
+            return Func(
+                "and", lambda a: numeric.bit_and(other, a), [self], result_type=int
+            )
+        return Func(
+            "and",
+            lambda a1, a2: numeric.bit_and(a1, a2),
+            [other, self],
+            result_type=int,
+        )
 
     def __or__(self, other: Union[ColT, float]) -> "Func":
-        return math_or(self, other)
+        if isinstance(other, (int, float)):
+            return Func(
+                "or", lambda a: numeric.bit_or(a, other), [self], result_type=int
+            )
+        return Func(
+            "or", lambda a1, a2: numeric.bit_or(a1, a2), [self, other], result_type=int
+        )
 
     def __ror__(self, other: Union[ColT, float]) -> "Func":
-        return math_or(other, self)
+        if isinstance(other, (int, float)):
+            return Func(
+                "or", lambda a: numeric.bit_or(other, a), [self], result_type=int
+            )
+        return Func(
+            "or", lambda a1, a2: numeric.bit_or(a1, a2), [other, self], result_type=int
+        )
 
     def __xor__(self, other: Union[ColT, float]) -> "Func":
-        return math_xor(self, other)
+        if isinstance(other, (int, float)):
+            return Func(
+                "xor", lambda a: numeric.bit_xor(a, other), [self], result_type=int
+            )
+        return Func(
+            "xor",
+            lambda a1, a2: numeric.bit_xor(a1, a2),
+            [self, other],
+            result_type=int,
+        )
 
     def __rxor__(self, other: Union[ColT, float]) -> "Func":
-        return math_xor(other, self)
+        if isinstance(other, (int, float)):
+            return Func(
+                "xor", lambda a: numeric.bit_xor(other, a), [self], result_type=int
+            )
+        return Func(
+            "xor",
+            lambda a1, a2: numeric.bit_xor(a1, a2),
+            [other, self],
+            result_type=int,
+        )
+
+    def __rshift__(self, other: Union[ColT, float]) -> "Func":
+        if isinstance(other, (int, float)):
+            return Func(
+                "rshift",
+                lambda a: numeric.bit_rshift(a, other),
+                [self],
+                result_type=int,
+            )
+        return Func(
+            "rshift",
+            lambda a1, a2: numeric.bit_rshift(a1, a2),
+            [self, other],
+            result_type=int,
+        )
+
+    def __rrshift__(self, other: Union[ColT, float]) -> "Func":
+        if isinstance(other, (int, float)):
+            return Func(
+                "rshift",
+                lambda a: numeric.bit_rshift(other, a),
+                [self],
+                result_type=int,
+            )
+        return Func(
+            "rshift",
+            lambda a1, a2: numeric.bit_rshift(a1, a2),
+            [other, self],
+            result_type=int,
+        )
+
+    def __lshift__(self, other: Union[ColT, float]) -> "Func":
+        if isinstance(other, (int, float)):
+            return Func(
+                "lshift",
+                lambda a: numeric.bit_lshift(a, other),
+                [self],
+                result_type=int,
+            )
+        return Func(
+            "lshift",
+            lambda a1, a2: numeric.bit_lshift(a1, a2),
+            [self, other],
+            result_type=int,
+        )
+
+    def __rlshift__(self, other: Union[ColT, float]) -> "Func":
+        if isinstance(other, (int, float)):
+            return Func(
+                "lshift",
+                lambda a: numeric.bit_lshift(other, a),
+                [self],
+                result_type=int,
+            )
+        return Func(
+            "lshift",
+            lambda a1, a2: numeric.bit_lshift(a1, a2),
+            [other, self],
+            result_type=int,
+        )
 
     def __lt__(self, other: Union[ColT, float]) -> "Func":
-        return math_lt(self, other)
+        if isinstance(other, (int, float)):
+            return Func("lt", lambda a: a < other, [self], result_type=bool)
+        return Func("lt", lambda a1, a2: a1 < a2, [self, other], result_type=bool)
 
     def __le__(self, other: Union[ColT, float]) -> "Func":
-        return math_le(self, other)
+        if isinstance(other, (int, float)):
+            return Func("le", lambda a: a <= other, [self], result_type=bool)
+        return Func("le", lambda a1, a2: a1 <= a2, [self, other], result_type=bool)
 
     def __eq__(self, other):
-        return math_eq(self, other)
+        if isinstance(other, (int, float)):
+            return Func("eq", lambda a: a == other, [self], result_type=bool)
+        return Func("eq", lambda a1, a2: a1 == a2, [self, other], result_type=bool)
 
     def __ne__(self, other):
-        return math_ne(self, other)
+        if isinstance(other, (int, float)):
+            return Func("ne", lambda a: a != other, [self], result_type=bool)
+        return Func("ne", lambda a1, a2: a1 != a2, [self, other], result_type=bool)
 
     def __gt__(self, other: Union[ColT, float]) -> "Func":
-        return math_gt(self, other)
+        if isinstance(other, (int, float)):
+            return Func("gt", lambda a: a > other, [self], result_type=bool)
+        return Func("gt", lambda a1, a2: a1 > a2, [self, other], result_type=bool)
 
     def __ge__(self, other: Union[ColT, float]) -> "Func":
-        return math_ge(self, other)
+        if isinstance(other, (int, float)):
+            return Func("ge", lambda a: a >= other, [self], result_type=bool)
+        return Func("ge", lambda a1, a2: a1 >= a2, [self, other], result_type=bool)
 
     def label(self, label: str) -> "Func":
         return Func(
@@ -273,112 +415,20 @@ def get_db_col_type(signals_schema: "SignalSchema", col: ColT) -> "DataType":
     if isinstance(col, Func):
         return col.get_result_type(signals_schema)
 
+    if isinstance(col, ColumnElement) and not hasattr(col, "name"):
+        return sql_to_python(col)
+
     return signals_schema.get_column_type(
         col.name if isinstance(col, ColumnElement) else col
     )
 
 
-def math_func(
-    name: str,
-    inner: Callable,
-    params: Sequence[Union[ColT, float]],
-    result_type: Optional["DataType"] = None,
-) -> Func:
-    """Returns math function from the columns."""
-    cols, args = [], []
-    for arg in params:
-        if isinstance(arg, (int, float)):
-            args.append(arg)
-        else:
-            cols.append(arg)
-    return Func(name, inner, cols=cols, args=args, result_type=result_type)
-
-
-def math_add(*args: Union[ColT, float]) -> Func:
-    """Computes the sum of the column."""
-    return math_func("add", lambda a1, a2: a1 + a2, args)
-
-
-def math_sub(*args: Union[ColT, float]) -> Func:
-    """Computes the diff of the column."""
-    return math_func("sub", lambda a1, a2: a1 - a2, args)
-
-
-def math_mul(*args: Union[ColT, float]) -> Func:
-    """Computes the product of the column."""
-    return math_func("mul", lambda a1, a2: a1 * a2, args)
-
-
-def math_truediv(*args: Union[ColT, float]) -> Func:
-    """Computes the division of the column."""
-    return math_func("div", lambda a1, a2: a1 / a2, args, result_type=float)
-
-
-def math_floordiv(*args: Union[ColT, float]) -> Func:
-    """Computes the floor division of the column."""
-    return math_func("floordiv", lambda a1, a2: a1 // a2, args, result_type=float)
-
-
-def math_mod(*args: Union[ColT, float]) -> Func:
-    """Computes the modulo of the column."""
-    return math_func("mod", lambda a1, a2: a1 % a2, args, result_type=float)
-
-
-def math_pow(*args: Union[ColT, float]) -> Func:
-    """Computes the power of the column."""
-    return math_func("pow", lambda a1, a2: a1**a2, args, result_type=float)
-
-
-def math_lshift(*args: Union[ColT, float]) -> Func:
-    """Computes the left shift of the column."""
-    return math_func("lshift", lambda a1, a2: a1 << a2, args, result_type=int)
-
-
-def math_rshift(*args: Union[ColT, float]) -> Func:
-    """Computes the right shift of the column."""
-    return math_func("rshift", lambda a1, a2: a1 >> a2, args, result_type=int)
-
-
-def math_and(*args: Union[ColT, float]) -> Func:
-    """Computes the logical AND of the column."""
-    return math_func("and", lambda a1, a2: a1 & a2, args, result_type=bool)
-
-
-def math_or(*args: Union[ColT, float]) -> Func:
-    """Computes the logical OR of the column."""
-    return math_func("or", lambda a1, a2: a1 | a2, args, result_type=bool)
-
-
-def math_xor(*args: Union[ColT, float]) -> Func:
-    """Computes the logical XOR of the column."""
-    return math_func("xor", lambda a1, a2: a1 ^ a2, args, result_type=bool)
-
-
-def math_lt(*args: Union[ColT, float]) -> Func:
-    """Computes the less than comparison of the column."""
-    return math_func("lt", lambda a1, a2: a1 < a2, args, result_type=bool)
-
-
-def math_le(*args: Union[ColT, float]) -> Func:
-    """Computes the less than or equal comparison of the column."""
-    return math_func("le", lambda a1, a2: a1 <= a2, args, result_type=bool)
-
-
-def math_eq(*args: Union[ColT, float]) -> Func:
-    """Computes the equality comparison of the column."""
-    return math_func("eq", lambda a1, a2: a1 == a2, args, result_type=bool)
-
-
-def math_ne(*args: Union[ColT, float]) -> Func:
-    """Computes the inequality comparison of the column."""
-    return math_func("ne", lambda a1, a2: a1 != a2, args, result_type=bool)
-
-
-def math_gt(*args: Union[ColT, float]) -> Func:
-    """Computes the greater than comparison of the column."""
-    return math_func("gt", lambda a1, a2: a1 > a2, args, result_type=bool)
+def _truediv(a, b):
+    # Using sqlalchemy.sql.func.divide here instead of / operator
+    # because of a bug in ClickHouse SQLAlchemy dialect
+    # See https://github.com/xzkostyan/clickhouse-sqlalchemy/issues/335
+    return sa_func.divide(a, b)
 
 
-def math_ge(*args: Union[ColT, float]) -> Func:
-    """Computes the greater than or equal comparison of the column."""
-    return math_func("ge", lambda a1, a2: a1 >= a2, args, result_type=bool)
+def _floordiv(a, b):
+    return cast(_truediv(a, b), Integer)

datachain/func/numeric.py

@@ -0,0 +1,162 @@
+from typing import Union
+
+from datachain.sql.functions import numeric
+
+from .func import ColT, Func
+
+
+def bit_and(*args: Union[ColT, int]) -> Func:
+    """
+    Computes the bitwise AND operation between two values.
+
+    Args:
+        args (str | int): Two values to compute the bitwise AND operation between.
+            If a string is provided, it is assumed to be the name of the column vector.
+            If an integer is provided, it is assumed to be a constant value.
+
+    Returns:
+        Func: A Func object that represents the bitwise AND function.
+
+    Example:
+        ```py
+        dc.mutate(
+            xor1=func.bit_and("signal.values", 0x0F),
+        )
+        ```
+
+    Notes:
+        - Result column will always be of type int.
+    """
+    cols, func_args = [], []
+    for arg in args:
+        if isinstance(arg, int):
+            func_args.append(arg)
+        else:
+            cols.append(arg)
+
+    if len(cols) + len(func_args) != 2:
+        raise ValueError("bit_and() requires exactly two arguments")
+
+    return Func(
+        "bit_and",
+        inner=numeric.bit_and,
+        cols=cols,
+        args=func_args,
+        result_type=int,
+    )
+
+
+def bit_or(*args: Union[ColT, int]) -> Func:
+    """
+    Computes the bitwise AND operation between two values.
+
+    Args:
+        args (str | int): Two values to compute the bitwise OR operation between.
+            If a string is provided, it is assumed to be the name of the column vector.
+            If an integer is provided, it is assumed to be a constant value.
+
+    Returns:
+        Func: A Func object that represents the bitwise OR function.
+
+    Example:
+        ```py
+        dc.mutate(
+            xor1=func.bit_or("signal.values", 0x0F),
+        )
+        ```
+
+    Notes:
+        - Result column will always be of type int.
+    """
+    cols, func_args = [], []
+    for arg in args:
+        if isinstance(arg, int):
+            func_args.append(arg)
+        else:
+            cols.append(arg)
+
+    if len(cols) + len(func_args) != 2:
+        raise ValueError("bit_or() requires exactly two arguments")
+
+    return Func(
+        "bit_or",
+        inner=numeric.bit_or,
+        cols=cols,
+        args=func_args,
+        result_type=int,
+    )
+
+
+def bit_xor(*args: Union[ColT, int]) -> Func:
+    """
+    Computes the bitwise XOR operation between two values.
+
+    Args:
+        args (str | int): Two values to compute the bitwise XOR operation between.
+            If a string is provided, it is assumed to be the name of the column vector.
+            If an integer is provided, it is assumed to be a constant value.
+
+    Returns:
+        Func: A Func object that represents the bitwise XOR function.
+
+    Example:
+        ```py
+        dc.mutate(
+            xor1=func.bit_xor("signal.values", 0x0F),
+        )
+        ```
+
+    Notes:
+        - Result column will always be of type int.
+    """
+    cols, func_args = [], []
+    for arg in args:
+        if isinstance(arg, int):
+            func_args.append(arg)
+        else:
+            cols.append(arg)
+
+    if len(cols) + len(func_args) != 2:
+        raise ValueError("bit_xor() requires exactly two arguments")
+
+    return Func(
+        "bit_xor",
+        inner=numeric.bit_xor,
+        cols=cols,
+        args=func_args,
+        result_type=int,
+    )
+
+
+def int_hash_64(col: Union[ColT, int]) -> Func:
+    """
+    Returns the 64-bit hash of an integer.
+
+    Args:
+        col (str | int): String to compute the hash of.
+            If a string is provided, it is assumed to be the name of the column.
+            If a int is provided, it is assumed to be an int literal.
+            If a Func is provided, it is assumed to be a function returning an int.
+
+    Returns:
+        Func: A Func object that represents the 64-bit hash function.
+
+    Example:
+        ```py
+        dc.mutate(
+            val_hash=func.int_hash_64("val"),
+        )
+        ```
+
+    Note:
+        - Result column will always be of type int.
+    """
+    cols, args = [], []
+    if isinstance(col, int):
+        args.append(col)
+    else:
+        cols.append(col)
+
+    return Func(
+        "int_hash_64", inner=numeric.int_hash_64, cols=cols, args=args, result_type=int
+    )

datachain/lib/dc.py

@@ -1150,7 +1150,7 @@ class DataChain:
     def group_by(
         self,
         *,
-        partition_by: Union[str, Func, Sequence[Union[str, Func]]],
+        partition_by: Optional[Union[str, Func, Sequence[Union[str, Func]]]] = None,
         **kwargs: Func,
     ) -> "Self":
         """Group rows by specified set of signals and return new signals
@@ -1167,10 +1167,10 @@ class DataChain:
             )
             ```
         """
-        if isinstance(partition_by, (str, Func)):
+        if partition_by is None:
+            partition_by = []
+        elif isinstance(partition_by, (str, Func)):
             partition_by = [partition_by]
-        if not partition_by:
-            raise ValueError("At least one column should be provided for partition_by")
 
         partition_by_columns: list[Column] = []
         signal_columns: list[Column] = []

datachain/query/dataset.py

@@ -966,8 +966,6 @@ class SQLGroupBy(SQLClause):
     def apply_sql_clause(self, query) -> Select:
         if not self.cols:
             raise ValueError("No columns to select")
-        if not self.group_by:
-            raise ValueError("No columns to group by")
 
         subquery = query.subquery()
 

datachain/sql/functions/array.py

@@ -38,6 +38,10 @@ class length(GenericFunction):  # noqa: N801
 
 
 class sip_hash_64(GenericFunction):  # noqa: N801
+    """
+    Computes the SipHash-64 hash of the array.
+    """
+
     type = Int64()
     package = "hash"
     name = "sip_hash_64"

datachain/sql/functions/numeric.py

@@ -0,0 +1,43 @@
+from sqlalchemy.sql.functions import GenericFunction, ReturnTypeFromArgs
+
+from datachain.sql.types import Int64
+from datachain.sql.utils import compiler_not_implemented
+
+
+class bit_and(ReturnTypeFromArgs):  # noqa: N801
+    inherit_cache = True
+
+
+class bit_or(ReturnTypeFromArgs):  # noqa: N801
+    inherit_cache = True
+
+
+class bit_xor(ReturnTypeFromArgs):  # noqa: N801
+    inherit_cache = True
+
+
+class bit_rshift(ReturnTypeFromArgs):  # noqa: N801
+    inherit_cache = True
+
+
+class bit_lshift(ReturnTypeFromArgs):  # noqa: N801
+    inherit_cache = True
+
+
+class int_hash_64(GenericFunction):  # noqa: N801
+    """
+    Computes the 64-bit hash of an integer.
+    """
+
+    type = Int64()
+    package = "hash"
+    name = "int_hash_64"
+    inherit_cache = True
+
+
+compiler_not_implemented(bit_and)
+compiler_not_implemented(bit_or)
+compiler_not_implemented(bit_xor)
+compiler_not_implemented(bit_rshift)
+compiler_not_implemented(bit_lshift)
+compiler_not_implemented(int_hash_64)

datachain/sql/sqlite/base.py

@@ -15,7 +15,14 @@ from sqlalchemy.sql.elements import literal
 from sqlalchemy.sql.expression import case
 from sqlalchemy.sql.functions import func
 
-from datachain.sql.functions import aggregate, array, conditional, random, string
+from datachain.sql.functions import (
+    aggregate,
+    array,
+    conditional,
+    numeric,
+    random,
+    string,
+)
 from datachain.sql.functions import path as sql_path
 from datachain.sql.selectable import Values, base_values_compiler
 from datachain.sql.sqlite.types import (
@@ -47,6 +54,8 @@ slash = literal("/")
 empty_str = literal("")
 dot = literal(".")
 
+MAX_INT64 = 2**64 - 1
+
 
 def setup():
     global setup_is_complete  # noqa: PLW0603
@@ -89,6 +98,12 @@ def setup():
     compiles(aggregate.group_concat, "sqlite")(compile_group_concat)
     compiles(aggregate.any_value, "sqlite")(compile_any_value)
     compiles(aggregate.collect, "sqlite")(compile_collect)
+    compiles(numeric.bit_and, "sqlite")(compile_bitwise_and)
+    compiles(numeric.bit_or, "sqlite")(compile_bitwise_or)
+    compiles(numeric.bit_xor, "sqlite")(compile_bitwise_xor)
+    compiles(numeric.bit_rshift, "sqlite")(compile_bitwise_rshift)
+    compiles(numeric.bit_lshift, "sqlite")(compile_bitwise_lshift)
+    compiles(numeric.int_hash_64, "sqlite")(compile_int_hash_64)
 
     if load_usearch_extension(sqlite3.connect(":memory:")):
         compiles(array.cosine_distance, "sqlite")(compile_cosine_distance_ext)
@@ -163,6 +178,19 @@ def sqlite_string_split(string: str, sep: str, maxsplit: int = -1) -> str:
     return orjson.dumps(string.split(sep, maxsplit)).decode("utf-8")
 
 
+def sqlite_int_hash_64(x: int) -> int:
+    """IntHash64 implementation from ClickHouse."""
+    x ^= 0x4CF2D2BAAE6DA887
+    x ^= x >> 33
+    x = (x * 0xFF51AFD7ED558CCD) & MAX_INT64
+    x ^= x >> 33
+    x = (x * 0xC4CEB9FE1A85EC53) & MAX_INT64
+    x ^= x >> 33
+    # SQLite does not support unsigned 64-bit integers,
+    # so we need to convert to signed 64-bit
+    return x if x < 1 << 63 else (x & MAX_INT64) - (1 << 64)
+
+
 def register_user_defined_sql_functions() -> None:
     # Register optional functions if we have the necessary dependencies
     # and otherwise register functions that will raise an exception with
@@ -185,6 +213,21 @@ def register_user_defined_sql_functions() -> None:
 
     _registered_function_creators["vector_functions"] = create_vector_functions
 
+    def create_numeric_functions(conn):
+        conn.create_function("divide", 2, lambda a, b: a / b, deterministic=True)
+        conn.create_function("bitwise_and", 2, lambda a, b: a & b, deterministic=True)
+        conn.create_function("bitwise_or", 2, lambda a, b: a | b, deterministic=True)
+        conn.create_function("bitwise_xor", 2, lambda a, b: a ^ b, deterministic=True)
+        conn.create_function(
+            "bitwise_rshift", 2, lambda a, b: a >> b, deterministic=True
+        )
+        conn.create_function(
+            "bitwise_lshift", 2, lambda a, b: a << b, deterministic=True
+        )
+        conn.create_function("int_hash_64", 1, sqlite_int_hash_64, deterministic=True)
+
+    _registered_function_creators["numeric_functions"] = create_numeric_functions
+
     def sqlite_regexp_replace(string: str, pattern: str, replacement: str) -> str:
         return re.sub(pattern, replacement, string)
 
@@ -316,6 +359,30 @@ def compile_euclidean_distance(element, compiler, **kwargs):
     return f"euclidean_distance({compiler.process(element.clauses, **kwargs)})"
 
 
+def compile_bitwise_and(element, compiler, **kwargs):
+    return compiler.process(func.bitwise_and(*element.clauses.clauses), **kwargs)
+
+
+def compile_bitwise_or(element, compiler, **kwargs):
+    return compiler.process(func.bitwise_or(*element.clauses.clauses), **kwargs)
+
+
+def compile_bitwise_xor(element, compiler, **kwargs):
+    return compiler.process(func.bitwise_xor(*element.clauses.clauses), **kwargs)
+
+
+def compile_bitwise_rshift(element, compiler, **kwargs):
+    return compiler.process(func.bitwise_rshift(*element.clauses.clauses), **kwargs)
+
+
+def compile_bitwise_lshift(element, compiler, **kwargs):
+    return compiler.process(func.bitwise_lshift(*element.clauses.clauses), **kwargs)
+
+
+def compile_int_hash_64(element, compiler, **kwargs):
+    return compiler.process(func.int_hash_64(*element.clauses.clauses), **kwargs)
+
+
 def py_json_array_length(arr):
     return len(orjson.loads(arr))
 

{datachain-0.7.6.dist-info → datachain-0.7.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.6
+Version: 0.7.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.7.6.dist-info → datachain-0.7.8.dist-info}/RECORD

@@ -37,12 +37,13 @@ datachain/data_storage/schema.py,sha256=-QVlRvD0dfu-ZFUxylEoSnLJLnleMEjVlcAb2OGu
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=D_ZQ0PHmZzHO2dinv4naVJocUDIZUwV4WAz692C1cyk,22521
 datachain/data_storage/warehouse.py,sha256=tjIkU-5JywBR0apCyqTcwSyaRtGxhu2L7IVjrz-55uc,30802
-datachain/func/__init__.py,sha256=4VUt5BaLdBAl_BnAku0Jb8plqd7kDOiYrQTMG3pN0c4,794
+datachain/func/__init__.py,sha256=oz-GbCcp5jnN82u6cghWTGzmU9IQvtvllOof73wE52g,934
 datachain/func/aggregate.py,sha256=7_IPrIwb2XSs3zG4iOr1eTvzn6kNVe2mkzvNzjusDHk,10942
 datachain/func/array.py,sha256=zHDNWuWLA7HVa9FEvQeHhVi00_xqenyleTqcLwkXWBI,5477
 datachain/func/base.py,sha256=wA0sBQAVyN9LPxoo7Ox83peS0zUVnyuKxukwAcjGLfY,534
 datachain/func/conditional.py,sha256=mQroxsoExpBW84Zm5dAYP4OpBblWmzfnF2qJq9rba54,2223
-datachain/func/func.py,sha256=9wqdxxisoDL0w8qKGQmL6sNdgJeIOzotEUPlxu9t2IQ,12326
+datachain/func/func.py,sha256=mJ_rOXMpoqnK4-d5eF9boSMx5hWzgKoMLPGpZQqLAfw,15222
+datachain/func/numeric.py,sha256=GcUX6ijZvzfac8CZrHE0gRc9WCPiutcMLKqNXtbn-Yo,4186
 datachain/func/path.py,sha256=mqN_mfkwv44z2II7DMTp_fGGw95hmTCNls_TOFNpr4k,3155
 datachain/func/random.py,sha256=pENOLj9rSmWfGCnOsUIaCsVC5486zQb66qfQvXaz9Z4,452
 datachain/func/string.py,sha256=NQzaXXYu7yb72HPADy4WrFlcgvTS77L9x7-qvCKJtnk,4522
@@ -52,7 +53,7 @@ datachain/lib/arrow.py,sha256=b5efxAUaNNYVwtXVJqj07D3zf5KC-BPlLCxKEZbEG6w,9429
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=zS4lmXHVBXc9ntcyea2a1CRLXGSAN_0glXcF88CohgY,2685
 datachain/lib/dataset_info.py,sha256=IjdF1E0TQNOq9YyynfWiCFTeZpbyGfyJvxgJY4YN810,2493
-datachain/lib/dc.py,sha256=J7liATKQBJCkeHanVLr0s3d1t5wxiiiSJuSbuxKBbLg,89527
+datachain/lib/dc.py,sha256=t5y5tsYyU7uuk3gEPPhhcDSZ1tL1aHkKG2W54eHiUq8,89492
 datachain/lib/file.py,sha256=-XMkL6ED1sE7TMhWoMRTEuOXswZJw8X6AEmJDONFP74,15019
 datachain/lib/hf.py,sha256=a-zFpDmZIR4r8dlNNTjfpAKSnuJ9xyRXlgcdENiXt3E,5864
 datachain/lib/image.py,sha256=AMXYwQsmarZjRbPCZY3M1jDsM2WAB_b3cTY4uOIuXNU,2675
@@ -87,7 +88,7 @@ datachain/model/ultralytics/pose.py,sha256=71KBTcoST2wcEtsyGXqLVpvUtqbp9gwZGA15p
 datachain/model/ultralytics/segment.py,sha256=Z1ab0tZRJubSYNH4KkFlzhYeGNTfAyC71KmkQcToHDQ,2760
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=5fEhORFe7li12SdYddaSK3LyqksMfCHhwN1_A6TfsA4,3485
-datachain/query/dataset.py,sha256=o9Ssa47t1IM78qcaoCeTL-rp4fZCpYfR7XFjw2hGWeY,54632
+datachain/query/dataset.py,sha256=J6SbCLnFlZgCxRchc3tVk5tcC7xo1Hp616JGlEZXCDo,54547
 datachain/query/dispatch.py,sha256=fZ0TgGFRcsrYh1iXQoZVjkUl4Xetom9PSHoeDes3IRs,11606
 datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
@@ -104,21 +105,22 @@ datachain/sql/default/__init__.py,sha256=XQ2cEZpzWiABqjV-6yYHUBGI9vN_UHxbxZENESm
 datachain/sql/default/base.py,sha256=QD-31C6JnyOXzogyDx90sUhm7QvgXIYpeHEASH84igU,628
 datachain/sql/functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/sql/functions/aggregate.py,sha256=3AQdA8YHPFdtCEfwZKQXTT8SlQWdG9gD5PBtGN3Odqs,944
-datachain/sql/functions/array.py,sha256=rvH27SWN9gdh_mFnp0GIiXuCrNW6n8ZbY4I_JUS-_e0,1140
+datachain/sql/functions/array.py,sha256=Zq59CaMHf_hFapU4kxvy2mwteH344k5Wksxja4MfBks,1204
 datachain/sql/functions/conditional.py,sha256=q7YUKfunXeEldXaxgT-p5pUTcOEVU_tcQ2BJlquTRPs,207
+datachain/sql/functions/numeric.py,sha256=DFTTEWsvBBXwbaaC4zdxhAoqUYwI6nbymG-nzbzdPv8,972
 datachain/sql/functions/path.py,sha256=zixpERotTFP6LZ7I4TiGtyRA8kXOoZmH1yzH9oRW0mg,1294
 datachain/sql/functions/random.py,sha256=vBwEEj98VH4LjWixUCygQ5Bz1mv1nohsCG0-ZTELlVg,271
 datachain/sql/functions/string.py,sha256=DYgiw8XSk7ge7GXvyRI1zbaMruIizNeI-puOjriQGZQ,1148
 datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7d04,166
-datachain/sql/sqlite/base.py,sha256=X4iEynOAqqvqz8lmgUKvURleKO6aguULgG8RoufKrSk,14772
+datachain/sql/sqlite/base.py,sha256=eQv2U32jChG9tnYSFE4SS2Mvfb7-W3Ok3Ffhew9qkKI,17254
 datachain/sql/sqlite/types.py,sha256=lPXS1XbkmUtlkkiRxy_A_UzsgpPv2VSkXYOD4zIHM4w,1734
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ZgDcrNiKiPXZmKD591_1z9qRIXitu5zwAsoVPB7ykiU,2508
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.7.6.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.7.6.dist-info/METADATA,sha256=KMChqSG7d_lMaF9BYNIgmijvnxZbDm5gCEg980gUGOA,18006
-datachain-0.7.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-datachain-0.7.6.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.7.6.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.7.6.dist-info/RECORD,,
+datachain-0.7.8.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.7.8.dist-info/METADATA,sha256=r8znUWHdmY5y6hk8N9NFdlrKaHKkteeji7NXJTb2Ges,18006
+datachain-0.7.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+datachain-0.7.8.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.7.8.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.7.8.dist-info/RECORD,,