datachain 0.14.2__py3-none-any.whl → 0.39.0__py3-none-any.whl

datachain/func/path.py

@@ -8,23 +8,26 @@ def parent(col: ColT) -> Func:
     Returns the directory component of a posix-style path.
 
     Args:
-        col (str | literal | Func): String to compute the path parent of.
+        col (str | Column | Func | literal): String to compute the path parent of.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column object.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the path parent function.
+        Func: A `Func` object that represents the path parent function.
 
     Example:
         ```py
         dc.mutate(
-            parent=func.path.parent("file.path"),
+            parent1=func.path.parent("file.path"),
+            parent2=func.path.parent(dc.C("file.path")),
+            parent3=func.path.parent(dc.func.literal("/path/to/file.txt")),
         )
         ```
 
     Note:
-        - Result column will always be of type string.
+        - The result column will always be of type string.
     """
     return Func("parent", inner=path.parent, cols=[col], result_type=str)
 
@@ -34,23 +37,26 @@ def name(col: ColT) -> Func:
     Returns the final component of a posix-style path.
 
     Args:
-        col (str | literal): String to compute the path name of.
+        col (str | Column | Func | literal): String to compute the path name of.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column object.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the path name function.
+        Func: A `Func` object that represents the path name function.
 
     Example:
         ```py
         dc.mutate(
-            file_name=func.path.name("file.path"),
+            filename1=func.path.name("file.path"),
+            filename2=func.path.name(dc.C("file.path")),
+            filename3=func.path.name(dc.func.literal("/path/to/file.txt")
         )
         ```
 
     Note:
-        - Result column will always be of type string.
+        - The result column will always be of type string.
     """
 
     return Func("name", inner=path.name, cols=[col], result_type=str)
@@ -61,23 +67,26 @@ def file_stem(col: ColT) -> Func:
     Returns the path without the extension.
 
     Args:
-        col (str | literal): String to compute the file stem of.
+        col (str | Column | Func | literal): String to compute the file stem of.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column object.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the file stem function.
+        Func: A `Func` object that represents the file stem function.
 
     Example:
         ```py
         dc.mutate(
-            file_stem=func.path.file_stem("file.path"),
+            filestem1=func.path.file_stem("file.path"),
+            filestem2=func.path.file_stem(dc.C("file.path")),
+            filestem3=func.path.file_stem(dc.func.literal("/path/to/file.txt")
         )
         ```
 
     Note:
-        - Result column will always be of type string.
+        - The result column will always be of type string.
     """
 
     return Func("file_stem", inner=path.file_stem, cols=[col], result_type=str)
@@ -88,23 +97,26 @@ def file_ext(col: ColT) -> Func:
     Returns the extension of the given path.
 
     Args:
-        col (str | literal): String to compute the file extension of.
+        col (str | Column | Func | literal): String to compute the file extension of.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column object.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the file extension function.
+        Func: A `Func` object that represents the file extension function.
 
     Example:
         ```py
         dc.mutate(
-            file_stem=func.path.file_ext("file.path"),
+            filestem1=func.path.file_ext("file.path"),
+            filestem2=func.path.file_ext(dc.C("file.path")),
+            filestem3=func.path.file_ext(dc.func.literal("/path/to/file.txt")
         )
         ```
 
     Note:
-        - Result column will always be of type string.
+        - The result column will always be of type string.
     """
 
     return Func("file_ext", inner=path.file_ext, cols=[col], result_type=str)

datachain/func/random.py

@@ -8,7 +8,7 @@ def rand() -> Func:
     Returns the random integer value.
 
     Returns:
-        Func: A Func object that represents the rand function.
+        Func: A `Func` object that represents the rand function.
 
     Example:
         ```py
@@ -18,6 +18,6 @@ def rand() -> Func:
         ```
 
     Note:
-        - Result column will always be of type integer.
+        - The result column will always be of type integer.
     """
     return Func("rand", inner=random.rand, result_type=int)

datachain/func/string.py

@@ -1,64 +1,76 @@
-from typing import Optional, Union, get_origin
+from typing import get_origin
 
 from sqlalchemy import literal
 
 from datachain.sql.functions import string
 
-from .func import Func
+from .func import ColT, Func
 
+__all__ = [
+    "byte_hamming_distance",
+    "length",
+    "regexp_replace",
+    "replace",
+    "split",
+]
 
-def length(col: Union[str, Func]) -> Func:
+
+def length(col: ColT) -> Func:
     """
     Returns the length of the string.
 
     Args:
-        col (str | literal | Func): String to compute the length of.
+        col (str | Column | Func | literal): String to compute the length of.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column in the dataset.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the string length function.
+        Func: A `Func` object that represents the string length function.
 
     Example:
         ```py
         dc.mutate(
             len1=func.string.length("file.path"),
-            len2=func.string.length("Random string"),
+            len2=func.string.length(dc.C("file.path")),
+            len3=func.string.length(dc.func.literal("Random string")),
         )
         ```
 
-    Note:
-        - Result column will always be of type int.
+    Notes:
+        - The result column will always be of type int.
     """
     return Func("length", inner=string.length, cols=[col], result_type=int)
 
 
-def split(col: Union[str, Func], sep: str, limit: Optional[int] = None) -> Func:
+def split(col: ColT, sep: str, limit: int | None = None) -> Func:
     """
     Takes a column and split character and returns an array of the parts.
 
     Args:
-        col (str | literal): Column to split.
+        col (str | Column | Func | literal): Column to split.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column in the dataset.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
         sep (str): Separator to split the string.
         limit (int, optional): Maximum number of splits to perform.
 
     Returns:
-        Func: A Func object that represents the split function.
+        Func: A `Func` object that represents the split function.
 
     Example:
         ```py
         dc.mutate(
             path_parts=func.string.split("file.path", "/"),
-            str_words=func.string.length("Random string", " "),
+            signal_values=func.string.split(dc.C("signal.value"), ","),
+            str_words=func.string.split(dc.func.literal("Random string"), " "),
         )
         ```
 
-    Note:
-        - Result column will always be of type array of strings.
+    Notes:
+        - The result column will always be of type array of strings.
     """
 
     def inner(arg):
@@ -76,30 +88,33 @@ def split(col: Union[str, Func], sep: str, limit: Optional[int] = None) -> Func:
     return Func("split", inner=inner, cols=cols, args=args, result_type=list[str])
 
 
-def replace(col: Union[str, Func], pattern: str, replacement: str) -> Func:
+def replace(col: ColT, pattern: str, replacement: str) -> Func:
     """
     Replaces substring with another string.
 
     Args:
-        col (str | literal): Column to split.
+        col (str | Column | Func | literal): Column to perform replacement on.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column in the dataset.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
         pattern (str): Pattern to replace.
         replacement (str): Replacement string.
 
     Returns:
-        Func: A Func object that represents the replace function.
+        Func: A `Func` object that represents the replace function.
 
     Example:
         ```py
         dc.mutate(
-            signal=func.string.replace("signal.name", "pattern", "replacement),
+            s1=func.string.replace("signal.name", "pattern", "replacement"),
+            s2=func.string.replace(dc.C("signal.name"), "pattern", "replacement"),
+            s3=func.string.replace(dc.func.literal("Random string"), "Random", "New"),
         )
         ```
 
-    Note:
-        - Result column will always be of type string.
+    Notes:
+        - The result column will always be of type string.
     """
 
     def inner(arg):
@@ -115,30 +130,37 @@ def replace(col: Union[str, Func], pattern: str, replacement: str) -> Func:
     return Func("replace", inner=inner, cols=cols, args=args, result_type=str)
 
 
-def regexp_replace(col: Union[str, Func], regex: str, replacement: str) -> Func:
+def regexp_replace(col: ColT, regex: str, replacement: str) -> Func:
     r"""
     Replaces substring that match a regular expression.
 
     Args:
-        col (str | literal): Column to split.
+        col (str | Column | Func | literal): Column to perform replacement on.
             If a string is provided, it is assumed to be the name of the column.
-            If a literal is provided, it is assumed to be a string literal.
+            If a Column is provided, it is assumed to be a column in the dataset.
             If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
         regex (str): Regular expression pattern to replace.
         replacement (str): Replacement string.
 
     Returns:
-        Func: A Func object that represents the regexp_replace function.
+        Func: A `Func` object that represents the regexp_replace function.
 
     Example:
         ```py
         dc.mutate(
-            signal=func.string.regexp_replace("signal.name", r"\d+", "X"),
+            s1=func.string.regexp_replace("signal.name", r"\d+", "X"),
+            s2=func.string.regexp_replace(dc.C("signal.name"), r"\d+", "X"),
+            s3=func.string.regexp_replace(
+                dc.func.literal("Random string"),
+                r"\s+",
+                "_",
+            ),
         )
         ```
 
-    Note:
-        - Result column will always be of type string.
+    Notes:
+        - The result column will always be of type string.
     """
 
     def inner(arg):
@@ -154,7 +176,7 @@ def regexp_replace(col: Union[str, Func], regex: str, replacement: str) -> Func:
     return Func("regexp_replace", inner=inner, cols=cols, args=args, result_type=str)
 
 
-def byte_hamming_distance(*args: Union[str, Func]) -> Func:
+def byte_hamming_distance(*args: ColT) -> Func:
     """
     Computes the Hamming distance between two strings.
 
@@ -164,22 +186,30 @@ def byte_hamming_distance(*args: Union[str, Func]) -> Func:
     of the strings indicate higher dissimilarity.
 
     Args:
-        args (str | literal): Two strings to compute the Hamming distance between.
-            If a str is provided, it is assumed to be the name of the column.
-            If a Literal is provided, it is assumed to be a string literal.
+        args (str | Column | Func | literal): Two strings to compute
+            the Hamming distance between.
+            If a string is provided, it is assumed to be the name of the column.
+            If a Column is provided, it is assumed to be a column in the dataset.
+            If a Func is provided, it is assumed to be a function returning a string.
+            If a literal is provided, it is assumed to be a string literal.
 
     Returns:
-        Func: A Func object that represents the Hamming distance function.
+        Func: A `Func` object that represents the Hamming distance function.
 
     Example:
         ```py
         dc.mutate(
-            ham_dist=func.byte_hamming_distance("file.phash", literal("hello")),
+            hd1=func.byte_hamming_distance("file.phash", literal("hello")),
+            hd2=func.byte_hamming_distance(dc.C("file.phash"), "hello"),
+            hd3=func.byte_hamming_distance(
+                dc.func.literal("hi"),
+                dc.func.literal("hello"),
+            ),
         )
         ```
 
     Notes:
-        - Result column will always be of type int.
+        - The result column will always be of type int.
     """
     cols, func_args = [], []
     for arg in args:

datachain/func/window.py

@@ -22,17 +22,16 @@ def window(partition_by: str, order_by: str, desc: bool = False) -> Window:
 
     Args:
         partition_by (str): The column name by which to partition the result set.
-                            Rows with the same value in the partition column
-                            will be grouped together for the window function.
-        order_by (str): The column name by which to order the rows
-                        within each partition. This determines the sequence in which
-                        the window function is applied.
+            Rows with the same value in the partition column will be grouped together
+            for the window function.
+        order_by (str): The column name by which to order the rows within
+            each partition. This determines the sequence in which the window function
+            is applied.
         desc (bool, optional): If True, the rows will be ordered in descending order.
-                               Defaults to False, which orders the rows
-                               in ascending order.
+            Defaults to False, which orders the rows in ascending order.
 
     Returns:
-        Window: A Window object representing the window specification.
+        Window: A `Window` object representing the window specification.
 
     Example:
         ```py

datachain/hash_utils.py

@@ -0,0 +1,123 @@
+import hashlib
+import inspect
+import textwrap
+from collections.abc import Sequence
+from typing import TypeAlias, TypeVar
+
+from sqlalchemy.sql.elements import ClauseElement, ColumnElement
+
+from datachain import json
+
+T = TypeVar("T", bound=ColumnElement)
+ColumnLike: TypeAlias = str | T
+
+
+def _serialize_value(val):  # noqa: PLR0911
+    """Helper to serialize arbitrary values recursively."""
+    if val is None:
+        return None
+    if isinstance(val, (str, int, float, bool)):
+        return val
+    if isinstance(val, ClauseElement):
+        return serialize_column_element(val)
+    if isinstance(val, dict):
+        # Sort dict keys for deterministic serialization
+        return {k: _serialize_value(v) for k, v in sorted(val.items())}
+    if isinstance(val, (list, tuple)):
+        return [_serialize_value(v) for v in val]
+    if callable(val):
+        return val.__name__ if hasattr(val, "__name__") else str(val)
+    return str(val)
+
+
+def serialize_column_element(expr: str | ColumnElement) -> dict:
+    """
+    Recursively serialize a SQLAlchemy ColumnElement into a deterministic structure.
+    Uses SQLAlchemy's _traverse_internals to automatically handle all expression types.
+    """
+    from sqlalchemy.sql.elements import BindParameter
+
+    # Special case: BindParameter has non-deterministic 'key' attribute, only use value
+    if isinstance(expr, BindParameter):
+        return {"type": "bind", "value": _serialize_value(expr.value)}
+
+    # Generic handling for all ClauseElement types using SQLAlchemy's internals
+    if isinstance(expr, ClauseElement):
+        # All standard SQLAlchemy types have _traverse_internals
+        if hasattr(expr, "_traverse_internals"):
+            result = {"type": expr.__class__.__name__}
+            for attr_name, _ in expr._traverse_internals:
+                # Skip 'table' attribute - table names can be auto-generated/random
+                # and are not semantically important for hashing
+                if attr_name == "table":
+                    continue
+                if hasattr(expr, attr_name):
+                    val = getattr(expr, attr_name)
+                    result[attr_name] = _serialize_value(val)
+            return result
+        # Rare case: custom user-defined ClauseElement without _traverse_internals
+        # We don't know its structure, so just stringify it
+        return {"type": expr.__class__.__name__, "repr": str(expr)}
+
+    # Absolute fallback: stringify completely unknown types
+    return {"type": "other", "repr": str(expr)}
+
+
+def hash_column_elements(columns: ColumnLike | Sequence[ColumnLike]) -> str:
+    """
+    Hash a list of ColumnElements deterministically, dialect agnostic.
+    Only accepts ordered iterables (like list or tuple).
+    """
+    # Handle case where a single ColumnElement is passed instead of a sequence
+    if isinstance(columns, (ColumnElement, str)):
+        columns = (columns,)
+
+    serialized = [serialize_column_element(c) for c in columns]
+    json_str = json.dumps(
+        serialized, sort_keys=True, separators=(", ", ": ")
+    )  # stable JSON
+    return hashlib.sha256(json_str.encode("utf-8")).hexdigest()
+
+
+def hash_callable(func):
+    """
+    Calculate a hash from a callable.
+    Rules:
+    - Named functions (def) → use source code for stable, cross-version hashing
+    - Lambdas → use bytecode (deterministic in same Python runtime)
+    """
+    if not callable(func):
+        raise TypeError("Expected a callable")
+
+    # Determine if it is a lambda
+    is_lambda = func.__name__ == "<lambda>"
+
+    if not is_lambda:
+        # Try to get exact source of named function
+        try:
+            lines, _ = inspect.getsourcelines(func)
+            payload = textwrap.dedent("".join(lines)).strip()
+        except (OSError, TypeError):
+            # Fallback: bytecode if source not available
+            payload = func.__code__.co_code
+    else:
+        # For lambdas, fall back directly to bytecode
+        payload = func.__code__.co_code
+
+    # Normalize annotations
+    annotations = {
+        k: getattr(v, "__name__", str(v)) for k, v in func.__annotations__.items()
+    }
+
+    # Extras to distinguish functions with same code but different metadata
+    extras = {
+        "name": func.__name__,
+        "defaults": func.__defaults__,
+        "annotations": annotations,
+    }
+
+    # Compute SHA256
+    h = hashlib.sha256()
+    h.update(str(payload).encode() if isinstance(payload, str) else payload)
+    h.update(str(extras).encode())
+    return h.hexdigest()

datachain/job.py

@@ -1,8 +1,9 @@
-import json
 import uuid
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Optional, TypeVar, Union
+from typing import Any, TypeVar
+
+from datachain import json
 
 J = TypeVar("J", bound="Job")
 
@@ -18,27 +19,29 @@ class Job:
     workers: int
     params: dict[str, str]
     metrics: dict[str, Any]
-    finished_at: Optional[datetime] = None
-    python_version: Optional[str] = None
+    finished_at: datetime | None = None
+    python_version: str | None = None
     error_message: str = ""
     error_stack: str = ""
+    parent_job_id: str | None = None
 
     @classmethod
     def parse(
         cls,
-        id: Union[str, uuid.UUID],
+        id: str | uuid.UUID,
         name: str,
         status: int,
         created_at: datetime,
-        finished_at: Optional[datetime],
+        finished_at: datetime | None,
         query: str,
         query_type: int,
         workers: int,
-        python_version: Optional[str],
+        python_version: str | None,
         error_message: str,
         error_stack: str,
         params: str,
         metrics: str,
+        parent_job_id: str | None,
     ) -> "Job":
         return cls(
             str(id),
@@ -54,4 +57,5 @@ class Job:
             python_version,
             error_message,
             error_stack,
+            str(parent_job_id) if parent_job_id else None,
         )