datachain 0.34.5__py3-none-any.whl → 0.34.7__py3-none-any.whl

datachain/lib/video.py

@@ -1,7 +1,6 @@
 import posixpath
 import shutil
 import tempfile
-from typing import Optional, Union
 
 from numpy import ndarray
 
@@ -18,7 +17,7 @@ except ImportError as exc:
     ) from exc
 
 
-def video_info(file: Union[File, VideoFile]) -> Video:
+def video_info(file: File | VideoFile) -> Video:
     """
     Returns video file information.
 
@@ -108,7 +107,7 @@ def video_frame_np(video: VideoFile, frame: int) -> ndarray:
 def validate_frame_range(
     video: VideoFile,
     start: int = 0,
-    end: Optional[int] = None,
+    end: int | None = None,
     step: int = 1,
 ) -> tuple[int, int, int]:
     """
@@ -186,7 +185,7 @@ def save_video_fragment(
     start: float,
     end: float,
     output: str,
-    format: Optional[str] = None,
+    format: str | None = None,
 ) -> VideoFile:
     """
     Saves video interval as a new video file. If output is a remote path,

datachain/lib/webdataset.py

@@ -1,17 +1,10 @@
 import json
 import tarfile
+import types
 import warnings
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from pathlib import Path
-from typing import (
-    Any,
-    Callable,
-    ClassVar,
-    Optional,
-    Union,
-    get_args,
-    get_origin,
-)
+from typing import Any, ClassVar, Union, get_args, get_origin
 
 from pydantic import Field
 
@@ -64,28 +57,28 @@ class WDSBasic(DataModel):
 
 
 class WDSAllFile(WDSBasic):
-    txt: Optional[str] = Field(default=None)
-    text: Optional[str] = Field(default=None)
-    cap: Optional[str] = Field(default=None)
-    transcript: Optional[str] = Field(default=None)
-    cls: Optional[int] = Field(default=None)
-    cls2: Optional[int] = Field(default=None)
-    index: Optional[int] = Field(default=None)
-    inx: Optional[int] = Field(default=None)
-    id: Optional[int] = Field(default=None)
-    json: Optional[dict] = Field(default=None)  # type: ignore[assignment]
-    jsn: Optional[dict] = Field(default=None)
-
-    pyd: Optional[bytes] = Field(default=None)
-    pickle: Optional[bytes] = Field(default=None)
-    pth: Optional[bytes] = Field(default=None)
-    ten: Optional[bytes] = Field(default=None)
-    tb: Optional[bytes] = Field(default=None)
-    mp: Optional[bytes] = Field(default=None)
-    msg: Optional[bytes] = Field(default=None)
-    npy: Optional[bytes] = Field(default=None)
-    npz: Optional[bytes] = Field(default=None)
-    cbor: Optional[bytes] = Field(default=None)
+    txt: str | None = Field(default=None)
+    text: str | None = Field(default=None)
+    cap: str | None = Field(default=None)
+    transcript: str | None = Field(default=None)
+    cls: int | None = Field(default=None)
+    cls2: int | None = Field(default=None)
+    index: int | None = Field(default=None)
+    inx: int | None = Field(default=None)
+    id: int | None = Field(default=None)
+    json: dict | None = Field(default=None)  # type: ignore[assignment]
+    jsn: dict | None = Field(default=None)
+
+    pyd: bytes | None = Field(default=None)
+    pickle: bytes | None = Field(default=None)
+    pth: bytes | None = Field(default=None)
+    ten: bytes | None = Field(default=None)
+    tb: bytes | None = Field(default=None)
+    mp: bytes | None = Field(default=None)
+    msg: bytes | None = Field(default=None)
+    npy: bytes | None = Field(default=None)
+    npz: bytes | None = Field(default=None)
+    cbor: bytes | None = Field(default=None)
 
 
 class WDSReadableSubclass(DataModel):
@@ -189,9 +182,11 @@ class Builder:
             return
 
         anno = field.annotation
-        if get_origin(anno) == Union:
-            args = get_args(anno)
-            anno = args[0]
+        anno_origin = get_origin(anno)
+        if anno_origin in (Union, types.UnionType):
+            anno_args = get_args(anno)
+            if len(anno_args) == 2 and type(None) in anno_args:
+                return anno_args[0] if anno_args[1] is type(None) else anno_args[1]
 
         return anno
 

datachain/lib/webdataset_laion.py

@@ -1,6 +1,5 @@
 import warnings
 from collections.abc import Iterator
-from typing import Optional
 
 import numpy as np
 from pydantic import BaseModel, Field
@@ -23,18 +22,18 @@ warnings.filterwarnings(
 
 class Laion(WDSReadableSubclass):
     uid: str = Field(default="")
-    face_bboxes: Optional[list[list[float]]] = Field(default=None)
-    caption: Optional[str] = Field(default=None)
-    url: Optional[str] = Field(default=None)
-    key: Optional[str] = Field(default=None)
-    status: Optional[str] = Field(default=None)
-    error_message: Optional[str] = Field(default=None)
-    width: Optional[int] = Field(default=None)
-    height: Optional[int] = Field(default=None)
-    original_width: Optional[int] = Field(default=None)
-    original_height: Optional[int] = Field(default=None)
-    exif: Optional[str] = Field(default=None)
-    sha256: Optional[str] = Field(default=None)
+    face_bboxes: list[list[float]] | None = Field(default=None)
+    caption: str | None = Field(default=None)
+    url: str | None = Field(default=None)
+    key: str | None = Field(default=None)
+    status: str | None = Field(default=None)
+    error_message: str | None = Field(default=None)
+    width: int | None = Field(default=None)
+    height: int | None = Field(default=None)
+    original_width: int | None = Field(default=None)
+    original_height: int | None = Field(default=None)
+    exif: str | None = Field(default=None)
+    sha256: str | None = Field(default=None)
 
     @staticmethod
     def _reader(builder, item):
@@ -42,13 +41,13 @@ class Laion(WDSReadableSubclass):
 
 
 class WDSLaion(WDSBasic):
-    txt: Optional[str] = Field(default=None)
-    json: Laion  # type: ignore[assignment]
+    txt: str | None = Field(default=None)
+    json: Laion = Field(default_factory=Laion)  # type: ignore[assignment]
 
 
 class LaionMeta(BaseModel):
     file: File
-    index: Optional[int] = Field(default=None)
+    index: int | None = Field(default=None)
     b32_img: list[float] = Field(default=[])
     b32_txt: list[float] = Field(default=[])
     l14_img: list[float] = Field(default=[])

datachain/listing.py

@@ -2,7 +2,7 @@ import glob
 import os
 from collections.abc import Iterable, Iterator
 from functools import cached_property
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from sqlalchemy import Column
 from sqlalchemy.sql import func
@@ -25,8 +25,8 @@ class Listing:
         metastore: "AbstractMetastore",
         warehouse: "AbstractWarehouse",
         client: "Client",
-        dataset_name: Optional["str"] = None,
-        dataset_version: Optional[str] = None,
+        dataset_name: str | None = None,
+        dataset_version: str | None = None,
         column: str = "file",
     ):
         self.metastore = metastore
@@ -102,7 +102,7 @@ class Listing:
     def collect_nodes_to_instantiate(
         self,
         sources: Iterable["DataSource"],
-        copy_to_filename: Optional[str],
+        copy_to_filename: str | None,
         recursive=False,
         copy_dir_contents=False,
         from_dataset=False,

datachain/model/bbox.py

@@ -198,7 +198,9 @@ class BBox(DataModel):
     def pose_inside(self, pose: Union["Pose", "Pose3D"]) -> bool:
         """Return True if the pose is inside the bounding box."""
         return all(
-            self.point_inside(x, y) for x, y in zip(pose.x, pose.y) if x > 0 or y > 0
+            self.point_inside(x, y)
+            for x, y in zip(pose.x, pose.y, strict=False)
+            if x > 0 or y > 0
         )
 
     @staticmethod

datachain/namespace.py

@@ -1,7 +1,7 @@
 import builtins
 from dataclasses import dataclass, fields
 from datetime import datetime
-from typing import Any, Optional, TypeVar
+from typing import Any, TypeVar
 
 from datachain.error import InvalidNamespaceNameError
 
@@ -9,7 +9,7 @@ N = TypeVar("N", bound="Namespace")
 NAMESPACE_NAME_RESERVED_CHARS = [".", "@"]
 
 
-def parse_name(name: str) -> tuple[str, Optional[str]]:
+def parse_name(name: str) -> tuple[str, str | None]:
     """
     Parses namespace name into namespace and optional project name.
     If both namespace and project are defined in name, they need to be split by dot
@@ -33,7 +33,7 @@ class Namespace:
     id: int
     uuid: str
     name: str
-    descr: Optional[str]
+    descr: str | None
     created_at: datetime
 
     @staticmethod
@@ -73,7 +73,7 @@ class Namespace:
         id: int,
         uuid: str,
         name: str,
-        descr: Optional[str],
+        descr: str | None,
         created_at: datetime,
     ) -> "Namespace":
         return cls(id, uuid, name, descr, created_at)

datachain/node.py

@@ -1,6 +1,6 @@
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any
 
 import attrs
 
@@ -53,11 +53,11 @@ class Node:
     sys__rand: int = 0
     path: str = ""
     etag: str = ""
-    version: Optional[str] = None
+    version: str | None = None
     is_latest: bool = True
-    last_modified: Optional[datetime] = None
+    last_modified: datetime | None = None
     size: int = 0
-    location: Optional[str] = None
+    location: str | None = None
     source: StorageURI = StorageURI("")  # noqa: RUF009
     dir_type: int = DirType.FILE
 
@@ -90,7 +90,7 @@ class Node:
             return self.path + "/"
         return self.path
 
-    def to_file(self, source: Optional[StorageURI] = None) -> File:
+    def to_file(self, source: StorageURI | None = None) -> File:
         if source is None:
             source = self.source
         return File(
@@ -189,7 +189,7 @@ class NodeWithPath:
 TIME_FMT = "%Y-%m-%d %H:%M"
 
 
-def long_line_str(name: str, timestamp: Optional[datetime]) -> str:
+def long_line_str(name: str, timestamp: datetime | None) -> str:
     if timestamp is None:
         time = "-"
     else:

datachain/nodes_thread_pool.py

@@ -1,4 +1,3 @@
-import concurrent
 import concurrent.futures
 import threading
 from abc import ABC, abstractmethod

datachain/plugins.py

@@ -17,13 +17,7 @@ def ensure_plugins_loaded() -> None:
 
     # Compatible across importlib.metadata versions
     eps_obj = importlib_metadata.entry_points()
-    if hasattr(eps_obj, "select"):
-        eps_list = eps_obj.select(group="datachain.callables")
-    else:
-        # Compatibility for older versions of importlib_metadata, Python 3.9
-        eps_list = eps_obj.get("datachain.callables", [])  # type: ignore[attr-defined]
-
-    for ep in eps_list:
+    for ep in eps_obj.select(group="datachain.callables"):
         func = ep.load()
         func()
 

datachain/project.py

@@ -1,7 +1,7 @@
 import builtins
 from dataclasses import dataclass, fields
 from datetime import datetime
-from typing import Any, Optional, TypeVar
+from typing import Any, TypeVar
 
 from datachain.error import InvalidProjectNameError
 from datachain.namespace import Namespace
@@ -15,7 +15,7 @@ class Project:
     id: int
     uuid: str
     name: str
-    descr: Optional[str]
+    descr: str | None
     created_at: datetime
     namespace: Namespace
 
@@ -52,12 +52,12 @@ class Project:
         namespace_id: int,
         namespace_uuid: str,
         namespace_name: str,
-        namespace_descr: Optional[str],
+        namespace_descr: str | None,
         namespace_created_at: datetime,
         project_id: int,
         uuid: str,
         name: str,
-        descr: Optional[str],
+        descr: str | None,
         created_at: datetime,
         project_namespace_id: int,
     ) -> "Project":

datachain/query/batch.py

@@ -1,8 +1,7 @@
 import contextlib
 import math
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Sequence
-from typing import Callable, Optional, Union
+from collections.abc import Callable, Generator, Sequence
 
 import sqlalchemy as sa
 
@@ -10,7 +9,7 @@ from datachain.data_storage.schema import PARTITION_COLUMN_ID
 from datachain.query.utils import get_query_column
 
 RowsOutputBatch = Sequence[Sequence]
-RowsOutput = Union[Sequence, RowsOutputBatch]
+RowsOutput = Sequence | RowsOutputBatch
 
 
 class BatchingStrategy(ABC):
@@ -23,7 +22,7 @@ class BatchingStrategy(ABC):
         self,
         execute: Callable,
         query: sa.Select,
-        id_col: Optional[sa.ColumnElement] = None,
+        id_col: sa.ColumnElement | None = None,
     ) -> Generator[RowsOutput, None, None]:
         """Apply the provided parameters to the UDF."""
 
@@ -40,7 +39,7 @@ class NoBatching(BatchingStrategy):
         self,
         execute: Callable,
         query: sa.Select,
-        id_col: Optional[sa.ColumnElement] = None,
+        id_col: sa.ColumnElement | None = None,
     ) -> Generator[Sequence, None, None]:
         ids_only = False
         if id_col is not None:
@@ -66,7 +65,7 @@ class Batch(BatchingStrategy):
         self,
         execute: Callable,
         query: sa.Select,
-        id_col: Optional[sa.ColumnElement] = None,
+        id_col: sa.ColumnElement | None = None,
     ) -> Generator[RowsOutput, None, None]:
         from datachain.data_storage.warehouse import SELECT_BATCH_SIZE
 
@@ -105,7 +104,7 @@ class Partition(BatchingStrategy):
         self,
         execute: Callable,
         query: sa.Select,
-        id_col: Optional[sa.ColumnElement] = None,
+        id_col: sa.ColumnElement | None = None,
     ) -> Generator[RowsOutput, None, None]:
         if (partition_col := get_query_column(query, PARTITION_COLUMN_ID)) is None:
             raise RuntimeError("partition column not found in query")
@@ -115,7 +114,7 @@ class Partition(BatchingStrategy):
             query = query.with_only_columns(id_col, partition_col)
             ids_only = True
 
-        current_partition: Optional[int] = None
+        current_partition: int | None = None
         batch: list = []
 
         query_fields = [str(c.name) for c in query.selected_columns]