datachain 0.11.11__py3-none-any.whl → 0.12.0__py3-none-any.whl

datachain/model/pose.py

@@ -1,3 +1,5 @@
+from collections.abc import Sequence
+
 from pydantic import Field
 
 from datachain.lib.data_model import DataModel
@@ -19,27 +21,30 @@ class Pose(DataModel):
     y: list[int] = Field(default=[])
 
     @staticmethod
-    def from_list(points: list[list[float]]) -> "Pose":
-        assert len(points) == 2, "Pose must be a list of 2 lists: x and y coordinates."
+    def from_list(points: Sequence[Sequence[float]]) -> "Pose":
+        if not isinstance(points, (list, tuple)):
+            raise TypeError("Pose must be a list of coordinates.")
+        if len(points) != 2:
+            raise ValueError("Pose must be a list of 2 lists: x and y coordinates.")
         points_x, points_y = points
-        assert len(points_x) == len(points_y) == 17, (
-            "Pose x and y coordinates must have the same length of 17."
-        )
-        assert all(
-            isinstance(value, (int, float)) for value in [*points_x, *points_y]
-        ), "Pose coordinates must be floats or integers."
-        return Pose(
-            x=[round(coord) for coord in points_x],
-            y=[round(coord) for coord in points_y],
-        )
+        if not isinstance(points_x, (list, tuple)) or not isinstance(
+            points_y, (list, tuple)
+        ):
+            raise TypeError("Pose x and y coordinates must be lists.")
+        if len(points_x) != len(points_y) != 17:
+            raise ValueError(
+                "Pose x and y coordinates must have the same length of 17."
+            )
+        if not all(isinstance(value, (int, float)) for value in [*points_x, *points_y]):
+            raise ValueError("Pose coordinates must be floats or integers.")
+        return Pose(x=list(map(round, points_x)), y=list(map(round, points_y)))
 
     @staticmethod
-    def from_dict(points: dict[str, list[float]]) -> "Pose":
-        assert isinstance(points, dict) and set(points) == {
-            "x",
-            "y",
-        }, "Pose must be a dict with keys 'x' and 'y'."
-        return Pose.from_list([points["x"], points["y"]])
+    def from_dict(points: dict[str, Sequence[float]]) -> "Pose":
+        keys = ("x", "y")
+        if not isinstance(points, dict) or set(points) != set(keys):
+            raise ValueError("Pose must be a dictionary with coordinates.")
+        return Pose.from_list([points[k] for k in keys])
 
 
 class Pose3D(DataModel):
@@ -60,29 +65,36 @@ class Pose3D(DataModel):
     visible: list[float] = Field(default=[])
 
     @staticmethod
-    def from_list(points: list[list[float]]) -> "Pose3D":
-        assert len(points) == 3, (
-            "Pose3D must be a list of 3 lists: x, y coordinates and visible."
-        )
+    def from_list(points: Sequence[Sequence[float]]) -> "Pose3D":
+        if not isinstance(points, (list, tuple)):
+            raise TypeError("Pose3D must be a list of coordinates.")
+        if len(points) != 3:
+            raise ValueError(
+                "Pose3D must be a list of 3 lists: x, y coordinates and visible."
+            )
         points_x, points_y, points_v = points
-        assert len(points_x) == len(points_y) == len(points_v) == 17, (
-            "Pose3D x, y coordinates and visible must have the same length of 17."
-        )
-        assert all(
+        if (
+            not isinstance(points_x, (list, tuple))
+            or not isinstance(points_y, (list, tuple))
+            or not isinstance(points_v, (list, tuple))
+        ):
+            raise TypeError("Pose3D x, y and visible must be lists.")
+        if len(points_x) != len(points_y) != len(points_v) != 17:
+            raise ValueError("Pose3D x, y and visible must have the same length of 17.")
+        if not all(
             isinstance(value, (int, float))
             for value in [*points_x, *points_y, *points_v]
-        ), "Pose3D coordinates must be floats or integers."
+        ):
+            raise ValueError("Pose3D coordinates must be floats or integers.")
         return Pose3D(
-            x=[round(coord) for coord in points_x],
-            y=[round(coord) for coord in points_y],
-            visible=points_v,
+            x=list(map(round, points_x)),
+            y=list(map(round, points_y)),
+            visible=list(points_v),
         )
 
     @staticmethod
-    def from_dict(points: dict[str, list[float]]) -> "Pose3D":
-        assert isinstance(points, dict) and set(points) == {
-            "x",
-            "y",
-            "visible",
-        }, "Pose3D must be a dict with keys 'x', 'y' and 'visible'."
-        return Pose3D.from_list([points["x"], points["y"], points["visible"]])
+    def from_dict(points: dict[str, Sequence[float]]) -> "Pose3D":
+        keys = ("x", "y", "visible")
+        if not isinstance(points, dict) or set(points) != set(keys):
+            raise ValueError("Pose3D must be a dictionary with coordinates.")
+        return Pose3D.from_list([points[k] for k in keys])

datachain/model/segment.py

@@ -1,3 +1,5 @@
+from collections.abc import Sequence
+
 from pydantic import Field
 
 from datachain.lib.data_model import DataModel
@@ -21,27 +23,29 @@ class Segment(DataModel):
     y: list[int] = Field(default=[])
 
     @staticmethod
-    def from_list(points: list[list[float]], title: str = "") -> "Segment":
-        assert len(points) == 2, (
-            "Segment must be a list of 2 lists: x and y coordinates."
-        )
+    def from_list(points: Sequence[Sequence[float]], title: str = "") -> "Segment":
+        if not isinstance(points, (list, tuple)):
+            raise TypeError("Segment must be a list of coordinates.")
+        if len(points) != 2:
+            raise ValueError("Segment must be a list of 2 lists: x and y coordinates.")
         points_x, points_y = points
-        assert len(points_x) == len(points_y), (
-            "Segment x and y coordinates must have the same length."
-        )
-        assert all(
-            isinstance(value, (int, float)) for value in [*points_x, *points_y]
-        ), "Segment coordinates must be floats or integers."
+        if not isinstance(points_x, (list, tuple)) or not isinstance(
+            points_y, (list, tuple)
+        ):
+            raise TypeError("Segment x and y coordinates must be lists.")
+        if len(points_x) != len(points_y):
+            raise ValueError("Segment x and y coordinates must have the same length.")
+        if not all(isinstance(value, (int, float)) for value in [*points_x, *points_y]):
+            raise ValueError("Segment coordinates must be floats or integers.")
         return Segment(
             title=title,
-            x=[round(coord) for coord in points_x],
-            y=[round(coord) for coord in points_y],
+            x=list(map(round, points_x)),
+            y=list(map(round, points_y)),
         )
 
     @staticmethod
-    def from_dict(points: dict[str, list[float]], title: str = "") -> "Segment":
-        assert isinstance(points, dict) and set(points) == {
-            "x",
-            "y",
-        }, "Segment must be a dict with keys 'x' and 'y'."
-        return Segment.from_list([points["x"], points["y"]], title=title)
+    def from_dict(points: dict[str, Sequence[float]], title: str = "") -> "Segment":
+        keys = ("x", "y")
+        if not isinstance(points, dict) or set(points) != set(keys):
+            raise ValueError("Segment must be a dictionary with coordinates.")
+        return Segment.from_list([points[k] for k in keys], title=title)

datachain/model/ultralytics/bbox.py

@@ -23,7 +23,7 @@ class YoloBBox(DataModel):
     cls: int = Field(default=-1)
     name: str = Field(default="")
     confidence: float = Field(default=0)
-    box: BBox
+    box: BBox = Field(default=BBox())
 
     @staticmethod
     def from_result(result: "Results") -> "YoloBBox":
@@ -55,10 +55,10 @@ class YoloBBoxes(DataModel):
         box: A list of bounding boxes of the detected objects
     """
 
-    cls: list[int]
-    name: list[str]
-    confidence: list[float]
-    box: list[BBox]
+    cls: list[int] = Field(default=[])
+    name: list[str] = Field(default=[])
+    confidence: list[float] = Field(default=[])
+    box: list[BBox] = Field(default=[])
 
     @staticmethod
     def from_results(results: list["Results"]) -> "YoloBBoxes":
@@ -124,10 +124,10 @@ class YoloOBBoxes(DataModel):
         box: A list of oriented bounding boxes of the detected objects.
     """
 
-    cls: list[int]
-    name: list[str]
-    confidence: list[float]
-    box: list[OBBox]
+    cls: list[int] = Field(default=[])
+    name: list[str] = Field(default=[])
+    confidence: list[float] = Field(default=[])
+    box: list[OBBox] = Field(default=[])
 
     @staticmethod
     def from_results(results: list["Results"]) -> "YoloOBBoxes":

datachain/model/ultralytics/pose.py

@@ -47,8 +47,8 @@ class YoloPose(DataModel):
     cls: int = Field(default=-1)
     name: str = Field(default="")
     confidence: float = Field(default=0)
-    box: BBox
-    pose: Pose3D
+    box: BBox = Field(default=BBox())
+    pose: Pose3D = Field(default=Pose3D())
 
     @staticmethod
     def from_result(result: "Results") -> "YoloPose":
@@ -87,11 +87,11 @@ class YoloPoses(DataModel):
         pose: The 3D pose keypoints of the poses.
     """
 
-    cls: list[int]
-    name: list[str]
-    confidence: list[float]
-    box: list[BBox]
-    pose: list[Pose3D]
+    cls: list[int] = Field(default=[])
+    name: list[str] = Field(default=[])
+    confidence: list[float] = Field(default=[])
+    box: list[BBox] = Field(default=[])
+    pose: list[Pose3D] = Field(default=[])
 
     @staticmethod
     def from_results(results: list["Results"]) -> "YoloPoses":

datachain/model/ultralytics/segment.py

@@ -25,8 +25,8 @@ class YoloSegment(DataModel):
     cls: int = Field(default=-1)
     name: str = Field(default="")
     confidence: float = Field(default=0)
-    box: BBox
-    segment: Segment
+    box: BBox = Field(default=BBox())
+    segment: Segment = Field(default=Segment())
 
     @staticmethod
     def from_result(result: "Results") -> "YoloSegment":
@@ -65,11 +65,11 @@ class YoloSegments(DataModel):
         segment (list[Segments]): The segments of the segments.
     """
 
-    cls: list[int]
-    name: list[str]
-    confidence: list[float]
-    box: list[BBox]
-    segment: list[Segment]
+    cls: list[int] = Field(default=[])
+    name: list[str] = Field(default=[])
+    confidence: list[float] = Field(default=[])
+    box: list[BBox] = Field(default=[])
+    segment: list[Segment] = Field(default=[])
 
     @staticmethod
     def from_results(results: list["Results"]) -> "YoloSegments":

datachain/model/utils.py

@@ -0,0 +1,191 @@
+from collections.abc import Sequence
+from typing import Literal
+
+BBoxType = Literal["albumentations", "coco", "voc", "yolo"]
+
+
+def validate_bbox(coords: Sequence[float], *types: type) -> None:
+    """Validate the bounding box coordinates."""
+    if not isinstance(coords, (list, tuple)):
+        raise TypeError(
+            f"Invalid bounding box coordinates: {coords}, should be a list or tuple"
+        )
+    if len(coords) != 4:
+        raise ValueError(
+            f"Invalid bounding box coordinates: {coords}, should have 4 values"
+        )
+    if any(not isinstance(c, types) for c in coords):
+        raise ValueError(
+            f"Invalid bounding box coordinates: {coords}, should be {types}"
+        )
+
+
+def convert_bbox(
+    coords: Sequence[float],
+    img_size: Sequence[int],
+    source: BBoxType,
+    target: BBoxType,
+) -> list[float]:
+    """
+    Convert the bounding box coordinates between different formats.
+
+    Supported formats: "albumentations", "coco", "voc", "yolo".
+
+    Albumentations format represents bounding boxes as [x_min, y_min, x_max, y_max],
+    where:
+        - (x_min, y_min) are the normalized coordinates of the top-left corner.
+        - (x_max, y_max) are the normalized coordinates of the bottom-right corner.
+
+    COCO format represents bounding boxes as [x_min, y_min, width, height], where:
+        - (x_min, y_min) are the pixel coordinates of the top-left corner.
+        - width and height define the size of the bounding box in pixels.
+
+    PASCAL VOC format represents bounding boxes as [x_min, y_min, x_max, y_max], where:
+        - (x_min, y_min) are the pixel coordinates of the top-left corner.
+        - (x_max, y_max) are the pixel coordinates of the bottom-right corner.
+
+    YOLO format represents bounding boxes as [x_center, y_center, width, height], where:
+        - (x_center, y_center) are the normalized coordinates of the box center.
+        - width and height normalized values define the size of the bounding box.
+
+    Normalized coordinates are floats between 0 and 1, representing the
+    relative position of the pixels in the image.
+
+    Args:
+        coords (Sequence[float]): The bounding box coordinates to convert.
+        img_size (Sequence[int]): The reference image size (width, height).
+        source (str): The source bounding box format.
+        target (str): The target bounding box format.
+
+    Returns:
+        list[float]: The bounding box coordinates in the target format.
+    """
+    if source == "albumentations":
+        return [
+            round(c, 4) for c in convert_albumentations_bbox(coords, img_size, target)
+        ]
+    if source == "coco":
+        return [round(c, 4) for c in convert_coco_bbox(coords, img_size, target)]
+    if source == "voc":
+        return [round(c, 4) for c in convert_voc_bbox(coords, img_size, target)]
+    if source == "yolo":
+        return [round(c, 4) for c in convert_yolo_bbox(coords, img_size, target)]
+    raise ValueError(f"Unsupported source format: {source}")
+
+
+def convert_albumentations_bbox(
+    coords: Sequence[float],
+    img_size: Sequence[int],
+    target: BBoxType,
+) -> list[float]:
+    """Convert the Albumentations bounding box coordinates to other formats."""
+    if target == "albumentations":
+        return list(coords)
+    if target == "coco":
+        return [
+            coords[0] * img_size[0],
+            coords[1] * img_size[1],
+            (coords[2] - coords[0]) * img_size[0],
+            (coords[3] - coords[1]) * img_size[1],
+        ]
+    if target == "voc":
+        return [coords[i] * img_size[i % 2] for i in range(4)]
+    if target == "yolo":
+        return [
+            (coords[0] + coords[2]) / 2,
+            (coords[1] + coords[3]) / 2,
+            coords[2] - coords[0],
+            coords[3] - coords[1],
+        ]
+    raise ValueError(f"Unsupported target format: {target}")
+
+
+def convert_coco_bbox(
+    coords: Sequence[float],
+    img_size: Sequence[int],
+    target: BBoxType,
+) -> list[float]:
+    """Convert the COCO bounding box coordinates to other formats."""
+    if target == "albumentations":
+        return [
+            coords[0] / img_size[0],
+            coords[1] / img_size[1],
+            (coords[0] + coords[2]) / img_size[0],
+            (coords[1] + coords[3]) / img_size[1],
+        ]
+    if target == "coco":
+        return list(coords)
+    if target == "voc":
+        return [coords[0], coords[1], coords[0] + coords[2], coords[1] + coords[3]]
+    if target == "yolo":
+        return [
+            (coords[0] + coords[2] / 2) / img_size[0],
+            (coords[1] + coords[3] / 2) / img_size[1],
+            coords[2] / img_size[0],
+            coords[3] / img_size[1],
+        ]
+    raise ValueError(f"Unsupported target format: {target}")
+
+
+def convert_voc_bbox(
+    coords: Sequence[float],
+    img_size: Sequence[int],
+    target: BBoxType,
+) -> list[float]:
+    """Convert the PASCAL VOC bounding box coordinates to other formats."""
+    if target == "albumentations":
+        return [
+            coords[0] / img_size[0],
+            coords[1] / img_size[1],
+            coords[2] / img_size[0],
+            coords[3] / img_size[1],
+        ]
+    if target == "coco":
+        return [
+            coords[0],
+            coords[1],
+            coords[2] - coords[0],
+            coords[3] - coords[1],
+        ]
+    if target == "voc":
+        return list(coords)
+    if target == "yolo":
+        return [
+            (coords[0] + coords[2]) / 2 / img_size[0],
+            (coords[1] + coords[3]) / 2 / img_size[1],
+            (coords[2] - coords[0]) / img_size[0],
+            (coords[3] - coords[1]) / img_size[1],
+        ]
+    raise ValueError(f"Unsupported target format: {target}")
+
+
+def convert_yolo_bbox(
+    coords: Sequence[float],
+    img_size: Sequence[int],
+    target: BBoxType,
+) -> list[float]:
+    """Convert the YOLO bounding box coordinates to other formats."""
+    if target == "albumentations":
+        return [
+            coords[0] - coords[2] / 2,
+            coords[1] - coords[3] / 2,
+            coords[0] + coords[2] / 2,
+            coords[1] + coords[3] / 2,
+        ]
+    if target == "coco":
+        return [
+            (coords[0] - coords[2] / 2) * img_size[0],
+            (coords[1] - coords[3] / 2) * img_size[1],
+            coords[2] * img_size[0],
+            coords[3] * img_size[1],
+        ]
+    if target == "voc":
+        return [
+            (coords[0] - coords[2] / 2) * img_size[0],
+            (coords[1] - coords[3] / 2) * img_size[1],
+            (coords[0] + coords[2] / 2) * img_size[0],
+            (coords[1] + coords[3] / 2) * img_size[1],
+        ]
+    if target == "yolo":
+        return list(coords)
+    raise ValueError(f"Unsupported target format: {target}")

datachain/query/dataset.py

@@ -22,7 +22,6 @@ from typing import (
 )
 
 import attrs
-import psutil
 import sqlalchemy
 import sqlalchemy as sa
 from attrs import frozen
@@ -52,7 +51,6 @@ from datachain.lib.udf import UDFAdapter, _get_cache
 from datachain.progress import CombinedDownloadCallback, TqdmCombinedDownloadCallback
 from datachain.query.schema import C, UDFParamSpec, normalize_param
 from datachain.query.session import Session
-from datachain.remote.studio import is_token_set
 from datachain.sql.functions.random import rand
 from datachain.utils import (
     batched,
@@ -333,6 +331,8 @@ def process_udf_outputs(
     batch_size: int = INSERT_BATCH_SIZE,
     cb: Callback = DEFAULT_CALLBACK,
 ) -> None:
+    import psutil
+
     rows: list[UDFResult] = []
     # Optimization: Compute row types once, rather than for every row.
     udf_col_types = get_udf_col_types(warehouse, udf)
@@ -1087,6 +1087,8 @@ class DatasetQuery:
         in_memory: bool = False,
         fallback_to_studio: bool = True,
     ) -> None:
+        from datachain.remote.studio import is_token_set
+
         self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
         self.steps: list[Step] = []

datachain/studio.py

@@ -57,7 +57,7 @@ def process_auth_cli_args(args: "Namespace"):
     if args.cmd == "login":
         return login(args)
     if args.cmd == "logout":
-        return logout()
+        return logout(args.local)
     if args.cmd == "token":
         return token()
 
@@ -110,13 +110,15 @@ def login(args: "Namespace"):
     except StudioAuthError as exc:
         raise DataChainError(f"Failed to authenticate with Studio: {exc}") from exc
 
-    config_path = save_config(hostname, access_token)
+    level = ConfigLevel.LOCAL if args.local else ConfigLevel.GLOBAL
+    config_path = save_config(hostname, access_token, level=level)
     print(f"Authentication complete. Saved token to {config_path}.")
     return 0
 
 
-def logout():
-    with Config(ConfigLevel.GLOBAL).edit() as conf:
+def logout(local: bool = False):
+    level = ConfigLevel.LOCAL if local else ConfigLevel.GLOBAL
+    with Config(level).edit() as conf:
         token = conf.get("studio", {}).get("token")
         if not token:
             raise DataChainError(
@@ -209,8 +211,8 @@ def remove_studio_dataset(
     print(f"Dataset '{name}' removed from Studio")
 
 
-def save_config(hostname, token):
-    config = Config(ConfigLevel.GLOBAL)
+def save_config(hostname, token, level=ConfigLevel.GLOBAL):
+    config = Config(level)
     with config.edit() as conf:
         studio_conf = conf.get("studio", {})
         studio_conf["url"] = hostname

datachain/utils.py

@@ -6,7 +6,6 @@ import os
 import os.path as osp
 import random
 import re
-import stat
 import sys
 import time
 from collections.abc import Iterable, Iterator, Sequence
@@ -193,14 +192,6 @@ def suffix_to_number(num_str: str) -> int:
         raise ValueError(f"Invalid number/suffix for: {num_str}") from None
 
 
-def force_create_dir(name):
-    if not os.path.exists(name):
-        os.mkdir(name)
-    elif not os.path.isdir(name):
-        os.remove(name)
-        os.mkdir(name)
-
-
 def datachain_paths_join(source_path: str, file_paths: Iterable[str]) -> Iterable[str]:
     source_parts = source_path.rstrip("/").split("/")
     if glob.has_magic(source_parts[-1]):
@@ -210,13 +201,6 @@ def datachain_paths_join(source_path: str, file_paths: Iterable[str]) -> Iterabl
     return (f"{source_stripped}/{path.lstrip('/')}" for path in file_paths)
 
 
-# From: https://docs.python.org/3/library/shutil.html#rmtree-example
-def remove_readonly(func, path, _):
-    "Clear the readonly bit and reattempt the removal"
-    os.chmod(path, stat.S_IWRITE)
-    func(path)
-
-
 def sql_escape_like(search: str, escape: str = "\\") -> str:
     return (
         search.replace(escape, escape * 2)

{datachain-0.11.11.dist-info → datachain-0.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.11.11
+Version: 0.12.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -71,6 +71,7 @@ Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
 Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"
 Requires-Dist: imageio[ffmpeg,pyav]>=2.37.0; extra == "video"
@@ -90,6 +91,7 @@ Requires-Dist: hypothesis; extra == "tests"
 Requires-Dist: aiotools>=1.7.0; extra == "tests"
 Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
+Requires-Dist: ultralytics; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
 Requires-Dist: mypy==1.15.0; extra == "dev"
@@ -103,7 +105,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.82; extra == "examples"
+Requires-Dist: ultralytics; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================