datachain 0.8.6__tar.gz → 0.8.8__tar.gz

{datachain-0.8.6 → datachain-0.8.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.6
+Version: 0.8.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/cache.py

@@ -9,8 +9,6 @@ from dvc_objects.fs.local import LocalFileSystem
 from dvc_objects.fs.utils import remove
 from fsspec.callbacks import Callback, TqdmCallback
 
-from .progress import Tqdm
-
 if TYPE_CHECKING:
     from datachain.client import Client
     from datachain.lib.file import File
@@ -86,9 +84,11 @@ class DataChainCache:
         size = file.size
         if size < 0:
             size = await client.get_size(from_path, version_id=file.version)
+        from tqdm.auto import tqdm
+
         cb = callback or TqdmCallback(
             tqdm_kwargs={"desc": odb_fs.name(from_path), "bytes": True, "leave": False},
-            tqdm_cls=Tqdm,
+            tqdm_cls=tqdm,
             size=size,
         )
         try:

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/catalog/catalog.py

@@ -28,7 +28,7 @@ import requests
 import sqlalchemy as sa
 import yaml
 from sqlalchemy import Column
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.cache import DataChainCache
 from datachain.client import Client

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/client/azure.py

@@ -2,7 +2,7 @@ from typing import Any, Optional
 from urllib.parse import parse_qs, urlsplit, urlunsplit
 
 from adlfs import AzureBlobFileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/client/fsspec.py

@@ -23,7 +23,7 @@ from botocore.exceptions import ClientError
 from dvc_objects.fs.system import reflink
 from fsspec.asyn import get_loop, sync
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.cache import DataChainCache
 from datachain.client.fileslice import FileWrapper

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/client/gcs.py

@@ -7,7 +7,7 @@ from typing import Any, Optional, cast
 
 from dateutil.parser import isoparse
 from gcsfs import GCSFileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/client/s3.py

@@ -5,7 +5,7 @@ from urllib.parse import parse_qs, urlsplit, urlunsplit
 
 from botocore.exceptions import NoCredentialsError
 from s3fs import S3FileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 
@@ -32,17 +32,20 @@ class ClientS3(Client):
         if "aws_token" in kwargs:
             kwargs.setdefault("token", kwargs.pop("aws_token"))
 
+        # We want to use newer v4 signature version since regions added after
+        # 2014 are not going to support v2 which is the older one.
+        # All regions support v4.
+        kwargs.setdefault("config_kwargs", {}).setdefault("signature_version", "s3v4")
+
+        if "region_name" in kwargs:
+            kwargs["config_kwargs"].setdefault("region_name", kwargs.pop("region_name"))
+
         # remove this `if` when https://github.com/fsspec/s3fs/pull/929 lands
         if not os.environ.get("AWS_REGION") and not os.environ.get("AWS_ENDPOINT_URL"):
             # caching bucket regions to use the right one in signed urls, otherwise
             # it tries to randomly guess and creates wrong signature
             kwargs.setdefault("cache_regions", True)
 
-        # We want to use newer v4 signature version since regions added after
-        # 2014 are not going to support v2 which is the older one.
-        # All regions support v4.
-        kwargs.setdefault("config_kwargs", {}).setdefault("signature_version", "s3v4")
-
         if not kwargs.get("anon"):
             try:
                 # Run an inexpensive check to see if credentials are available

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/data_storage/sqlite.py

@@ -21,7 +21,7 @@ from sqlalchemy.schema import CreateIndex, CreateTable, DropTable
 from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import bindparam, cast
 from sqlalchemy.sql.selectable import Select
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 import datachain.sql.sqlite
 from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/data_storage/warehouse.py

@@ -14,7 +14,7 @@ import sqlalchemy as sa
 from sqlalchemy import Table, case, select
 from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import true
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.client import Client
 from datachain.data_storage.schema import convert_rows_custom_column_types

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/lib/arrow.py

@@ -7,7 +7,7 @@ import orjson
 import pyarrow as pa
 from fsspec.core import split_protocol
 from pyarrow.dataset import CsvFileFormat, dataset
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.data_model import dict_to_data_model
 from datachain.lib.file import ArrowRow, File

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/lib/dc.py

@@ -1330,19 +1330,27 @@ class DataChain:
 
         Parameters:
             right_ds: Chain to join with.
-            on: Predicate or list of Predicates to join on. If both chains have the
-                same predicates then this predicate is enough for the join. Otherwise,
-                `right_on` parameter has to specify the predicates for the other chain.
-            right_on: Optional predicate or list of Predicates
-                    for the `right_ds` to join.
+            on: Predicate ("column.name", C("column.name"), or Func) or list of
+                Predicates to join on. If both chains have the same predicates then
+                this predicate is enough for the join. Otherwise, `right_on` parameter
+                has to specify the predicates for the other chain.
+            right_on: Optional predicate or list of Predicates for the `right_ds`
+                to join.
             inner (bool): Whether to run inner join or outer join.
-            rname (str): name prefix for conflicting signal names.
+            rname (str): Name prefix for conflicting signal names.
 
-        Example:
+        Examples:
             ```py
             meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                                   right_on=(C.name, C.pq__index))
             ```
+
+            ```py
+            imgs.merge(captions,
+                       on=func.path.file_stem(imgs.c("file.path")),
+                       right_on=func.path.file_stem(captions.c("file.path"))
+            ```
+        )
         """
         if on is None:
             raise DatasetMergeError(["None"], None, "'on' must be specified")

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/lib/hf.py

@@ -29,7 +29,7 @@ from io import BytesIO
 from typing import TYPE_CHECKING, Any, Union
 
 import PIL
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.arrow import arrow_type_mapper
 from datachain.lib.data_model import DataModel, DataType, dict_to_data_model

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/listing.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Optional
 
 from sqlalchemy import Column
 from sqlalchemy.sql import func
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.node import DirType, Node, NodeWithPath
 from datachain.sql.functions import path as pathfunc

datachain-0.8.8/src/datachain/progress.py

@@ -0,0 +1,36 @@
+"""Manages progress bars."""
+
+import logging
+from threading import RLock
+
+from fsspec import Callback
+from fsspec.callbacks import TqdmCallback
+from tqdm.auto import tqdm
+
+logger = logging.getLogger(__name__)
+tqdm.set_lock(RLock())
+
+
+class CombinedDownloadCallback(Callback):
+    def set_size(self, size):
+        # This is a no-op to prevent fsspec's .get_file() from setting the combined
+        # download size to the size of the current file.
+        pass
+
+    def increment_file_count(self, n: int = 1) -> None:
+        pass
+
+
+class TqdmCombinedDownloadCallback(CombinedDownloadCallback, TqdmCallback):
+    def __init__(self, tqdm_kwargs=None, *args, **kwargs):
+        self.files_count = 0
+        tqdm_kwargs = tqdm_kwargs or {}
+        tqdm_kwargs.setdefault("postfix", {}).setdefault("files", self.files_count)
+        kwargs = kwargs or {}
+        kwargs["tqdm_cls"] = tqdm
+        super().__init__(tqdm_kwargs, *args, **kwargs)
+
+    def increment_file_count(self, n: int = 1) -> None:
+        self.files_count += n
+        if self.tqdm is not None:
+            self.tqdm.postfix = f"{self.files_count} files"

{datachain-0.8.6 → datachain-0.8.8}/src/datachain/query/dataset.py

@@ -33,6 +33,7 @@ from sqlalchemy.sql.elements import ColumnClause, ColumnElement
 from sqlalchemy.sql.expression import label
 from sqlalchemy.sql.schema import TableClause
 from sqlalchemy.sql.selectable import Select
+from tqdm.auto import tqdm
 
 from datachain.asyn import ASYNC_WORKERS, AsyncMapper, OrderedMapper
 from datachain.catalog.catalog import clone_catalog_with_cache
@@ -366,12 +367,16 @@ def get_download_callback(suffix: str = "", **kwargs) -> CombinedDownloadCallbac
 
 
 def get_processed_callback() -> Callback:
-    return TqdmCallback({"desc": "Processed", "unit": " rows", "leave": False})
+    return TqdmCallback(
+        {"desc": "Processed", "unit": " rows", "leave": False}, tqdm_cls=tqdm
+    )
 
 
 def get_generated_callback(is_generator: bool = False) -> Callback:
     if is_generator:
-        return TqdmCallback({"desc": "Generated", "unit": " rows", "leave": False})
+        return TqdmCallback(
+            {"desc": "Generated", "unit": " rows", "leave": False}, tqdm_cls=tqdm
+        )
     return DEFAULT_CALLBACK
 
 

{datachain-0.8.6 → datachain-0.8.8}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.6
+Version: 0.8.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

datachain-0.8.6/src/datachain/progress.py

@@ -1,156 +0,0 @@
-"""Manages progress bars."""
-
-import logging
-import sys
-from threading import RLock
-from typing import Any, ClassVar
-
-from fsspec import Callback
-from fsspec.callbacks import TqdmCallback
-from tqdm import tqdm
-
-from datachain.utils import env2bool
-
-logger = logging.getLogger(__name__)
-tqdm.set_lock(RLock())
-
-
-class Tqdm(tqdm):
-    """
-    maximum-compatibility tqdm-based progressbars
-    """
-
-    BAR_FMT_DEFAULT = (
-        "{percentage:3.0f}% {desc}|{bar}|"
-        "{postfix[info]}{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
-    )
-    # nested bars should have fixed bar widths to align nicely
-    BAR_FMT_DEFAULT_NESTED = (
-        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
-        "{postfix[info]}{n_fmt}/{total_fmt}"
-        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
-    )
-    BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
-    BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
-        "unit": "B",
-        "unit_scale": True,
-        "unit_divisor": 1024,
-        "miniters": 1,
-    }
-
-    def __init__(
-        self,
-        iterable=None,
-        disable=None,
-        level=logging.ERROR,
-        desc=None,
-        leave=False,
-        bar_format=None,
-        bytes=False,
-        file=None,
-        total=None,
-        postfix=None,
-        **kwargs,
-    ):
-        """
-        bytes   : shortcut for
-            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
-        desc  : persists after `close()`
-        level  : effective logging level for determining `disable`;
-            used only if `disable` is unspecified
-        disable  : If (default: None) or False,
-            will be determined by logging level.
-            May be overridden to `True` due to non-TTY status.
-            Skip override by specifying env var `DATACHAIN_IGNORE_ISATTY`.
-        kwargs  : anything accepted by `tqdm.tqdm()`
-        """
-        kwargs = kwargs.copy()
-        if bytes:
-            kwargs = self.BYTES_DEFAULTS | kwargs
-        else:
-            kwargs.setdefault("unit_scale", total > 999 if total else True)
-        if file is None:
-            file = sys.stderr
-        # auto-disable based on `logger.level`
-        if not disable:
-            disable = logger.getEffectiveLevel() > level
-        # auto-disable based on TTY
-        if (
-            not disable
-            and not env2bool("DATACHAIN_IGNORE_ISATTY")
-            and hasattr(file, "isatty")
-        ):
-            disable = not file.isatty()
-        super().__init__(
-            iterable=iterable,
-            disable=disable,
-            leave=leave,
-            desc=desc,
-            bar_format="!",
-            lock_args=(False,),
-            total=total,
-            **kwargs,
-        )
-        self.postfix = postfix or {"info": ""}
-        if bar_format is None:
-            if self.__len__():
-                self.bar_format = (
-                    self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
-                )
-            else:
-                self.bar_format = self.BAR_FMT_NOTOTAL
-        else:
-            self.bar_format = bar_format
-        self.refresh()
-
-    def close(self):
-        self.postfix["info"] = ""
-        # remove ETA (either unknown or zero); remove completed bar
-        self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
-            "|{bar:10}|", " "
-        )
-        super().close()
-
-    @property
-    def format_dict(self):
-        """inject `ncols_desc` to fill the display width (`ncols`)"""
-        d = super().format_dict
-        ncols = d["ncols"] or 80
-        # assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
-
-        meter = self.format_meter(  # type: ignore[call-arg]
-            ncols_desc=1, ncols_info=1, **d
-        )
-        ncols_left = ncols - len(meter) + 1
-        ncols_left = max(ncols_left, 0)
-        if ncols_left:
-            d["ncols_desc"] = d["ncols_info"] = ncols_left
-        else:
-            # work-around for zero-width description
-            d["ncols_desc"] = d["ncols_info"] = 1
-            d["prefix"] = ""
-        return d
-
-
-class CombinedDownloadCallback(Callback):
-    def set_size(self, size):
-        # This is a no-op to prevent fsspec's .get_file() from setting the combined
-        # download size to the size of the current file.
-        pass
-
-    def increment_file_count(self, n: int = 1) -> None:
-        pass
-
-
-class TqdmCombinedDownloadCallback(CombinedDownloadCallback, TqdmCallback):
-    def __init__(self, tqdm_kwargs=None, *args, **kwargs):
-        self.files_count = 0
-        tqdm_kwargs = tqdm_kwargs or {}
-        tqdm_kwargs.setdefault("postfix", {}).setdefault("files", self.files_count)
-        super().__init__(tqdm_kwargs, *args, **kwargs)
-
-    def increment_file_count(self, n: int = 1) -> None:
-        self.files_count += n
-        if self.tqdm is not None:
-            self.tqdm.postfix = f"{self.files_count} files"