datachain 0.6.2__tar.gz → 0.6.4__tar.gz

{datachain-0.6.2 → datachain-0.6.4}/.github/workflows/tests-studio.yml

@@ -101,6 +101,6 @@ jobs:
           pytest
           --config-file=pyproject.toml -rs
           --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
-          -m 'not benchmark'
+          --benchmark-skip
           tests ../datachain/tests
         working-directory: backend/datachain_server

{datachain-0.6.2 → datachain-0.6.4}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.0'
+    rev: 'v0.7.1'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.6.2/src/datachain.egg-info → datachain-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.2
+Version: 0.6.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -82,7 +82,7 @@ Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.12.1; extra == "dev"
+Requires-Dist: mypy==1.13.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"

{datachain-0.6.2 → datachain-0.6.4}/noxfile.py

@@ -22,8 +22,7 @@ def bench(session: nox.Session) -> None:
     session.install(".[tests]")
     session.run(
         "pytest",
-        "-m",
-        "benchmark",
+        "--benchmark-only",
         "--benchmark-group-by",
         "func",
         *session.posargs,

{datachain-0.6.2 → datachain-0.6.4}/pyproject.toml

@@ -94,7 +94,7 @@ tests = [
 ]
 dev = [
   "datachain[docs,tests]",
-  "mypy==1.12.1",
+  "mypy==1.13.0",
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
@@ -127,9 +127,8 @@ namespaces = false
 [tool.setuptools_scm]
 
 [tool.pytest.ini_options]
-addopts = "-rfEs -m 'not benchmark and not examples'"
+addopts = "-rfEs -m 'not examples' --benchmark-skip"
 markers = [
-  "benchmark: benchmarks.",
   "e2e: End-to-end tests",
   "examples: All examples",
   "computer_vision: Computer vision examples",
@@ -214,6 +213,7 @@ ignore = [
 select = [
   "B",  # flake8-bugbear
   "C4",  # flake8-comprehensions
+  "C420",  # unnecessary-dict-comprehension-for-iterable
   "C90",  # mccabe
   "W",  # pycodestyle - Warning
   "E",  # pycodestyle - Error
@@ -252,11 +252,10 @@ select = [
   "NPY",  # numpy
   "TRY004",  # type-check-without-type-error
   "TRY201",  # verbose-raise
-  "TRY302",  # useless-try-except
+  "TRY203",  # useless-try-except
   "TRY401",  # verbose-log-message
   "RUF022",  # unsorted-dunder-all
   "RUF023",  # unsorted-dunder-slots
-  "RUF025",  # unnecessary-dict-comprehension-for-iterable
   "RUF027",  # missing-f-string-syntax
   "RUF030",  # assert-with-print-message
   "RUF101",  # redirected-noqa

{datachain-0.6.2 → datachain-0.6.4}/src/datachain/catalog/catalog.py

@@ -42,6 +42,7 @@ from datachain.dataset import (
     DatasetStats,
     DatasetStatus,
     RowDict,
+    StorageURI,
     create_dataset_uri,
     parse_dataset_uri,
 )
@@ -58,7 +59,6 @@ from datachain.node import DirType, Node, NodeWithPath
 from datachain.nodes_thread_pool import NodesThreadPool
 from datachain.remote.studio import StudioClient
 from datachain.sql.types import DateTime, SQLType, String
-from datachain.storage import StorageURI
 from datachain.utils import (
     DataChainDir,
     batched,
@@ -1702,31 +1702,9 @@ class Catalog:
         *,
         client_config=None,
     ) -> None:
-        root_sources = [
-            src for src in sources if Client.get_implementation(src).is_root_url(src)
-        ]
-        non_root_sources = [
-            src
-            for src in sources
-            if not Client.get_implementation(src).is_root_url(src)
-        ]
-
-        client_config = client_config or self.client_config
-
-        # for root sources (e.g s3://) we are just getting all buckets and
-        # saving them as storages, without further indexing in each bucket
-        for source in root_sources:
-            for bucket in Client.get_implementation(source).ls_buckets(**client_config):
-                client = self.get_client(bucket.uri, **client_config)
-                print(f"Registering storage {client.uri}")
-                self.metastore.create_storage_if_not_registered(client.uri)
-
         self.enlist_sources(
-            non_root_sources,
+            sources,
             update,
-            client_config=client_config,
+            client_config=client_config or self.client_config,
             only_index=True,
         )
-
-    def find_stale_storages(self) -> None:
-        self.metastore.find_stale_storages()

{datachain-0.6.2 → datachain-0.6.4}/src/datachain/cli.py

@@ -568,12 +568,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
     add_sources_arg(parse_index)
 
-    subp.add_parser(
-        "find-stale-storages",
-        parents=[parent_parser],
-        description="Finds and marks stale storages",
-    )
-
     show_parser = subp.add_parser(
         "show",
         parents=[parent_parser],
@@ -1100,8 +1094,6 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
             )
         elif args.command == "completion":
             print(completion(args.shell))
-        elif args.command == "find-stale-storages":
-            catalog.find_stale_storages()
         elif args.command == "query":
             query(
                 catalog,

{datachain-0.6.2 → datachain-0.6.4}/src/datachain/client/fsspec.py

@@ -31,11 +31,12 @@ from datachain.error import ClientError as DataChainClientError
 from datachain.lib.file import File
 from datachain.nodes_fetcher import NodesFetcher
 from datachain.nodes_thread_pool import NodeChunk
-from datachain.storage import StorageURI
 
 if TYPE_CHECKING:
     from fsspec.spec import AbstractFileSystem
 
+    from datachain.dataset import StorageURI
+
 
 logger = logging.getLogger("datachain")
 
@@ -63,7 +64,7 @@ def _is_win_local_path(uri: str) -> bool:
 
 class Bucket(NamedTuple):
     name: str
-    uri: StorageURI
+    uri: "StorageURI"
     created: Optional[datetime]
 
 
@@ -115,7 +116,7 @@ class Client(ABC):
         return DATA_SOURCE_URI_PATTERN.match(name) is not None
 
     @staticmethod
-    def parse_url(source: str) -> tuple[StorageURI, str]:
+    def parse_url(source: str) -> tuple["StorageURI", str]:
         cls = Client.get_implementation(source)
         storage_name, rel_path = cls.split_url(source)
         return cls.get_uri(storage_name), rel_path
@@ -148,7 +149,7 @@ class Client(ABC):
     @classmethod
     def from_source(
         cls,
-        uri: StorageURI,
+        uri: "StorageURI",
         cache: DataChainCache,
         **kwargs,
     ) -> "Client":
@@ -156,6 +157,8 @@ class Client(ABC):
 
     @classmethod
     def ls_buckets(cls, **kwargs) -> Iterator[Bucket]:
+        from datachain.dataset import StorageURI
+
         for entry in cls.create_fs(**kwargs).ls(cls.PREFIX, detail=True):
             name = entry["name"].rstrip("/")
             yield Bucket(
@@ -169,7 +172,9 @@ class Client(ABC):
         return url == cls.PREFIX
 
     @classmethod
-    def get_uri(cls, name) -> StorageURI:
+    def get_uri(cls, name) -> "StorageURI":
+        from datachain.dataset import StorageURI
+
         return StorageURI(f"{cls.PREFIX}{name}")
 
     @classmethod

{datachain-0.6.2 → datachain-0.6.4}/src/datachain/client/local.py

@@ -2,16 +2,18 @@ import os
 import posixpath
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 from fsspec.implementations.local import LocalFileSystem
 
 from datachain.lib.file import File
-from datachain.storage import StorageURI
 
 from .fsspec import Client
 
+if TYPE_CHECKING:
+    from datachain.dataset import StorageURI
+
 
 class FileClient(Client):
     FS_CLASS = LocalFileSystem
@@ -28,7 +30,9 @@ class FileClient(Client):
         raise TypeError("Signed urls are not implemented for local file system")
 
     @classmethod
-    def get_uri(cls, name) -> StorageURI:
+    def get_uri(cls, name) -> "StorageURI":
+        from datachain.dataset import StorageURI
+
         return StorageURI(f'{cls.PREFIX}/{name.removeprefix("/")}')
 
     @classmethod