datachain 0.14.2__py3-none-any.whl → 0.39.0__py3-none-any.whl

datachain/__init__.py

@@ -5,8 +5,13 @@ from datachain.lib.dc import (
     DataChain,
     Sys,
     datasets,
+    delete_dataset,
+    is_local,
+    is_studio,
     listings,
+    move_dataset,
     read_csv,
+    read_database,
     read_dataset,
     read_hf,
     read_json,
@@ -18,6 +23,9 @@ from datachain.lib.dc import (
 )
 from datachain.lib.file import (
     ArrowRow,
+    Audio,
+    AudioFile,
+    AudioFragment,
     File,
     FileError,
     Image,
@@ -30,6 +38,8 @@ from datachain.lib.file import (
     VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
+from datachain.lib.namespaces import delete_namespace
+from datachain.lib.projects import create as create_project
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
 from datachain.query import metrics, param
@@ -39,6 +49,9 @@ __all__ = [
     "AbstractUDF",
     "Aggregator",
     "ArrowRow",
+    "Audio",
+    "AudioFile",
+    "AudioFragment",
     "C",
     "Column",
     "DataChain",
@@ -60,12 +73,19 @@ __all__ = [
     "VideoFile",
     "VideoFragment",
     "VideoFrame",
+    "create_project",
     "datasets",
+    "delete_dataset",
+    "delete_namespace",
     "is_chain_type",
+    "is_local",
+    "is_studio",
     "listings",
     "metrics",
+    "move_dataset",
     "param",
     "read_csv",
+    "read_database",
     "read_dataset",
     "read_hf",
     "read_json",

datachain/asyn.py

@@ -3,6 +3,7 @@ import threading
 from collections.abc import (
     AsyncIterable,
     Awaitable,
+    Callable,
     Coroutine,
     Generator,
     Iterable,
@@ -10,7 +11,7 @@ from collections.abc import (
 )
 from concurrent.futures import ThreadPoolExecutor, wait
 from heapq import heappop, heappush
-from typing import Any, Callable, Generic, Optional, TypeVar
+from typing import Any, Generic, TypeVar
 
 from fsspec.asyn import get_loop
 
@@ -49,7 +50,7 @@ class AsyncMapper(Generic[InputT, ResultT]):
         iterable: Iterable[InputT],
         *,
         workers: int = ASYNC_WORKERS,
-        loop: Optional[asyncio.AbstractEventLoop] = None,
+        loop: asyncio.AbstractEventLoop | None = None,
     ):
         self.func = func
         self.iterable = iterable
@@ -107,9 +108,7 @@ class AsyncMapper(Generic[InputT, ResultT]):
 
     async def init(self) -> None:
         self.work_queue = asyncio.Queue(2 * self.workers)
-        self.result_queue: asyncio.Queue[Optional[ResultT]] = asyncio.Queue(
-            self.workers
-        )
+        self.result_queue: asyncio.Queue[ResultT | None] = asyncio.Queue(self.workers)
 
     async def run(self) -> None:
         producer = self.start_task(self.produce())
@@ -149,10 +148,10 @@ class AsyncMapper(Generic[InputT, ResultT]):
             if exc:
                 raise exc
 
-    async def _pop_result(self) -> Optional[ResultT]:
+    async def _pop_result(self) -> ResultT | None:
         return await self.result_queue.get()
 
-    def next_result(self, timeout=None) -> Optional[ResultT]:
+    def next_result(self, timeout=None) -> ResultT | None:
         """
         Return the next available result.
 
@@ -212,17 +211,17 @@ class OrderedMapper(AsyncMapper[InputT, ResultT]):
         iterable: Iterable[InputT],
         *,
         workers: int = ASYNC_WORKERS,
-        loop: Optional[asyncio.AbstractEventLoop] = None,
+        loop: asyncio.AbstractEventLoop | None = None,
     ):
         super().__init__(func, iterable, workers=workers, loop=loop)
         self._waiters: dict[int, Any] = {}
-        self._getters: dict[int, asyncio.Future[Optional[ResultT]]] = {}
-        self.heap: list[tuple[int, Optional[ResultT]]] = []
+        self._getters: dict[int, asyncio.Future[ResultT | None]] = {}
+        self.heap: list[tuple[int, ResultT | None]] = []
         self._next_yield = 0
         self._items_seen = 0
         self._window = 2 * workers
 
-    def _push_result(self, i: int, result: Optional[ResultT]) -> None:
+    def _push_result(self, i: int, result: ResultT | None) -> None:
         if i in self._getters:
             future = self._getters.pop(i)
             future.set_result(result)
@@ -243,7 +242,7 @@ class OrderedMapper(AsyncMapper[InputT, ResultT]):
     async def init(self) -> None:
         self.work_queue = asyncio.Queue(2 * self.workers)
 
-    async def _pop_result(self) -> Optional[ResultT]:
+    async def _pop_result(self) -> ResultT | None:
         if self.heap and self.heap[0][0] == self._next_yield:
             _i, out = heappop(self.heap)
         else:

datachain/cache.py

@@ -2,7 +2,7 @@ import os
 from collections.abc import Iterator
 from contextlib import contextmanager
 from tempfile import mkdtemp
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from dvc_data.hashfile.db.local import LocalHashFileDB
 from dvc_objects.fs.local import LocalFileSystem
@@ -22,14 +22,14 @@ def try_scandir(path):
         pass
 
 
-def get_temp_cache(tmp_dir: str, prefix: Optional[str] = None) -> "Cache":
+def get_temp_cache(tmp_dir: str, prefix: str | None = None) -> "Cache":
     cache_dir = mkdtemp(prefix=prefix, dir=tmp_dir)
     return Cache(cache_dir, tmp_dir=tmp_dir)
 
 
 @contextmanager
 def temporary_cache(
-    tmp_dir: str, prefix: Optional[str] = None, delete: bool = True
+    tmp_dir: str, prefix: str | None = None, delete: bool = True
 ) -> Iterator["Cache"]:
     cache = get_temp_cache(tmp_dir, prefix=prefix)
     try:
@@ -39,7 +39,7 @@ def temporary_cache(
             cache.destroy()
 
 
-class Cache:
+class Cache:  # noqa: PLW1641
     def __init__(self, cache_dir: str, tmp_dir: str):
         self.odb = LocalHashFileDB(
             LocalFileSystem(),
@@ -58,7 +58,7 @@ class Cache:
     def tmp_dir(self):
         return self.odb.tmp_dir
 
-    def get_path(self, file: "File") -> Optional[str]:
+    def get_path(self, file: "File") -> str | None:
         if self.contains(file):
             return self.path_from_checksum(file.get_hash())
         return None
@@ -74,11 +74,11 @@ class Cache:
         self.odb.delete(file.get_hash())
 
     async def download(
-        self, file: "File", client: "Client", callback: Optional[Callback] = None
+        self, file: "File", client: "Client", callback: Callback | None = None
     ) -> None:
-        from_path = f"{file.source}/{file.path}"
         from dvc_objects.fs.utils import tmp_fname
 
+        from_path = file.get_uri()
         odb_fs = self.odb.fs
         tmp_info = odb_fs.join(self.odb.tmp_dir, tmp_fname())  # type: ignore[arg-type]
         size = file.size

datachain/catalog/__init__.py

@@ -1,15 +1,15 @@
 from .catalog import (
     QUERY_DATASET_PREFIX,
     QUERY_SCRIPT_CANCELED_EXIT_CODE,
-    QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE,
     Catalog,
+    is_namespace_local,
 )
 from .loader import get_catalog
 
 __all__ = [
     "QUERY_DATASET_PREFIX",
     "QUERY_SCRIPT_CANCELED_EXIT_CODE",
-    "QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE",
     "Catalog",
     "get_catalog",
+    "is_namespace_local",
 ]