datachain 0.3.8__tar.gz → 0.3.10__tar.gz

{datachain-0.3.8 → datachain-0.3.10}/.github/workflows/tests.yml

@@ -50,7 +50,7 @@ jobs:
         run: nox -s lint
 
   datachain:
-    timeout-minutes: 30
+    timeout-minutes: 40
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

{datachain-0.3.8 → datachain-0.3.10}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.6.1'
+    rev: 'v0.6.3'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.3.8/src/datachain.egg-info → datachain-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.8
+Version: 0.3.10
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -43,6 +43,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<11,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
+Requires-Dist: huggingface_hub
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -61,7 +62,7 @@ Provides-Extra: vector
 Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
-Requires-Dist: datasets[audio,vision]; extra == "hf"
+Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
 Provides-Extra: tests
 Requires-Dist: datachain[hf,remote,torch,vector]; extra == "tests"
 Requires-Dist: pytest<9,>=8; extra == "tests"
@@ -115,31 +116,30 @@ AI 🔗 DataChain
 
 DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
 It is made to organize your unstructured data into datasets and wrangle it at scale on
-your local machine.
+your local machine. Datachain does not abstract or hide the AI models and API calls, but helps to integrate them into the postmodern data stack.
 
 Key Features
 ============
 
 📂 **Storage as a Source of Truth.**
-   - Process unstructured data without redundant copies: S3, GCP, Azure, and local
+   - Process unstructured data without redundant copies from S3, GCP, Azure, and local
      file systems.
-   - Multimodal data: images, video, text, PDFs, JSONs, CSVs, parquet.
-   - Join files and metadata together into persistent, versioned, columnar datasets.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet.
+   - Unite files and metadata together into persistent, versioned, columnar datasets.
 
 🐍 **Python-friendly data pipelines.**
    - Operate on Python objects and object fields.
-   - Built-in parallelization and out-of-memory compute without a need in SQL or
-     Spark jobs.
+   - Built-in parallelization and out-of-memory compute without SQL or Spark.
 
 🧠 **Data Enrichment and Processing.**
-   - Generate metadata columns using local AI models and LLM APIs.
-   - Filter, join, and group by AI metadata. Vector similarity search.
-   - Pass datasets to Pytorch and Tensorflow, or export back into storage.
+   - Generate metadata using local AI models and LLM APIs.
+   - Filter, join, and group by metadata. Search by vector embeddings.
+   - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
 🚀 **Efficiency.**
    - Parallelization, out-of-memory workloads and data caching.
    - Vectorized operations on Python object fields: sum, count, avg, etc.
-   - Vector search on embeddings.
+   - Optimized vector search.
 
 
 Quick Start
@@ -164,7 +164,7 @@ where each image has a matching JSON file like `cat.1009.json`:
         "inference": {"class": "dog", "confidence": 0.68}
     }
 
-Example of downloading only high-confidence cat images using JSON metadata:
+Example of downloading only "high-confidence cat" inferred images using JSON metadata:
 
 
 .. code:: py
@@ -234,7 +234,7 @@ detected are then copied to the local directory.
 LLM judging chatbots
 =============================
 
-LLMs can work as efficient universal classifiers. In the example below,
+LLMs can work as universal classifiers. In the example below,
 we employ a free API from Mistral to judge the `publicly available`_ chatbot dialogs. Please get a free
 Mistral API key at https://console.mistral.ai
 

{datachain-0.3.8 → datachain-0.3.10}/README.rst

@@ -18,31 +18,30 @@ AI 🔗 DataChain
 
 DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
 It is made to organize your unstructured data into datasets and wrangle it at scale on
-your local machine.
+your local machine. Datachain does not abstract or hide the AI models and API calls, but helps to integrate them into the postmodern data stack.
 
 Key Features
 ============
 
 📂 **Storage as a Source of Truth.**
-   - Process unstructured data without redundant copies: S3, GCP, Azure, and local
+   - Process unstructured data without redundant copies from S3, GCP, Azure, and local
      file systems.
-   - Multimodal data: images, video, text, PDFs, JSONs, CSVs, parquet.
-   - Join files and metadata together into persistent, versioned, columnar datasets.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet.
+   - Unite files and metadata together into persistent, versioned, columnar datasets.
 
 🐍 **Python-friendly data pipelines.**
    - Operate on Python objects and object fields.
-   - Built-in parallelization and out-of-memory compute without a need in SQL or
-     Spark jobs.
+   - Built-in parallelization and out-of-memory compute without SQL or Spark.
 
 🧠 **Data Enrichment and Processing.**
-   - Generate metadata columns using local AI models and LLM APIs.
-   - Filter, join, and group by AI metadata. Vector similarity search.
-   - Pass datasets to Pytorch and Tensorflow, or export back into storage.
+   - Generate metadata using local AI models and LLM APIs.
+   - Filter, join, and group by metadata. Search by vector embeddings.
+   - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
 🚀 **Efficiency.**
    - Parallelization, out-of-memory workloads and data caching.
    - Vectorized operations on Python object fields: sum, count, avg, etc.
-   - Vector search on embeddings.
+   - Optimized vector search.
 
 
 Quick Start
@@ -67,7 +66,7 @@ where each image has a matching JSON file like `cat.1009.json`:
         "inference": {"class": "dog", "confidence": 0.68}
     }
 
-Example of downloading only high-confidence cat images using JSON metadata:
+Example of downloading only "high-confidence cat" inferred images using JSON metadata:
 
 
 .. code:: py
@@ -137,7 +136,7 @@ detected are then copied to the local directory.
 LLM judging chatbots
 =============================
 
-LLMs can work as efficient universal classifiers. In the example below,
+LLMs can work as universal classifiers. In the example below,
 we employ a free API from Mistral to judge the `publicly available`_ chatbot dialogs. Please get a free
 Mistral API key at https://console.mistral.ai
 

{datachain-0.3.8 → datachain-0.3.10}/examples/get_started/udfs/parallel.py

@@ -31,7 +31,7 @@ def path_len_benchmark(path):
 
 # Run in chain
 DataChain.from_storage(
-    path="gs://datachain-demo/dogs-and-cats/",
+    "gs://datachain-demo/dogs-and-cats/",
 ).settings(parallel=-1).map(
     path_len_benchmark,
     params=["file.path"],

{datachain-0.3.8 → datachain-0.3.10}/examples/get_started/udfs/simple.py

@@ -11,7 +11,7 @@ def path_len(path):
 if __name__ == "__main__":
     # Run in chain
     DataChain.from_storage(
-        path="gs://datachain-demo/dogs-and-cats/",
+        uri="gs://datachain-demo/dogs-and-cats/",
     ).map(
         path_len,
         params=["file.path"],

{datachain-0.3.8 → datachain-0.3.10}/examples/llm_and_nlp/unstructured-text.py

@@ -1,5 +1,5 @@
 #
-# pip install unstructured[pdf] nltk==3.8.1 huggingface_hub[hf_transfer]
+# pip install unstructured[pdf] huggingface_hub[hf_transfer]
 #
 import os
 

{datachain-0.3.8 → datachain-0.3.10}/examples/multimodal/wds_filtered.py

@@ -1,13 +1,11 @@
 import datachain.error
 from datachain import C, DataChain
-from datachain.lib.model_store import ModelStore
 from datachain.lib.webdataset import process_webdataset
-from datachain.lib.webdataset_laion import LaionMeta, WDSLaion
+from datachain.lib.webdataset_laion import WDSLaion
 from datachain.sql import literal
 from datachain.sql.functions import array, greatest, least, string
 
 name = "wds"
-ModelStore.register(LaionMeta)
 try:
     wds = DataChain.from_dataset(name=name)
 except datachain.error.DatasetNotFoundError:

{datachain-0.3.8 → datachain-0.3.10}/pyproject.toml

@@ -45,7 +45,8 @@ dependencies = [
   "datamodel-code-generator>=0.25",
   "Pillow>=10.0.0,<11",
   "msgpack>=1.0.4,<2",
-  "psutil"
+  "psutil",
+  "huggingface_hub"
 ]
 
 [project.optional-dependencies]
@@ -71,7 +72,7 @@ vector = [
 ]
 hf = [
   "numba>=0.60.0",
-  "datasets[audio,vision]"
+  "datasets[audio,vision]>=2.21.0"
 ]
 tests = [
   "datachain[torch,remote,vector,hf]",

{datachain-0.3.8 → datachain-0.3.10}/src/datachain/catalog/catalog.py

@@ -156,8 +156,6 @@ class QueryResult(NamedTuple):
     dataset: Optional[DatasetRecord]
     version: Optional[int]
     output: str
-    preview: Optional[list[dict]]
-    metrics: dict[str, Any]
 
 
 class DatasetRowsFetcher(NodesThreadPool):
@@ -1020,20 +1018,6 @@ class Catalog:
 
         return node_groups
 
-    def unlist_source(self, uri: StorageURI) -> None:
-        self.metastore.clone(uri=uri).mark_storage_not_indexed(uri)
-
-    def storage_stats(self, uri: StorageURI) -> Optional[DatasetStats]:
-        """
-        Returns tuple with storage stats: total number of rows and total dataset size.
-        """
-        partial_path = self.metastore.get_last_partial_path(uri)
-        if partial_path is None:
-            return None
-        dataset = self.get_dataset(Storage.dataset_name(uri, partial_path))
-
-        return self.dataset_stats(dataset.name, dataset.latest_version)
-
     def create_dataset(
         self,
         name: str,
@@ -1297,19 +1281,6 @@ class Catalog:
 
         return self.get_dataset(name)
 
-    def register_new_dataset(
-        self,
-        source_dataset: DatasetRecord,
-        source_version: int,
-        target_name: str,
-    ) -> DatasetRecord:
-        target_dataset = self.metastore.create_dataset(
-            target_name,
-            query_script=source_dataset.query_script,
-            schema=source_dataset.serialized_schema,
-        )
-        return self.register_dataset(source_dataset, source_version, target_dataset, 1)
-
     def register_dataset(
         self,
         dataset: DatasetRecord,
@@ -1422,17 +1393,18 @@ class Catalog:
 
         return direct_dependencies
 
-    def ls_datasets(self) -> Iterator[DatasetRecord]:
+    def ls_datasets(self, include_listing: bool = False) -> Iterator[DatasetRecord]:
         datasets = self.metastore.list_datasets()
         for d in datasets:
-            if not d.is_bucket_listing:
+            if not d.is_bucket_listing or include_listing:
                 yield d
 
     def list_datasets_versions(
         self,
+        include_listing: bool = False,
     ) -> Iterator[tuple[DatasetRecord, "DatasetVersion", Optional["Job"]]]:
         """Iterate over all dataset versions with related jobs."""
-        datasets = list(self.ls_datasets())
+        datasets = list(self.ls_datasets(include_listing=include_listing))
 
         # preselect dataset versions jobs from db to avoid multiple queries
         jobs_ids: set[str] = {
@@ -1560,17 +1532,8 @@ class Catalog:
         version = self.get_dataset(dataset_name).get_version(dataset_version)
 
         file_signals_values = {}
-        file_schemas = {}
-        # TODO: To remove after we properly fix deserialization
-        for signal, type_name in version.feature_schema.items():
-            from datachain.lib.model_store import ModelStore
-
-            type_name_parsed, v = ModelStore.parse_name_version(type_name)
-            fr = ModelStore.get(type_name_parsed, v)
-            if fr and issubclass(fr, File):
-                file_schemas[signal] = type_name
 
-        schema = SignalSchema.deserialize(file_schemas)
+        schema = SignalSchema.deserialize(version.feature_schema)
         for file_signals in schema.get_signals(File):
             prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
             file_signals_values[file_signals] = {
@@ -1641,15 +1604,6 @@ class Catalog:
         for source in data_sources:  # type: ignore [union-attr]
             yield source, source.ls(fields)
 
-    def ls_storage_uris(self) -> Iterator[str]:
-        yield from self.metastore.get_all_storage_uris()
-
-    def get_storage(self, uri: StorageURI) -> Storage:
-        return self.metastore.get_storage(uri)
-
-    def ls_storages(self) -> list[Storage]:
-        return self.metastore.list_storages()
-
     def pull_dataset(
         self,
         dataset_uri: str,
@@ -1883,10 +1837,6 @@ class Catalog:
         envs: Optional[Mapping[str, str]] = None,
         python_executable: Optional[str] = None,
         save: bool = False,
-        save_as: Optional[str] = None,
-        preview_limit: int = 10,
-        preview_offset: int = 0,
-        preview_columns: Optional[list[str]] = None,
         capture_output: bool = True,
         output_hook: Callable[[str], None] = noop,
         params: Optional[dict[str, str]] = None,
@@ -1914,9 +1864,8 @@ class Catalog:
                 C.size > 1000
             )
         """
-        from datachain.query.dataset import ExecutionResult
 
-        feature_file = tempfile.NamedTemporaryFile(
+        feature_file = tempfile.NamedTemporaryFile(  # noqa: SIM115
             dir=os.getcwd(), suffix=".py", delete=False
         )
         _, feature_module = os.path.split(feature_file.name)
@@ -1931,11 +1880,7 @@ class Catalog:
                 feature_module,
                 output_hook,
                 params,
-                preview_columns,
-                preview_limit,
-                preview_offset,
                 save,
-                save_as,
                 job_id,
             )
         finally:
@@ -1964,25 +1909,18 @@ class Catalog:
             )
 
         try:
-            response = json.loads(response_text)
+            result = json.loads(response_text)
         except ValueError:
-            response = {}
-        exec_result = ExecutionResult(**response)
+            result = None
 
         dataset: Optional[DatasetRecord] = None
         version: Optional[int] = None
-        if save or save_as:
+        if save:
             dataset, version = self.save_result(
-                query_script, exec_result, output, version, job_id
+                query_script, result, output, version, job_id
             )
 
-        return QueryResult(
-            dataset=dataset,
-            version=version,
-            output=output,
-            preview=exec_result.preview,
-            metrics=exec_result.metrics,
-        )
+        return QueryResult(dataset=dataset, version=version, output=output)
 
     def run_query(
         self,
@@ -1994,11 +1932,7 @@ class Catalog:
         feature_module: str,
         output_hook: Callable[[str], None],
         params: Optional[dict[str, str]],
-        preview_columns: Optional[list[str]],
-        preview_limit: int,
-        preview_offset: int,
         save: bool,
-        save_as: Optional[str],
         job_id: Optional[str],
     ) -> tuple[list[str], subprocess.Popen, str]:
         try:
@@ -2013,10 +1947,6 @@ class Catalog:
             raise QueryScriptCompileError(
                 f"Query script failed to compile, reason: {exc}"
             ) from exc
-        if save_as and save_as.startswith(QUERY_DATASET_PREFIX):
-            raise ValueError(
-                f"Cannot use {QUERY_DATASET_PREFIX} prefix for dataset name"
-            )
         r, w = os.pipe()
         if os.name == "nt":
             import msvcrt
@@ -2039,15 +1969,7 @@ class Catalog:
             {
                 "DATACHAIN_QUERY_PARAMS": json.dumps(params or {}),
                 "PYTHONPATH": os.getcwd(),  # For local imports
-                "DATACHAIN_QUERY_PREVIEW_ARGS": json.dumps(
-                    {
-                        "limit": preview_limit,
-                        "offset": preview_offset,
-                        "columns": preview_columns,
-                    }
-                ),
                 "DATACHAIN_QUERY_SAVE": "1" if save else "",
-                "DATACHAIN_QUERY_SAVE_AS": save_as or "",
                 "PYTHONUNBUFFERED": "1",
                 "DATACHAIN_OUTPUT_FD": str(handle),
                 "DATACHAIN_JOB_ID": job_id or "",
@@ -2077,12 +1999,12 @@ class Catalog:
         return lines, proc, response_text
 
     def save_result(self, query_script, exec_result, output, version, job_id):
-        if not exec_result.dataset:
+        if not exec_result:
             raise QueryScriptDatasetNotFound(
                 "No dataset found after running Query script",
                 output=output,
             )
-        name, version = exec_result.dataset
+        name, version = exec_result
         # finding returning dataset
         try:
             dataset = self.get_dataset(name)

{datachain-0.3.8 → datachain-0.3.10}/src/datachain/cli.py

@@ -14,6 +14,7 @@ import shtab
 
 from datachain import utils
 from datachain.cli_utils import BooleanOptionalAction, CommaSeparatedArgs, KeyValueArgs
+from datachain.lib.dc import DataChain
 from datachain.utils import DataChainDir
 
 if TYPE_CHECKING:
@@ -472,9 +473,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     query_parser.add_argument(
         "script", metavar="<script.py>", type=str, help="Filepath for script"
     )
-    query_parser.add_argument(
-        "dataset_name", nargs="?", type=str, help="Save result dataset as"
-    )
     query_parser.add_argument(
         "--parallel",
         nargs="?",
@@ -487,7 +485,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
             "N defaults to the CPU count."
         ),
     )
-    add_show_args(query_parser)
     query_parser.add_argument(
         "-p",
         "--param",
@@ -619,18 +616,6 @@ def _ls_urls_flat(
                 raise FileNotFoundError(f"No such file or directory: {source}")
 
 
-def ls_indexed_storages(catalog: "Catalog", long: bool = False) -> Iterator[str]:
-    from datachain.node import long_line_str
-
-    storage_uris = catalog.ls_storage_uris()
-    if long:
-        for uri in storage_uris:
-            # TODO: add Storage.created so it can be used here
-            yield long_line_str(uri, None, "")
-    else:
-        yield from storage_uris
-
-
 def ls_local(
     sources,
     long: bool = False,
@@ -661,8 +646,9 @@ def ls_local(
                 for entry in entries:
                     print(format_ls_entry(entry))
     else:
-        for entry in ls_indexed_storages(catalog, long=long):
-            print(format_ls_entry(entry))
+        chain = DataChain.listings()
+        for ls in chain.collect("listing"):
+            print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 
 
 def format_ls_entry(entry: str) -> str:
@@ -813,16 +799,10 @@ def show(
 def query(
     catalog: "Catalog",
     script: str,
-    dataset_name: Optional[str] = None,
     parallel: Optional[int] = None,
-    limit: int = 10,
-    offset: int = 0,
-    columns: Optional[list[str]] = None,
-    no_collapse: bool = False,
     params: Optional[dict[str, str]] = None,
 ) -> None:
     from datachain.data_storage import JobQueryType, JobStatus
-    from datachain.utils import show_records
 
     with open(script, encoding="utf-8") as f:
         script_content = f.read()
@@ -843,13 +823,9 @@ def query(
     )
 
     try:
-        result = catalog.query(
+        catalog.query(
             script_content,
             python_executable=python_executable,
-            save_as=dataset_name,
-            preview_limit=limit,
-            preview_offset=offset,
-            preview_columns=columns,
             capture_output=False,
             params=params,
             job_id=job_id,
@@ -864,10 +840,7 @@ def query(
             error_stack=error_stack,
         )
         raise
-
-    catalog.metastore.set_job_status(job_id, JobStatus.COMPLETE, metrics=result.metrics)
-
-    show_records(result.preview, collapse_columns=not no_collapse)
+    catalog.metastore.set_job_status(job_id, JobStatus.COMPLETE)
 
 
 def clear_cache(catalog: "Catalog"):
@@ -1042,12 +1015,7 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
             query(
                 catalog,
                 args.script,
-                dataset_name=args.dataset_name,
                 parallel=args.parallel,
-                limit=args.limit,
-                offset=args.offset,
-                columns=args.columns,
-                no_collapse=args.no_collapse,
                 params=args.param,
             )
         elif args.command == "apply-udf":

{datachain-0.3.8 → datachain-0.3.10}/src/datachain/client/fsspec.py

@@ -87,6 +87,7 @@ class Client(ABC):
     def get_implementation(url: str) -> type["Client"]:
         from .azure import AzureClient
         from .gcs import GCSClient
+        from .hf import HfClient
         from .local import FileClient
         from .s3 import ClientS3
 
@@ -104,6 +105,8 @@ class Client(ABC):
             return AzureClient
         if protocol == FileClient.protocol:
             return FileClient
+        if protocol == HfClient.protocol:
+            return HfClient
 
         raise NotImplementedError(f"Unsupported protocol: {protocol}")
 

datachain-0.3.10/src/datachain/client/hf.py

@@ -0,0 +1,47 @@
+import os
+import posixpath
+from typing import Any, cast
+
+from huggingface_hub import HfFileSystem
+
+from datachain.lib.file import File
+from datachain.node import Entry
+
+from .fsspec import Client
+
+
+class HfClient(Client):
+    FS_CLASS = HfFileSystem
+    PREFIX = "hf://"
+    protocol = "hf"
+
+    @classmethod
+    def create_fs(cls, **kwargs) -> HfFileSystem:
+        if os.environ.get("HF_TOKEN"):
+            kwargs["token"] = os.environ["HF_TOKEN"]
+
+        return cast(HfFileSystem, super().create_fs(**kwargs))
+
+    def convert_info(self, v: dict[str, Any], path: str) -> Entry:
+        return Entry.from_file(
+            path=path,
+            size=v["size"],
+            version=v["last_commit"].oid,
+            etag=v.get("blob_id", ""),
+            last_modified=v["last_commit"].date,
+        )
+
+    def info_to_file(self, v: dict[str, Any], path: str) -> File:
+        return File(
+            path=path,
+            size=v["size"],
+            version=v["last_commit"].oid,
+            etag=v.get("blob_id", ""),
+            last_modified=v["last_commit"].date,
+        )
+
+    async def ls_dir(self, path):
+        return self.fs.ls(path, detail=True)
+
+    def rel_path(self, path):
+        return posixpath.relpath(path, self.name)