datachain 0.6.7__tar.gz → 0.6.9__tar.gz

{datachain-0.6.7 → datachain-0.6.9}/.github/workflows/tests-studio.yml

@@ -32,7 +32,7 @@ jobs:
           POSTGRES_DB: database
           POSTGRES_HOST_AUTH_METHOD: trust
       clickhouse:
-        image: clickhouse/clickhouse-server:24
+        image: clickhouse/clickhouse-server:24.6
         ports:
           - 8123:8123
           - 9010:9000

{datachain-0.6.7 → datachain-0.6.9}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.2'
+    rev: 'v0.7.3'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.6.7/src/datachain.egg-info → datachain-0.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.7
+Version: 0.6.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -45,6 +45,7 @@ Requires-Dist: huggingface_hub
 Requires-Dist: iterative-telemetry>=0.0.9
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
+Requires-Dist: tabulate
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -87,6 +88,7 @@ Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
 Provides-Extra: examples
 Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: numpy<2,>=1; extra == "examples"
@@ -118,33 +120,41 @@ Requires-Dist: onnx==1.16.1; extra == "examples"
    :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
    :alt: Tests
 
-DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
-It is made to organize your unstructured data into datasets and wrangle it at scale on
-your local machine. Datachain does not abstract or hide the AI models and API calls, but helps to integrate them into the postmodern data stack.
+DataChain is a Python-based AI-data warehouse for transforming and analyzing unstructured
+data like images, audio, videos, text and PDFs. It integrates with external storage
+(e.g., S3) to process data efficiently without data duplication and manages metadata
+in an internal database for easy and efficient querying.
+
+
+Use Cases
+=========
+
+1. **Multimodal Dataset Preparation and Curation**: ideal for organizing and
+   refining data in pre-training, finetuning or LLM evaluating stages.
+2. **GenAI Data Analytics**: Enables advanced analytics for multimodal data and
+   ad-hoc analytics using LLMs.
 
 Key Features
 ============
 
-📂 **Storage as a Source of Truth.**
-   - Process unstructured data without redundant copies from S3, GCP, Azure, and local
-     file systems.
-   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet.
+📂 **Multimodal Dataset Versioning.**
+   - Version unstructured data without redundant data copies, by supporitng
+     references to S3, GCP, Azure, and local file systems.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet, etc.
    - Unite files and metadata together into persistent, versioned, columnar datasets.
 
-🐍 **Python-friendly data pipelines.**
-   - Operate on Python objects and object fields.
-   - Built-in parallelization and out-of-memory compute without SQL or Spark.
+🐍 **Python-friendly.**
+   - Operate on Python objects and object fields: float scores, strings, matrixes,
+     LLM response objects.
+   - Run Python code in a high-scale, terabytes size datasets, with built-in
+     parallelization and memory-efficient computing — no SQL or Spark required.
 
 🧠 **Data Enrichment and Processing.**
    - Generate metadata using local AI models and LLM APIs.
-   - Filter, join, and group by metadata. Search by vector embeddings.
+   - Filter, join, and group datasets by metadata. Search by vector embeddings.
+   - High-performance vectorized operations on Python objects: sum, count, avg, etc.
    - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
-🚀 **Efficiency.**
-   - Parallelization, out-of-memory workloads and data caching.
-   - Vectorized operations on Python object fields: sum, count, avg, etc.
-   - Optimized vector search.
-
 
 Quick Start
 -----------
@@ -194,7 +204,7 @@ Batch inference with a simple sentiment model using the `transformers` library:
 
     pip install transformers
 
-The code below downloads files the cloud, and applies a user-defined function
+The code below downloads files from the cloud, and applies a user-defined function
 to each one of them. All files with a positive sentiment
 detected are then copied to the local directory.
 
@@ -427,6 +437,19 @@ name suffix, the following code will do it:
     loader = DataLoader(chain, batch_size=1)
 
 
+DataChain Studio Platform
+-------------------------
+
+`DataChain Studio`_ is a proprietary solution for teams that offers:
+
+- **Centralized dataset registry** to manage data, code and dependency
+  dependencies in one place.
+- **Data Lineage** for data sources as well as direvative dataset.
+- **UI for Multimodal Data** like images, videos, and PDFs.
+- **Scalable Compute** to handle large datasets (100M+ files) and in-house
+  AI model inference.
+- **Access control** including SSO and team based collaboration.
+
 Tutorials
 ---------
 
@@ -460,6 +483,5 @@ Community and Support
 .. _Pydantic: https://github.com/pydantic/pydantic
 .. _publicly available: https://radar.kit.edu/radar/en/dataset/FdJmclKpjHzLfExE.ExpBot%2B-%2BA%2Bdataset%2Bof%2B79%2Bdialogs%2Bwith%2Ban%2Bexperimental%2Bcustomer%2Bservice%2Bchatbot
 .. _SQLite: https://www.sqlite.org/
-.. _Getting Started: https://datachain.dvc.ai/
-.. |Flowchart| image:: https://github.com/iterative/datachain/blob/main/docs/assets/flowchart.png?raw=true
-   :alt: DataChain FlowChart
+.. _Getting Started: https://docs.datachain.ai/
+.. _DataChain Studio: https://studio.datachain.ai/

{datachain-0.6.7 → datachain-0.6.9}/README.rst

@@ -19,33 +19,41 @@
    :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
    :alt: Tests
 
-DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
-It is made to organize your unstructured data into datasets and wrangle it at scale on
-your local machine. Datachain does not abstract or hide the AI models and API calls, but helps to integrate them into the postmodern data stack.
+DataChain is a Python-based AI-data warehouse for transforming and analyzing unstructured
+data like images, audio, videos, text and PDFs. It integrates with external storage
+(e.g., S3) to process data efficiently without data duplication and manages metadata
+in an internal database for easy and efficient querying.
+
+
+Use Cases
+=========
+
+1. **Multimodal Dataset Preparation and Curation**: ideal for organizing and
+   refining data in pre-training, finetuning or LLM evaluating stages.
+2. **GenAI Data Analytics**: Enables advanced analytics for multimodal data and
+   ad-hoc analytics using LLMs.
 
 Key Features
 ============
 
-📂 **Storage as a Source of Truth.**
-   - Process unstructured data without redundant copies from S3, GCP, Azure, and local
-     file systems.
-   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet.
+📂 **Multimodal Dataset Versioning.**
+   - Version unstructured data without redundant data copies, by supporitng
+     references to S3, GCP, Azure, and local file systems.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet, etc.
    - Unite files and metadata together into persistent, versioned, columnar datasets.
 
-🐍 **Python-friendly data pipelines.**
-   - Operate on Python objects and object fields.
-   - Built-in parallelization and out-of-memory compute without SQL or Spark.
+🐍 **Python-friendly.**
+   - Operate on Python objects and object fields: float scores, strings, matrixes,
+     LLM response objects.
+   - Run Python code in a high-scale, terabytes size datasets, with built-in
+     parallelization and memory-efficient computing — no SQL or Spark required.
 
 🧠 **Data Enrichment and Processing.**
    - Generate metadata using local AI models and LLM APIs.
-   - Filter, join, and group by metadata. Search by vector embeddings.
+   - Filter, join, and group datasets by metadata. Search by vector embeddings.
+   - High-performance vectorized operations on Python objects: sum, count, avg, etc.
    - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
-🚀 **Efficiency.**
-   - Parallelization, out-of-memory workloads and data caching.
-   - Vectorized operations on Python object fields: sum, count, avg, etc.
-   - Optimized vector search.
-
 
 Quick Start
 -----------
@@ -95,7 +103,7 @@ Batch inference with a simple sentiment model using the `transformers` library:
 
     pip install transformers
 
-The code below downloads files the cloud, and applies a user-defined function
+The code below downloads files from the cloud, and applies a user-defined function
 to each one of them. All files with a positive sentiment
 detected are then copied to the local directory.
 
@@ -328,6 +336,19 @@ name suffix, the following code will do it:
     loader = DataLoader(chain, batch_size=1)
 
 
+DataChain Studio Platform
+-------------------------
+
+`DataChain Studio`_ is a proprietary solution for teams that offers:
+
+- **Centralized dataset registry** to manage data, code and dependency
+  dependencies in one place.
+- **Data Lineage** for data sources as well as direvative dataset.
+- **UI for Multimodal Data** like images, videos, and PDFs.
+- **Scalable Compute** to handle large datasets (100M+ files) and in-house
+  AI model inference.
+- **Access control** including SSO and team based collaboration.
+
 Tutorials
 ---------
 
@@ -361,6 +382,5 @@ Community and Support
 .. _Pydantic: https://github.com/pydantic/pydantic
 .. _publicly available: https://radar.kit.edu/radar/en/dataset/FdJmclKpjHzLfExE.ExpBot%2B-%2BA%2Bdataset%2Bof%2B79%2Bdialogs%2Bwith%2Ban%2Bexperimental%2Bcustomer%2Bservice%2Bchatbot
 .. _SQLite: https://www.sqlite.org/
-.. _Getting Started: https://datachain.dvc.ai/
-.. |Flowchart| image:: https://github.com/iterative/datachain/blob/main/docs/assets/flowchart.png?raw=true
-   :alt: DataChain FlowChart
+.. _Getting Started: https://docs.datachain.ai/
+.. _DataChain Studio: https://studio.datachain.ai/

{datachain-0.6.7 → datachain-0.6.9}/pyproject.toml

@@ -47,7 +47,8 @@ dependencies = [
   "huggingface_hub",
   "iterative-telemetry>=0.0.9",
   "platformdirs",
-  "dvc-studio-client>=0.21,<1"
+  "dvc-studio-client>=0.21,<1",
+  "tabulate"
 ]
 
 [project.optional-dependencies]
@@ -98,7 +99,8 @@ dev = [
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
-  "types-requests"
+  "types-requests",
+  "types-tabulate"
 ]
 examples = [
   "datachain[tests]",

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/__init__.py

@@ -1,4 +1,4 @@
-from datachain.lib import func
+from datachain.lib import func, models
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import C, Column, DataChain, Sys
 from datachain.lib.file import (
@@ -38,5 +38,6 @@ __all__ = [
     "func",
     "is_chain_type",
     "metrics",
+    "models",
     "param",
 ]

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/catalog/catalog.py

@@ -769,6 +769,7 @@ class Catalog:
         create_rows: Optional[bool] = True,
         validate_version: Optional[bool] = True,
         listing: Optional[bool] = False,
+        uuid: Optional[str] = None,
     ) -> "DatasetRecord":
         """
         Creates new dataset of a specific version.
@@ -816,6 +817,7 @@ class Catalog:
             query_script=query_script,
             create_rows_table=create_rows,
             columns=columns,
+            uuid=uuid,
         )
 
     def create_new_dataset_version(
@@ -832,6 +834,7 @@ class Catalog:
         script_output="",
         create_rows_table=True,
         job_id: Optional[str] = None,
+        uuid: Optional[str] = None,
     ) -> DatasetRecord:
         """
         Creates dataset version if it doesn't exist.
@@ -855,6 +858,7 @@ class Catalog:
             schema=schema,
             job_id=job_id,
             ignore_if_exists=True,
+            uuid=uuid,
         )
 
         if create_rows_table:
@@ -1400,6 +1404,7 @@ class Catalog:
             columns=columns,
             feature_schema=remote_dataset_version.feature_schema,
             validate_version=False,
+            uuid=remote_dataset_version.uuid,
         )
 
         # asking remote to export dataset rows table to s3 and to return signed

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/cli.py

@@ -4,18 +4,21 @@ import shlex
 import sys
 import traceback
 from argparse import Action, ArgumentParser, ArgumentTypeError, Namespace
-from collections.abc import Iterable, Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Sequence
 from importlib.metadata import PackageNotFoundError, version
 from itertools import chain
 from multiprocessing import freeze_support
 from typing import TYPE_CHECKING, Optional, Union
 
 import shtab
+from tabulate import tabulate
 
 from datachain import Session, utils
 from datachain.cli_utils import BooleanOptionalAction, CommaSeparatedArgs, KeyValueArgs
+from datachain.config import Config
+from datachain.error import DataChainError
 from datachain.lib.dc import DataChain
-from datachain.studio import process_studio_cli_args
+from datachain.studio import list_datasets, process_studio_cli_args
 from datachain.telemetry import telemetry
 
 if TYPE_CHECKING:
@@ -416,7 +419,36 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="Dataset labels",
     )
 
-    subp.add_parser("ls-datasets", parents=[parent_parser], description="List datasets")
+    datasets_parser = subp.add_parser(
+        "datasets", parents=[parent_parser], description="List datasets"
+    )
+    datasets_parser.add_argument(
+        "--studio",
+        action="store_true",
+        default=False,
+        help="List the files in the Studio",
+    )
+    datasets_parser.add_argument(
+        "-L",
+        "--local",
+        action="store_true",
+        default=False,
+        help="List local files only",
+    )
+    datasets_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=True,
+        help="List all files including hidden files",
+    )
+    datasets_parser.add_argument(
+        "--team",
+        action="store",
+        default=None,
+        help="The team to list datasets for. By default, it will use team from config.",
+    )
+
     rm_dataset_parser = subp.add_parser(
         "rm-dataset", parents=[parent_parser], description="Removes dataset"
     )
@@ -474,10 +506,30 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="List files in the long format",
     )
     parse_ls.add_argument(
-        "--remote",
+        "--studio",
+        action="store_true",
+        default=False,
+        help="List the files in the Studio",
+    )
+    parse_ls.add_argument(
+        "-L",
+        "--local",
+        action="store_true",
+        default=False,
+        help="List local files only",
+    )
+    parse_ls.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=True,
+        help="List all files including hidden files",
+    )
+    parse_ls.add_argument(
+        "--team",
         action="store",
-        default="",
-        help="Name of remote to use",
+        default=None,
+        help="The team to list datasets for. By default, it will use team from config.",
     )
 
     parse_du = subp.add_parser(
@@ -758,11 +810,12 @@ def format_ls_entry(entry: str) -> str:
 def ls_remote(
     paths: Iterable[str],
     long: bool = False,
+    team: Optional[str] = None,
 ):
     from datachain.node import long_line_str
     from datachain.remote.studio import StudioClient
 
-    client = StudioClient()
+    client = StudioClient(team=team)
     first = True
     for path, response in client.ls(paths):
         if not first:
@@ -789,28 +842,66 @@ def ls_remote(
 def ls(
     sources,
     long: bool = False,
-    remote: str = "",
-    config: Optional[Mapping[str, str]] = None,
+    studio: bool = False,
+    local: bool = False,
+    all: bool = True,
+    team: Optional[str] = None,
     **kwargs,
 ):
-    if config is None:
-        from .config import Config
+    token = Config().read().get("studio", {}).get("token")
+    all, local, studio = _determine_flavors(studio, local, all, token)
 
-        config = Config().get_remote_config(remote=remote)
-    remote_type = config["type"]
-    if remote_type == "local":
+    if all or local:
         ls_local(sources, long=long, **kwargs)
-    else:
-        ls_remote(
-            sources,
-            long=long,
+
+    if (all or studio) and token:
+        ls_remote(sources, long=long, team=team)
+
+
+def datasets(
+    catalog: "Catalog",
+    studio: bool = False,
+    local: bool = False,
+    all: bool = True,
+    team: Optional[str] = None,
+):
+    token = Config().read().get("studio", {}).get("token")
+    all, local, studio = _determine_flavors(studio, local, all, token)
+
+    local_datasets = set(list_datasets_local(catalog)) if all or local else set()
+    studio_datasets = (
+        set(list_datasets(team=team)) if (all or studio) and token else set()
+    )
+
+    rows = [
+        _datasets_tabulate_row(
+            name=name,
+            version=version,
+            both=(all or (local and studio)) and token,
+            local=(name, version) in local_datasets,
+            studio=(name, version) in studio_datasets,
         )
+        for name, version in local_datasets.union(studio_datasets)
+    ]
+
+    print(tabulate(rows, headers="keys"))
 
 
-def ls_datasets(catalog: "Catalog"):
+def list_datasets_local(catalog: "Catalog"):
     for d in catalog.ls_datasets():
         for v in d.versions:
-            print(f"{d.name} (v{v.version})")
+            yield (d.name, v.version)
+
+
+def _datasets_tabulate_row(name, version, both, local, studio):
+    row = {
+        "Name": name,
+        "Version": version,
+    }
+    if both:
+        row["Studio"] = "\u2714" if studio else "\u2716"
+        row["Local"] = "\u2714" if local else "\u2716"
+    return row
 
 
 def rm_dataset(
@@ -953,6 +1044,20 @@ def completion(shell: str) -> str:
     )
 
 
+def _determine_flavors(studio: bool, local: bool, all: bool, token: Optional[str]):
+    if studio and not token:
+        raise DataChainError(
+            "Not logged in to Studio. Log in with 'datachain studio login'."
+        )
+
+    if local or studio:
+        all = False
+
+    all = all and not (local or studio)
+
+    return all, local, studio
+
+
 def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR0915
     # Required for Windows multiprocessing support
     freeze_support()
@@ -1032,12 +1137,21 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
             ls(
                 args.sources,
                 long=bool(args.long),
-                remote=args.remote,
+                studio=args.studio,
+                local=args.local,
+                all=args.all,
+                team=args.team,
                 update=bool(args.update),
                 client_config=client_config,
             )
-        elif args.command == "ls-datasets":
-            ls_datasets(catalog)
+        elif args.command == "datasets":
+            datasets(
+                catalog=catalog,
+                studio=args.studio,
+                local=args.local,
+                all=args.all,
+                team=args.team,
+            )
         elif args.command == "show":
             show(
                 catalog,

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/client/fsspec.py

@@ -358,7 +358,7 @@ class Client(ABC):
     ) -> BinaryIO:
         """Open a file, including files in tar archives."""
         if use_cache and (cache_path := self.cache.get_path(file)):
-            return open(cache_path, mode="rb")  # noqa: SIM115
+            return open(cache_path, mode="rb")
         assert not file.location
         return FileWrapper(self.fs.open(self.get_full_path(file.path)), cb)  # type: ignore[return-value]
 

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/data_storage/metastore.py

@@ -138,6 +138,7 @@ class AbstractMetastore(ABC, Serializable):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
+        uuid: Optional[str] = None,
     ) -> DatasetRecord:
         """Creates new dataset version."""
 
@@ -352,6 +353,7 @@ class AbstractDBMetastore(AbstractMetastore):
         """Datasets versions table columns."""
         return [
             Column("id", Integer, primary_key=True),
+            Column("uuid", Text, nullable=False, default=uuid4()),
             Column(
                 "dataset_id",
                 Integer,
@@ -545,6 +547,7 @@ class AbstractDBMetastore(AbstractMetastore):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
+        uuid: Optional[str] = None,
         conn=None,
     ) -> DatasetRecord:
         """Creates new dataset version."""
@@ -555,6 +558,7 @@ class AbstractDBMetastore(AbstractMetastore):
 
         query = self._datasets_versions_insert().values(
             dataset_id=dataset.id,
+            uuid=uuid or str(uuid4()),
             version=version,
             status=status,
             feature_schema=json.dumps(feature_schema or {}),

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/dataset.py

@@ -163,6 +163,7 @@ class DatasetStatus:
 @dataclass
 class DatasetVersion:
     id: int
+    uuid: str
     dataset_id: int
     version: int
     status: int
@@ -184,6 +185,7 @@ class DatasetVersion:
     def parse(  # noqa: PLR0913
         cls: type[V],
         id: int,
+        uuid: str,
         dataset_id: int,
         version: int,
         status: int,
@@ -203,6 +205,7 @@ class DatasetVersion:
     ):
         return cls(
             id,
+            uuid,
             dataset_id,
             version,
             status,
@@ -306,6 +309,7 @@ class DatasetRecord:
         query_script: str,
         schema: str,
         version_id: int,
+        version_uuid: str,
         version_dataset_id: int,
         version: int,
         version_status: int,
@@ -331,6 +335,7 @@ class DatasetRecord:
 
         dataset_version = DatasetVersion.parse(
             version_id,
+            version_uuid,
             version_dataset_id,
             version,
             version_status,

{datachain-0.6.7 → datachain-0.6.9}/src/datachain/lib/dataset_info.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Optional, Union
+from uuid import uuid4
 
 from pydantic import Field, field_validator
 
@@ -15,6 +16,7 @@ if TYPE_CHECKING:
 
 class DatasetInfo(DataModel):
     name: str
+    uuid: str = Field(default=str(uuid4()))
     version: int = Field(default=1)
     status: int = Field(default=DatasetStatus.CREATED)
     created_at: datetime = Field(default=TIME_ZERO)
@@ -60,6 +62,7 @@ class DatasetInfo(DataModel):
         job: Optional[Job],
     ) -> "Self":
         return cls(
+            uuid=version.uuid,
             name=dataset.name,
             version=version.version,
             status=version.status,