datachain 0.14.2__py3-none-any.whl → 0.39.0__py3-none-any.whl

datachain/lib/dc/datasets.py

@@ -1,18 +1,19 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-)
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, get_origin, get_type_hints
 
-from datachain.lib.dataset_info import DatasetInfo
-from datachain.lib.file import (
-    File,
+from datachain.error import (
+    DatasetNotFoundError,
+    DatasetVersionNotFoundError,
+    ProjectNotFoundError,
 )
+from datachain.lib.dataset_info import DatasetInfo
+from datachain.lib.projects import get as get_project
 from datachain.lib.settings import Settings
 from datachain.lib.signal_schema import SignalSchema
 from datachain.query import Session
 from datachain.query.dataset import DatasetQuery
 
-from .utils import Sys
+from .utils import Sys, is_studio
 from .values import read_values
 
 if TYPE_CHECKING:
@@ -25,21 +26,64 @@ if TYPE_CHECKING:
 
 def read_dataset(
     name: str,
-    version: Optional[int] = None,
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
-    fallback_to_studio: bool = True,
+    namespace: str | None = None,
+    project: str | None = None,
+    version: str | int | None = None,
+    session: Session | None = None,
+    settings: dict | None = None,
+    delta: bool | None = False,
+    delta_on: str | Sequence[str] | None = (
+        "file.path",
+        "file.etag",
+        "file.version",
+    ),
+    delta_result_on: str | Sequence[str] | None = None,
+    delta_compare: str | Sequence[str] | None = None,
+    delta_retry: bool | str | None = None,
+    delta_unsafe: bool = False,
+    update: bool = False,
 ) -> "DataChain":
     """Get data from a saved Dataset. It returns the chain itself.
     If dataset or version is not found locally, it will try to pull it from Studio.
 
     Parameters:
-        name : dataset name
-        version : dataset version
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
-        fallback_to_studio : Try to pull dataset from Studio if not found locally.
-            Default is True.
+        name: The dataset name, which can be a fully qualified name including the
+            namespace and project. Alternatively, it can be a regular name, in which
+            case the explicitly defined namespace and project will be used if they are
+            set; otherwise, default values will be applied.
+        namespace: optional name of namespace in which dataset to read is created
+        project: optional name of project in which dataset to read is created
+        version: dataset version. Supports:
+            - Exact version strings: "1.2.3"
+            - Legacy integer versions: 1, 2, 3 (finds latest major version)
+            - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
+        delta: If True, only process new or changed files instead of reprocessing
+            everything. This saves time by skipping files that were already processed in
+            previous versions. The optimization is working when a new version of the
+            dataset is created.
+            Default is False.
+        delta_on: Field(s) that uniquely identify each record in the source data.
+            Used to detect which records are new or changed.
+            Default is ("file.path", "file.etag", "file.version").
+        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
+            Only needed if you rename the identifying fields during processing.
+            Default is None.
+        delta_compare: Field(s) used to detect if a record has changed.
+            If not specified, all fields except `delta_on` fields are used.
+            Default is None.
+        delta_retry: Controls retry behavior for failed records:
+            - String (field name): Reprocess records where this field is not empty
+              (error mode)
+            - True: Reprocess records missing from the result dataset (missing mode)
+            - None: No retry processing (default)
+        update: If True always checks for newer versions available on Studio, even if
+            some version of the dataset exists locally already. If False (default), it
+            will only fetch the dataset from Studio if it is not found locally.
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct.
+
 
     Example:
         ```py
@@ -48,11 +92,27 @@ def read_dataset(
         ```
 
         ```py
-        chain = dc.read_dataset("my_cats", fallback_to_studio=False)
+        import datachain as dc
+        chain = dc.read_dataset("dev.animals.my_cats")
+        ```
+
+        ```py
+        chain = dc.read_dataset("my_cats", version="1.0.0")
+        ```
+
+        ```py
+        # Using version specifiers (PEP 440)
+        chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")
+        ```
+
+        ```py
+        # Legacy integer version support (finds latest in major version)
+        chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version
         ```
 
         ```py
-        chain = dc.read_dataset("my_cats", version=1)
+        # Always check for newer versions matching a version specifier from Studio
+        chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)
         ```
 
         ```py
@@ -66,10 +126,9 @@ def read_dataset(
         }
         chain = dc.read_dataset(
             name="my_cats",
-            version=1,
+            version="1.0.0",
             session=session,
             settings=settings,
-            fallback_to_studio=True,
         )
         ```
     """
@@ -77,34 +136,96 @@ def read_dataset(
 
     from .datachain import DataChain
 
-    query = DatasetQuery(
-        name=name,
-        version=version,
-        session=session,
-        indexing_column_types=File._datachain_column_types,
-        fallback_to_studio=fallback_to_studio,
-    )
     telemetry.send_event_once("class", "datachain_init", name=name, version=version)
+
+    session = Session.get(session)
+    catalog = session.catalog
+
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
+    )
+
+    if version is not None:
+        dataset = session.catalog.get_dataset_with_remote_fallback(
+            name, namespace_name, project_name, update=update
+        )
+
+        # Convert legacy integer versions to version specifiers
+        # For backward compatibility we still allow users to put version as integer
+        # in which case we convert it to a version specifier that finds the latest
+        # version where major part is equal to that input version.
+        # For example if user sets version=2, we convert it to ">=2.0.0,<3.0.0"
+        # which will find something like 2.4.3 (assuming 2.4.3 is the biggest among
+        # all 2.* dataset versions)
+        if isinstance(version, int):
+            version_spec = f">={version}.0.0,<{version + 1}.0.0"
+        else:
+            version_spec = str(version)
+
+        from packaging.specifiers import InvalidSpecifier, SpecifierSet
+
+        try:
+            # Try to parse as version specifier
+            SpecifierSet(version_spec)
+            # If it's a valid specifier set, find the latest compatible version
+            latest_compatible = dataset.latest_compatible_version(version_spec)
+            if not latest_compatible:
+                raise DatasetVersionNotFoundError(
+                    f"No dataset {name} version matching specifier {version_spec}"
+                )
+            version = latest_compatible
+        except InvalidSpecifier:
+            # If not a valid specifier, treat as exact version string
+            # This handles cases like "1.2.3" which are exact versions, not specifiers
+            pass
+
     if settings:
         _settings = Settings(**settings)
     else:
         _settings = Settings()
 
+    query = DatasetQuery(
+        name=name,
+        project_name=project_name,
+        namespace_name=namespace_name,
+        version=version,  #  type: ignore[arg-type]
+        session=session,
+        update=update,
+    )
+
     signals_schema = SignalSchema({"sys": Sys})
     if query.feature_schema:
         signals_schema |= SignalSchema.deserialize(query.feature_schema)
     else:
         signals_schema |= SignalSchema.from_column_types(query.column_types or {})
-    return DataChain(query, _settings, signals_schema)
+
+    if delta:
+        signals_schema = signals_schema.clone_without_sys_signals()
+
+    chain = DataChain(query, _settings, signals_schema)
+
+    if delta:
+        chain = chain._as_delta(
+            on=delta_on,
+            right_on=delta_result_on,
+            compare=delta_compare,
+            delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
+        )
+
+    return chain
 
 
 def datasets(
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    session: Session | None = None,
+    settings: dict | None = None,
     in_memory: bool = False,
-    object_name: str = "dataset",
+    column: str | None = None,
     include_listing: bool = False,
     studio: bool = False,
+    attrs: list[str] | None = None,
 ) -> "DataChain":
     """Generate chain with list of registered datasets.
 
@@ -112,10 +233,15 @@ def datasets(
         session: Optional session instance. If not provided, uses default session.
         settings: Optional dictionary of settings to configure the chain.
         in_memory: If True, creates an in-memory session. Defaults to False.
-        object_name: Name of the output object in the chain. Defaults to "dataset".
+        column: Name of the output column in the chain. Defaults to None which
+            means no top level column will be created.
         include_listing: If True, includes listing datasets. Defaults to False.
         studio: If True, returns datasets from Studio only,
             otherwise returns all local datasets. Defaults to False.
+        attrs: Optional list of attributes to filter datasets on. It can be just
+            attribute without value e.g "NLP", or attribute with value
+            e.g "location=US". Attribute with value can also accept "*" to target
+            all that have specific name e.g "location=*"
 
     Returns:
         DataChain: A new DataChain instance containing dataset information.
@@ -124,8 +250,8 @@ def datasets(
         ```py
         import datachain as dc
 
-        chain = dc.datasets()
-        for ds in chain.collect("dataset"):
+        chain = dc.datasets(column="dataset")
+        for ds in chain.to_iter("dataset"):
             print(f"{ds.name}@v{ds.version}")
         ```
     """
@@ -139,11 +265,167 @@ def datasets(
             include_listing=include_listing, studio=studio
         )
     ]
+    datasets_values = [d for d in datasets_values if not d.is_temp]
+
+    if attrs:
+        for attr in attrs:
+            datasets_values = [d for d in datasets_values if d.has_attr(attr)]
+
+    if not column:
+        # flattening dataset fields
+        schema = {
+            k: get_origin(v) if get_origin(v) is dict else v
+            for k, v in get_type_hints(DatasetInfo).items()
+            if k in DatasetInfo.model_fields
+        }
+        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
+        for d in [d.model_dump() for d in datasets_values]:
+            for field, value in d.items():
+                data[field].append(value)
+
+        return read_values(
+            session=session,
+            settings=settings,
+            in_memory=in_memory,
+            output=schema,
+            **data,  # type: ignore[arg-type]
+        )
 
     return read_values(
         session=session,
         settings=settings,
         in_memory=in_memory,
-        output={object_name: DatasetInfo},
-        **{object_name: datasets_values},  # type: ignore[arg-type]
+        output={column: DatasetInfo},
+        **{column: datasets_values},  # type: ignore[arg-type]
+    )
+
+
+def delete_dataset(
+    name: str,
+    namespace: str | None = None,
+    project: str | None = None,
+    version: str | None = None,
+    force: bool | None = False,
+    studio: bool | None = False,
+    session: Session | None = None,
+    in_memory: bool = False,
+) -> None:
+    """Removes specific dataset version or all dataset versions, depending on
+    a force flag.
+
+    Args:
+        name: The dataset name, which can be a fully qualified name including the
+            namespace and project. Alternatively, it can be a regular name, in which
+            case the explicitly defined namespace and project will be used if they are
+            set; otherwise, default values will be applied.
+        namespace: optional name of namespace in which dataset to delete is created
+        project: optional name of project in which dataset to delete is created
+        version: Optional dataset version
+        force: If true, all datasets versions will be removed. Defaults to False.
+        studio: If True, removes dataset from Studio only, otherwise removes local
+            dataset. Defaults to False.
+        session: Optional session instance. If not provided, uses default session.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns: None
+
+    Example:
+        ```py
+        import datachain as dc
+        dc.delete_dataset("cats")
+        ```
+
+        ```py
+        import datachain as dc
+        dc.delete_dataset("cats", version="1.0.0")
+        ```
+    """
+    from datachain.studio import remove_studio_dataset
+
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
+    )
+
+    if not is_studio() and studio:
+        return remove_studio_dataset(
+            None, name, namespace_name, project_name, version=version, force=force
+        )
+
+    try:
+        ds_project = get_project(project_name, namespace_name, session=session)
+    except ProjectNotFoundError:
+        raise DatasetNotFoundError(
+            f"Dataset {name} not found in namespace {namespace_name} and project",
+            f" {project_name}",
+        ) from None
+
+    if not force:
+        version = (
+            version
+            or catalog.get_dataset(
+                name,
+                namespace_name=ds_project.namespace.name,
+                project_name=ds_project.name,
+            ).latest_version
+        )
+    else:
+        version = None
+    catalog.remove_dataset(name, ds_project, version=version, force=force)
+
+
+def move_dataset(
+    src: str,
+    dest: str,
+    session: Session | None = None,
+    in_memory: bool = False,
+) -> None:
+    """Moves an entire dataset between namespaces and projects.
+
+    Args:
+        src: The source dataset name. This can be a fully qualified name that includes
+            the namespace and project, or a regular name. If a regular name is used,
+            default values will be applied. The source dataset will no longer exist
+            after the move.
+        dest: The destination dataset name. This can also be a fully qualified
+            name with a namespace and project, or just a regular name (default values
+            will be used in that case). The original dataset will be moved here.
+        session: An optional session instance. If not provided, the default session
+            will be used.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns:
+        None
+
+    Examples:
+        ```python
+        import datachain as dc
+        dc.move_dataset("cats", "new_cats")
+        ```
+
+        ```python
+        import datachain as dc
+        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
+        ```
+    """
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+
+    namespace, project, name = catalog.get_full_dataset_name(src)
+    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)
+
+    dataset = catalog.get_dataset(name, namespace_name=namespace, project_name=project)
+
+    catalog.update_dataset(
+        dataset,
+        name=dest_name,
+        project_id=catalog.metastore.get_project(
+            dest_project,
+            dest_namespace,
+            create=is_studio(),
+        ).id,
     )

datachain/lib/dc/hf.py

@@ -1,8 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Any
 
 from datachain.lib.data_model import dict_to_data_model
 from datachain.query import Session
@@ -19,24 +15,29 @@ if TYPE_CHECKING:
 
 
 def read_hf(
-    dataset: Union[str, "HFDatasetType"],
-    *args,
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
-    object_name: str = "",
+    dataset: "HFDatasetType",
+    *args: Any,
+    session: Session | None = None,
+    settings: dict | None = None,
+    column: str = "",
     model_name: str = "",
-    **kwargs,
+    limit: int = 0,
+    **kwargs: Any,
 ) -> "DataChain":
-    """Generate chain from huggingface hub dataset.
+    """Generate chain from Hugging Face Hub dataset.
 
     Parameters:
-        dataset : Path or name of the dataset to read from Hugging Face Hub,
+        dataset: Path or name of the dataset to read from Hugging Face Hub,
             or an instance of `datasets.Dataset`-like object.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
-        object_name : Generated object column name.
-        model_name : Generated model name.
-        kwargs : Parameters to pass to datasets.load_dataset.
+        args: Additional positional arguments to pass to `datasets.load_dataset`.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
+        column: Generated object column name.
+        model_name: Generated model name.
+        limit: The maximum number of items to read from the HF dataset.
+            Applies `take(limit)` to `datasets.load_dataset`.
+            Defaults to 0 (no limit).
+        kwargs: Parameters to pass to `datasets.load_dataset`.
 
     Example:
         Load from Hugging Face Hub:
@@ -52,6 +53,18 @@ def read_hf(
         import datachain as dc
         chain = dc.read_hf(ds)
         ```
+
+        Streaming with limit, for large datasets:
+        ```py
+        import datachain as dc
+        ds = dc.read_hf("beans", split="train", streaming=True, limit=10)
+        ```
+
+        or use HF split syntax (not supported if streaming is enabled):
+        ```py
+        import datachain as dc
+        ds = dc.read_hf("beans", split="train[%10]")
+        ```
     """
     from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits
 
@@ -62,12 +75,13 @@ def read_hf(
     if len(ds_dict) > 1:
         output = {"split": str}
 
-    model_name = model_name or object_name or ""
+    model_name = model_name or column or ""
     hf_features = next(iter(ds_dict.values())).features
-    output = output | get_output_schema(hf_features)
-    model = dict_to_data_model(model_name, output)
-    if object_name:
-        output = {object_name: model}
+    hf_output, normalized_names = get_output_schema(hf_features, list(output.keys()))
+    output = output | hf_output
+    model = dict_to_data_model(model_name, output, list(normalized_names.values()))
+    if column:
+        output = {column: model}
 
     chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
-    return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)
+    return chain.gen(HFGenerator(dataset, model, limit, *args, **kwargs), output=output)

datachain/lib/dc/json.py

@@ -1,18 +1,12 @@
 import os
-import os.path
 import re
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING
 
+import cloudpickle
+
+from datachain.lib import meta_formats
 from datachain.lib.data_model import DataType
-from datachain.lib.file import (
-    File,
-    FileType,
-)
-from datachain.lib.meta_formats import read_meta
+from datachain.lib.file import File, FileType
 
 if TYPE_CHECKING:
     from typing_extensions import ParamSpec
@@ -23,30 +17,30 @@ if TYPE_CHECKING:
 
 
 def read_json(
-    path: Union[str, os.PathLike[str]],
+    path: str | os.PathLike[str],
     type: FileType = "text",
-    spec: Optional[DataType] = None,
-    schema_from: Optional[str] = "auto",
-    jmespath: Optional[str] = None,
-    object_name: Optional[str] = "",
-    model_name: Optional[str] = None,
-    format: Optional[str] = "json",
-    nrows=None,
+    spec: DataType | None = None,
+    schema_from: str | None = "auto",
+    jmespath: str | None = None,
+    column: str | None = "",
+    model_name: str | None = None,
+    format: str | None = "json",
+    nrows: int | None = None,
     **kwargs,
 ) -> "DataChain":
     """Get data from JSON. It returns the chain itself.
 
     Parameters:
-        path : storage URI with directory. URI must start with storage prefix such
+        path: storage URI with directory. URI must start with storage prefix such
             as `s3://`, `gs://`, `az://` or "file:///"
-        type : read file as "binary", "text", or "image" data. Default is "text".
-        spec : optional Data Model
-        schema_from : path to sample to infer spec (if schema not provided)
-        object_name : generated object column name
-        model_name : optional generated model name
+        type: read file as "binary", "text", or "image" data. Default is "text".
+        spec: optional Data Model
+        schema_from: path to sample to infer spec (if schema not provided)
+        column: generated column name
+        model_name: optional generated model name
         format: "json", "jsonl"
-        jmespath : optional JMESPATH expression to reduce JSON
-        nrows : optional row limit for jsonl and JSON arrays
+        jmespath: optional JMESPATH expression to reduce JSON
+        nrows: optional row limit for jsonl and JSON arrays
 
     Example:
         infer JSON schema from data, reduce using JMESPATH
@@ -70,13 +64,13 @@ def read_json(
         name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
         return s[:name_end]
 
-    if (not object_name) and jmespath:
-        object_name = jmespath_to_name(jmespath)
-    if not object_name:
-        object_name = format
+    if (not column) and jmespath:
+        column = jmespath_to_name(jmespath)
+    if not column:
+        column = format
     chain = read_storage(uri=path, type=type, **kwargs)
     signal_dict = {
-        object_name: read_meta(
+        column: meta_formats.read_meta(
             schema_from=schema_from,
             format=format,
             spec=spec,
@@ -88,4 +82,7 @@ def read_json(
     }
     # disable prefetch if nrows is set
     settings = {"prefetch": 0} if nrows else {}
+
+    cloudpickle.register_pickle_by_value(meta_formats)
+
     return chain.settings(**settings).gen(**signal_dict)  # type: ignore[misc, arg-type]