datachain 0.30.5__py3-none-any.whl → 0.39.0__py3-none-any.whl

datachain/lib/dc/datasets.py

@@ -1,5 +1,5 @@
 from collections.abc import Sequence
-from typing import TYPE_CHECKING, Optional, Union, get_origin, get_type_hints
+from typing import TYPE_CHECKING, get_origin, get_type_hints
 
 from datachain.error import (
     DatasetNotFoundError,
@@ -26,20 +26,21 @@ if TYPE_CHECKING:
 
 def read_dataset(
     name: str,
-    namespace: Optional[str] = None,
-    project: Optional[str] = None,
-    version: Optional[Union[str, int]] = None,
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
-    delta: Optional[bool] = False,
-    delta_on: Optional[Union[str, Sequence[str]]] = (
+    namespace: str | None = None,
+    project: str | None = None,
+    version: str | int | None = None,
+    session: Session | None = None,
+    settings: dict | None = None,
+    delta: bool | None = False,
+    delta_on: str | Sequence[str] | None = (
         "file.path",
         "file.etag",
         "file.version",
     ),
-    delta_result_on: Optional[Union[str, Sequence[str]]] = None,
-    delta_compare: Optional[Union[str, Sequence[str]]] = None,
-    delta_retry: Optional[Union[bool, str]] = None,
+    delta_result_on: str | Sequence[str] | None = None,
+    delta_compare: str | Sequence[str] | None = None,
+    delta_retry: bool | str | None = None,
+    delta_unsafe: bool = False,
     update: bool = False,
 ) -> "DataChain":
     """Get data from a saved Dataset. It returns the chain itself.
@@ -50,14 +51,14 @@ def read_dataset(
             namespace and project. Alternatively, it can be a regular name, in which
             case the explicitly defined namespace and project will be used if they are
             set; otherwise, default values will be applied.
-        namespace : optional name of namespace in which dataset to read is created
-        project : optional name of project in which dataset to read is created
-        version : dataset version. Supports:
+        namespace: optional name of namespace in which dataset to read is created
+        project: optional name of project in which dataset to read is created
+        version: dataset version. Supports:
             - Exact version strings: "1.2.3"
             - Legacy integer versions: 1, 2, 3 (finds latest major version)
             - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
         delta: If True, only process new or changed files instead of reprocessing
             everything. This saves time by skipping files that were already processed in
             previous versions. The optimization is working when a new version of the
@@ -80,6 +81,8 @@ def read_dataset(
         update: If True always checks for newer versions available on Studio, even if
             some version of the dataset exists locally already. If False (default), it
             will only fetch the dataset from Studio if it is not found locally.
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct.
 
 
     Example:
@@ -197,6 +200,10 @@ def read_dataset(
         signals_schema |= SignalSchema.deserialize(query.feature_schema)
     else:
         signals_schema |= SignalSchema.from_column_types(query.column_types or {})
+
+    if delta:
+        signals_schema = signals_schema.clone_without_sys_signals()
+
     chain = DataChain(query, _settings, signals_schema)
 
     if delta:
@@ -205,19 +212,20 @@ def read_dataset(
             right_on=delta_result_on,
             compare=delta_compare,
             delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
 
     return chain
 
 
 def datasets(
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    session: Session | None = None,
+    settings: dict | None = None,
     in_memory: bool = False,
-    column: Optional[str] = None,
+    column: str | None = None,
     include_listing: bool = False,
     studio: bool = False,
-    attrs: Optional[list[str]] = None,
+    attrs: list[str] | None = None,
 ) -> "DataChain":
     """Generate chain with list of registered datasets.
 
@@ -294,12 +302,12 @@ def datasets(
 
 def delete_dataset(
     name: str,
-    namespace: Optional[str] = None,
-    project: Optional[str] = None,
-    version: Optional[str] = None,
-    force: Optional[bool] = False,
-    studio: Optional[bool] = False,
-    session: Optional[Session] = None,
+    namespace: str | None = None,
+    project: str | None = None,
+    version: str | None = None,
+    force: bool | None = False,
+    studio: bool | None = False,
+    session: Session | None = None,
     in_memory: bool = False,
 ) -> None:
     """Removes specific dataset version or all dataset versions, depending on
@@ -310,9 +318,9 @@ def delete_dataset(
             namespace and project. Alternatively, it can be a regular name, in which
             case the explicitly defined namespace and project will be used if they are
             set; otherwise, default values will be applied.
-        namespace : optional name of namespace in which dataset to delete is created
-        project : optional name of project in which dataset to delete is created
-        version : Optional dataset version
+        namespace: optional name of namespace in which dataset to delete is created
+        project: optional name of project in which dataset to delete is created
+        version: Optional dataset version
         force: If true, all datasets versions will be removed. Defaults to False.
         studio: If True, removes dataset from Studio only, otherwise removes local
             dataset. Defaults to False.
@@ -373,7 +381,7 @@ def delete_dataset(
 def move_dataset(
     src: str,
     dest: str,
-    session: Optional[Session] = None,
+    session: Session | None = None,
     in_memory: bool = False,
 ) -> None:
     """Moves an entire dataset between namespaces and projects.

datachain/lib/dc/hf.py

@@ -1,8 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Any
 
 from datachain.lib.data_model import dict_to_data_model
 from datachain.query import Session
@@ -19,29 +15,29 @@ if TYPE_CHECKING:
 
 
 def read_hf(
-    dataset: Union[str, "HFDatasetType"],
-    *args,
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    dataset: "HFDatasetType",
+    *args: Any,
+    session: Session | None = None,
+    settings: dict | None = None,
     column: str = "",
     model_name: str = "",
     limit: int = 0,
-    **kwargs,
+    **kwargs: Any,
 ) -> "DataChain":
     """Generate chain from Hugging Face Hub dataset.
 
     Parameters:
-        dataset : Path or name of the dataset to read from Hugging Face Hub,
+        dataset: Path or name of the dataset to read from Hugging Face Hub,
             or an instance of `datasets.Dataset`-like object.
-        args : Additional positional arguments to pass to `datasets.load_dataset`.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
-        column : Generated object column name.
-        model_name : Generated model name.
-        limit : Limit the number of items to read from the HF dataset.
-                Adds `take(limit)` to the `datasets.load_dataset`.
-                Defaults to 0 (no limit).
-        kwargs : Parameters to pass to `datasets.load_dataset`.
+        args: Additional positional arguments to pass to `datasets.load_dataset`.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
+        column: Generated object column name.
+        model_name: Generated model name.
+        limit: The maximum number of items to read from the HF dataset.
+            Applies `take(limit)` to `datasets.load_dataset`.
+            Defaults to 0 (no limit).
+        kwargs: Parameters to pass to `datasets.load_dataset`.
 
     Example:
         Load from Hugging Face Hub:

datachain/lib/dc/json.py

@@ -1,7 +1,6 @@
 import os
-import os.path
 import re
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING
 
 import cloudpickle
 
@@ -18,30 +17,30 @@ if TYPE_CHECKING:
 
 
 def read_json(
-    path: Union[str, os.PathLike[str]],
+    path: str | os.PathLike[str],
     type: FileType = "text",
-    spec: Optional[DataType] = None,
-    schema_from: Optional[str] = "auto",
-    jmespath: Optional[str] = None,
-    column: Optional[str] = "",
-    model_name: Optional[str] = None,
-    format: Optional[str] = "json",
-    nrows=None,
+    spec: DataType | None = None,
+    schema_from: str | None = "auto",
+    jmespath: str | None = None,
+    column: str | None = "",
+    model_name: str | None = None,
+    format: str | None = "json",
+    nrows: int | None = None,
     **kwargs,
 ) -> "DataChain":
     """Get data from JSON. It returns the chain itself.
 
     Parameters:
-        path : storage URI with directory. URI must start with storage prefix such
+        path: storage URI with directory. URI must start with storage prefix such
             as `s3://`, `gs://`, `az://` or "file:///"
-        type : read file as "binary", "text", or "image" data. Default is "text".
-        spec : optional Data Model
-        schema_from : path to sample to infer spec (if schema not provided)
-        column : generated column name
-        model_name : optional generated model name
+        type: read file as "binary", "text", or "image" data. Default is "text".
+        spec: optional Data Model
+        schema_from: path to sample to infer spec (if schema not provided)
+        column: generated column name
+        model_name: optional generated model name
         format: "json", "jsonl"
-        jmespath : optional JMESPATH expression to reduce JSON
-        nrows : optional row limit for jsonl and JSON arrays
+        jmespath: optional JMESPATH expression to reduce JSON
+        nrows: optional row limit for jsonl and JSON arrays
 
     Example:
         infer JSON schema from data, reduce using JMESPATH

datachain/lib/dc/listings.py

@@ -1,7 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-)
+from typing import TYPE_CHECKING
 
 from datachain.lib.listing import LISTING_PREFIX, ls
 from datachain.lib.listing_info import ListingInfo
@@ -56,7 +53,7 @@ class ReadOnlyQueryStep(QueryStep):
 
 
 def listings(
-    session: Optional[Session] = None,
+    session: Session | None = None,
     in_memory: bool = False,
     column: str = "listing",
     **kwargs,
@@ -84,10 +81,10 @@ def listings(
 
 def read_listing_dataset(
     name: str,
-    version: Optional[str] = None,
+    version: str | None = None,
     path: str = "",
-    session: Optional["Session"] = None,
-    settings: Optional[dict] = None,
+    session: Session | None = None,
+    settings: dict | None = None,
 ) -> tuple["DataChain", "DatasetVersion"]:
     """Read a listing dataset and return a DataChain and listing version.
 

datachain/lib/dc/pandas.py

@@ -1,7 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-)
+from typing import TYPE_CHECKING
 
 from datachain.query import Session
 
@@ -19,8 +16,8 @@ if TYPE_CHECKING:
 def read_pandas(  # type: ignore[override]
     df: "pd.DataFrame",
     name: str = "",
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    session: Session | None = None,
+    settings: dict | None = None,
     in_memory: bool = False,
     column: str = "",
 ) -> "DataChain":

datachain/lib/dc/parquet.py

@@ -1,8 +1,5 @@
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Optional,
-)
+import os
+from typing import TYPE_CHECKING, Any
 
 from datachain.lib.data_model import DataType
 from datachain.query import Session
@@ -16,28 +13,34 @@ if TYPE_CHECKING:
 
 
 def read_parquet(
-    path,
+    path: str | os.PathLike[str] | list[str] | list[os.PathLike[str]],
     partitioning: Any = "hive",
-    output: Optional[dict[str, DataType]] = None,
+    output: dict[str, DataType] | None = None,
     column: str = "",
     model_name: str = "",
     source: bool = True,
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    session: Session | None = None,
+    settings: dict | None = None,
     **kwargs,
 ) -> "DataChain":
     """Generate chain from parquet files.
 
     Parameters:
-        path : Storage URI with directory. URI must start with storage prefix such
-            as `s3://`, `gs://`, `az://` or "file:///".
-        partitioning : Any pyarrow partitioning schema.
-        output : Dictionary defining column names and their corresponding types.
-        column : Created column name.
-        model_name : Generated model name.
-        source : Whether to include info about the source file.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
+        path: Storage path(s) or URI(s). Can be a local path or start with a
+            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
+            Supports glob patterns:
+              - `*` : wildcard
+              - `**` : recursive wildcard
+              - `?` : single character
+              - `{a,b}` : brace expansion list
+              - `{1..9}` : brace numeric or alphabetic range
+        partitioning: Any pyarrow partitioning schema.
+        output: Dictionary defining column names and their corresponding types.
+        column: Created column name.
+        model_name: Generated model name.
+        source: Whether to include info about the source file.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
 
     Example:
         Reading a single file:
@@ -46,10 +49,19 @@ def read_parquet(
         dc.read_parquet("s3://mybucket/file.parquet")
         ```
 
-        Reading a partitioned dataset from a directory:
+        All files from a directory:
         ```py
-        import datachain as dc
-        dc.read_parquet("s3://mybucket/dir")
+        dc.read_parquet("s3://mybucket/dir/")
+        ```
+
+        Only parquet files from a directory, and all it's subdirectories:
+        ```py
+        dc.read_parquet("s3://mybucket/dir/**/*.parquet")
+        ```
+
+        Using filename patterns - numeric, list, starting with zeros:
+        ```py
+        dc.read_parquet("s3://mybucket/202{1..4}/{yellow,green}-{01..12}.parquet")
         ```
     """
     from .storage import read_storage

datachain/lib/dc/records.py

@@ -1,5 +1,5 @@
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING
 
 import sqlalchemy
 
@@ -19,20 +19,20 @@ READ_RECORDS_BATCH_SIZE = 10000
 
 
 def read_records(
-    to_insert: Optional[Union[dict, Iterable[dict]]],
-    session: Optional[Session] = None,
-    settings: Optional[dict] = None,
+    to_insert: dict | Iterable[dict] | None,
+    session: Session | None = None,
+    settings: dict | None = None,
     in_memory: bool = False,
-    schema: Optional[dict[str, DataType]] = None,
+    schema: dict[str, DataType] | None = None,
 ) -> "DataChain":
     """Create a DataChain from the provided records. This method can be used for
     programmatically generating a chain in contrast of reading data from storages
     or other sources.
 
     Parameters:
-        to_insert : records (or a single record) to insert. Each record is
-                    a dictionary of signals and theirs values.
-        schema : describes chain signals and their corresponding types
+        to_insert: records (or a single record) to insert. Each record is
+                    a dictionary of signals and their values.
+        schema: describes chain signals and their corresponding types
 
     Example:
         ```py
@@ -45,7 +45,6 @@ def read_records(
     """
     from datachain.query.dataset import adjust_outputs, get_col_types
     from datachain.sql.types import SQLType
-    from datachain.utils import batched
 
     from .datasets import read_dataset
 
@@ -79,8 +78,6 @@ def read_records(
         ),
     )
 
-    session.add_dataset_version(dsr, dsr.latest_version)
-
     if isinstance(to_insert, dict):
         to_insert = [to_insert]
     elif not to_insert:
@@ -96,7 +93,6 @@ def read_records(
         {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
     )
     records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
-    for chunk in batched(records, READ_RECORDS_BATCH_SIZE):
-        warehouse.insert_rows(table, chunk)
+    warehouse.insert_rows(table, records, batch_size=READ_RECORDS_BATCH_SIZE)
     warehouse.insert_rows_done(table)
     return read_dataset(name=dsr.full_name, session=session, settings=settings)