datachain 0.31.0__py3-none-any.whl → 0.31.2__py3-none-any.whl

datachain/client/fsspec.py

@@ -44,6 +44,7 @@ FETCH_WORKERS = 100
 DELIMITER = "/"  # Path delimiter.
 
 DATA_SOURCE_URI_PATTERN = re.compile(r"^[\w]+:\/\/.*$")
+CLOUD_STORAGE_PROTOCOLS = {"s3", "gs", "az", "hf"}
 
 ResultQueue = asyncio.Queue[Optional[Sequence["File"]]]
 
@@ -62,6 +63,16 @@ def _is_win_local_path(uri: str) -> bool:
     return False
 
 
+def is_cloud_uri(uri: str) -> bool:
+    protocol = urlparse(uri).scheme
+    return protocol in CLOUD_STORAGE_PROTOCOLS
+
+
+def get_cloud_schemes() -> list[str]:
+    """Get list of cloud storage scheme prefixes."""
+    return [f"{p}://" for p in CLOUD_STORAGE_PROTOCOLS]
+
+
 class Bucket(NamedTuple):
     name: str
     uri: "StorageURI"

datachain/lib/clip.py

@@ -45,15 +45,15 @@ def clip_similarity_scores(
     Calculate CLIP similarity scores between one or more images and/or text.
 
     Parameters:
-        images : Images to use as inputs.
-        text : Text to use as inputs.
-        model : Model from clip or open_clip packages.
-        preprocess : Image preprocessor to apply.
-        tokenizer : Text tokenizer.
-        prob : Compute softmax probabilities.
-        image_to_text : Whether to compute for image-to-text or text-to-image. Ignored
-            if only one of images or text provided.
-        device : Device to use. Defaults is None - use model's device.
+        images: Images to use as inputs.
+        text: Text to use as inputs.
+        model: Model from clip or open_clip packages.
+        preprocess: Image preprocessor to apply.
+        tokenizer: Text tokenizer.
+        prob: Compute softmax probabilities.
+        image_to_text: Whether to compute for image-to-text or text-to-image. Ignored
+            if only one of the images or text provided.
+        device: Device to use. Default is None - use model's device.
 
 
     Example:

datachain/lib/dc/csv.py

@@ -1,10 +1,6 @@
+import os
 from collections.abc import Sequence
-from typing import (
-    TYPE_CHECKING,
-    Callable,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Callable, Optional, Union
 
 from datachain.lib.dc.utils import DatasetPrepareError, OutputType
 from datachain.lib.model_store import ModelStore
@@ -17,14 +13,14 @@ if TYPE_CHECKING:
 
 
 def read_csv(
-    path,
+    path: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
     delimiter: Optional[str] = None,
     header: bool = True,
     output: OutputType = None,
     column: str = "",
     model_name: str = "",
     source: bool = True,
-    nrows=None,
+    nrows: Optional[int] = None,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
@@ -34,21 +30,21 @@ def read_csv(
     """Generate chain from csv files.
 
     Parameters:
-        path : Storage URI with directory. URI must start with storage prefix such
+        path: Storage URI with directory. URI must start with storage prefix such
             as `s3://`, `gs://`, `az://` or "file:///".
-        delimiter : Character for delimiting columns. Takes precedence if also
+        delimiter: Character for delimiting columns. Takes precedence if also
             specified in `parse_options`. Defaults to ",".
-        header : Whether the files include a header row.
-        output : Dictionary or feature class defining column names and their
+        header: Whether the files include a header row.
+        output: Dictionary or feature class defining column names and their
             corresponding types. List of column names is also accepted, in which
             case types will be inferred.
-        column : Created column name.
-        model_name : Generated model name.
-        source : Whether to include info about the source file.
-        nrows : Optional row limit.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
-        column_types : Dictionary of column names and their corresponding types.
+        column: Created column name.
+        model_name: Generated model name.
+        source: Whether to include info about the source file.
+        nrows: Optional row limit.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
+        column_types: Dictionary of column names and their corresponding types.
             It is passed to CSV reader and for each column specified type auto
             inference is disabled.
         parse_options: Tells the parser how to process lines.

datachain/lib/dc/datachain.py

@@ -40,11 +40,7 @@ from datachain.lib.data_model import (
     StandardType,
     dict_to_data_model,
 )
-from datachain.lib.file import (
-    EXPORT_FILES_MAX_THREADS,
-    ArrowRow,
-    FileExporter,
-)
+from datachain.lib.file import EXPORT_FILES_MAX_THREADS, ArrowRow, FileExporter
 from datachain.lib.file import ExportPlacement as FileExportPlacement
 from datachain.lib.model_store import ModelStore
 from datachain.lib.settings import Settings
@@ -352,24 +348,28 @@ class DataChain:
         batch_size: Optional[int] = None,
         sys: Optional[bool] = None,
     ) -> "Self":
-        """Change settings for chain.
-
-        This function changes specified settings without changing not specified ones.
-        It returns chain, so, it can be chained later with next operation.
+        """
+        Set chain execution parameters. Returns the chain itself, allowing method
+        chaining for subsequent operations. To restore all settings to their default
+        values, use `reset_settings()`.
 
         Parameters:
-            cache : data caching. (default=False)
-            prefetch : number of workers to use for downloading files in advance.
-                      This is enabled by default and uses 2 workers.
-                      To disable prefetching, set it to 0 or False.
-            parallel : number of thread for processors. True is a special value to
-                enable all available CPUs. (default=1)
-            workers : number of distributed workers. Only for Studio mode. (default=1)
-            namespace : namespace name.
-            project : project name.
-            min_task_size : minimum number of tasks. (default=1)
-            batch_size : row limit per insert to balance speed and memory usage.
-                      (default=2000)
+            cache: Enable files caching to speed up subsequent accesses to the same
+                files from the same or different chains. Defaults to False.
+            prefetch: Enable prefetching of files. This will download files in
+                advance in parallel. If an integer is provided, it specifies the number
+                of files to prefetch concurrently for each process on each worker.
+                Defaults to 2. Set to 0 or False to disable prefetching.
+            parallel: Number of processes to use for processing user-defined functions
+                (UDFs) in parallel. If an integer is provided, it specifies the number
+                of CPUs to use. If True, all available CPUs are used. Defaults to 1.
+            namespace: Namespace to use for the chain by default.
+            project: Project to use for the chain by default.
+            min_task_size: Minimum number of rows per worker/process for parallel
+                processing by UDFs. Defaults to 1.
+            batch_size: Number of rows per insert by UDF to fine tune and balance speed
+                and memory usage. This might be useful when processing large rows
+                or when running into memory issues. Defaults to 2000.
 
         Example:
             ```py
@@ -398,7 +398,7 @@ class DataChain:
         return self._evolve(settings=settings, _sys=sys)
 
     def reset_settings(self, settings: Optional[Settings] = None) -> "Self":
-        """Reset all settings to default values."""
+        """Reset all chain settings to default values."""
         self._settings = settings if settings else Settings()
         return self
 
@@ -580,14 +580,14 @@ class DataChain:
         """Save to a Dataset. It returns the chain itself.
 
         Parameters:
-            name : dataset name. It can be full name consisting of namespace and
-                project, but it can also be just a regular dataset name in which
-                case we are taking namespace and project from settings, if they
-                are defined there, or default ones instead.
-            version : version of a dataset. If version is not specified and dataset
+            name: dataset name. This can be either a fully qualified name, including
+                the namespace and project, or just a regular dataset name. In the latter
+                case, the namespace and project will be taken from the settings
+                (if specified) or from the default values otherwise.
+            version: version of a dataset. If version is not specified and dataset
                 already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.
-            description : description of a dataset.
-            attrs : attributes of a dataset. They can be without value, e.g "NLP",
+            description: description of a dataset.
+            attrs: attributes of a dataset. They can be without value, e.g "NLP",
                 or with a value, e.g "location=US".
             update_version: which part of the dataset version to automatically increase.
                 Available values: `major`, `minor` or `patch`. Default is `patch`.
@@ -661,7 +661,9 @@ class DataChain:
                 # current latest version instead.
                 from .datasets import read_dataset
 
-                return read_dataset(name, **kwargs)
+                return read_dataset(
+                    name, namespace=namespace_name, project=project_name, **kwargs
+                )
 
         return self._evolve(
             query=self._query.save(
@@ -704,7 +706,7 @@ class DataChain:
         func: Optional[Callable] = None,
         params: Union[None, str, Sequence[str]] = None,
         output: OutputType = None,
-        **signal_map,
+        **signal_map: Any,
     ) -> "Self":
         """Apply a function to each row to create new signals. The function should
         return a new object for each row. It returns a chain itself with new signals.
@@ -712,17 +714,17 @@ class DataChain:
         Input-output relationship: 1:1
 
         Parameters:
-            func : Function applied to each row.
-            params : List of column names used as input for the function. Default
+            func: Function applied to each row.
+            params: List of column names used as input for the function. Default
                     is taken from function signature.
-            output : Dictionary defining new signals and their corresponding types.
+            output: Dictionary defining new signals and their corresponding types.
                     Default type is taken from function signature. Default can be also
                     taken from kwargs - **signal_map (see below).
                     If signal name is defined using signal_map (see below) only a single
                     type value can be used.
-            **signal_map : kwargs can be used to define `func` together with it's return
+            **signal_map: kwargs can be used to define `func` together with its return
                     signal name in format of `map(my_sign=my_func)`. This helps define
-                    signal names and function in a nicer way.
+                    signal names and functions in a nicer way.
 
         Example:
             Using signal_map and single type in output:
@@ -941,7 +943,7 @@ class DataChain:
         It accepts the same parameters plus an
         additional parameter:
 
-            batch : Size of each batch passed to `func`. Defaults to 1000.
+            batch: Size of each batch passed to `func`. Defaults to 1000.
 
         Example:
             ```py
@@ -1309,9 +1311,9 @@ class DataChain:
         """Yields flattened rows of values as a tuple.
 
         Args:
-            row_factory : A callable to convert row to a custom format.
-                          It should accept two arguments: a list of column names and
-                          a tuple of row values.
+            row_factory: A callable to convert row to a custom format.
+                It should accept two arguments: a list of column names and
+                a tuple of row values.
             include_hidden: Whether to include hidden signals from the schema.
         """
         db_signals = self._effective_signals_schema.db_signals(
@@ -1956,19 +1958,19 @@ class DataChain:
         model_name: str = "",
         source: bool = True,
         nrows: Optional[int] = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> "Self":
         """Generate chain from list of tabular files.
 
         Parameters:
-            output : Dictionary or feature class defining column names and their
+            output: Dictionary or feature class defining column names and their
                 corresponding types. List of column names is also accepted, in which
                 case types will be inferred.
-            column : Generated column name.
-            model_name : Generated model name.
-            source : Whether to include info about the source file.
-            nrows : Optional row limit.
-            kwargs : Parameters to pass to pyarrow.dataset.dataset.
+            column: Generated column name.
+            model_name: Generated model name.
+            source: Whether to include info about the source file.
+            nrows: Optional row limit.
+            kwargs: Parameters to pass to pyarrow.dataset.dataset.
 
         Example:
             Reading a json lines file:
@@ -2098,12 +2100,12 @@ class DataChain:
         """Save chain to parquet file with SignalSchema metadata.
 
         Parameters:
-            path : Path or a file-like binary object to save the file. This supports
+            path: Path or a file-like binary object to save the file. This supports
                 local paths as well as remote paths, such as s3:// or hf:// with fsspec.
-            partition_cols : Column names by which to partition the dataset.
-            chunk_size : The chunk size of results to read and convert to columnar
+            partition_cols: Column names by which to partition the dataset.
+            chunk_size: The chunk size of results to read and convert to columnar
                 data, to avoid running out of memory.
-            fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
+            fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
                 write, for fsspec-type URLs, such as s3:// or hf:// when
                 provided as the destination path.
         """
@@ -2195,10 +2197,10 @@ class DataChain:
         """Save chain to a csv (comma-separated values) file.
 
         Parameters:
-            path : Path to save the file. This supports local paths as well as
+            path: Path to save the file. This supports local paths as well as
                 remote paths, such as s3:// or hf:// with fsspec.
-            delimiter : Delimiter to use for the resulting file.
-            fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
+            delimiter: Delimiter to use for the resulting file.
+            fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
                 write, for fsspec-type URLs, such as s3:// or hf:// when
                 provided as the destination path.
         """
@@ -2241,12 +2243,12 @@ class DataChain:
         """Save chain to a JSON file.
 
         Parameters:
-            path : Path to save the file. This supports local paths as well as
+            path: Path to save the file. This supports local paths as well as
                 remote paths, such as s3:// or hf:// with fsspec.
-            fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
+            fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
                 write, for fsspec-type URLs, such as s3:// or hf:// when
                 provided as the destination path.
-            include_outer_list : Sets whether to include an outer list for all rows.
+            include_outer_list: Sets whether to include an outer list for all rows.
                 Setting this to True makes the file valid JSON, while False instead
                 writes in the JSON lines format.
         """
@@ -2301,9 +2303,9 @@ class DataChain:
         """Save chain to a JSON lines file.
 
         Parameters:
-            path : Path to save the file. This supports local paths as well as
+            path: Path to save the file. This supports local paths as well as
                 remote paths, such as s3:// or hf:// with fsspec.
-            fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
+            fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
                 write, for fsspec-type URLs, such as s3:// or hf:// when
                 provided as the destination path.
         """
@@ -2571,9 +2573,9 @@ class DataChain:
                 The possible values are: "filename", "etag", "fullpath", and "checksum".
             link_type: Method to use for exporting files.
                 Falls back to `'copy'` if symlinking fails.
-            num_threads : number of threads to use for exporting files.
-                By default it uses 5 threads.
-            anon: If True, we will treat cloud bucket as public one. Default behavior
+            num_threads: number of threads to use for exporting files.
+                By default, it uses 5 threads.
+            anon: If True, we will treat cloud bucket as a public one. Default behavior
                 depends on the previous session configuration (e.g. happens in the
                 initial `read_storage`) and particular cloud storage client
                 implementation (e.g. S3 fallbacks to anonymous access if no credentials

datachain/lib/dc/datasets.py

@@ -51,14 +51,14 @@ def read_dataset(
             namespace and project. Alternatively, it can be a regular name, in which
             case the explicitly defined namespace and project will be used if they are
             set; otherwise, default values will be applied.
-        namespace : optional name of namespace in which dataset to read is created
-        project : optional name of project in which dataset to read is created
-        version : dataset version. Supports:
+        namespace: optional name of namespace in which dataset to read is created
+        project: optional name of project in which dataset to read is created
+        version: dataset version. Supports:
             - Exact version strings: "1.2.3"
             - Legacy integer versions: 1, 2, 3 (finds latest major version)
             - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
         delta: If True, only process new or changed files instead of reprocessing
             everything. This saves time by skipping files that were already processed in
             previous versions. The optimization is working when a new version of the
@@ -314,9 +314,9 @@ def delete_dataset(
             namespace and project. Alternatively, it can be a regular name, in which
             case the explicitly defined namespace and project will be used if they are
             set; otherwise, default values will be applied.
-        namespace : optional name of namespace in which dataset to delete is created
-        project : optional name of project in which dataset to delete is created
-        version : Optional dataset version
+        namespace: optional name of namespace in which dataset to delete is created
+        project: optional name of project in which dataset to delete is created
+        version: Optional dataset version
         force: If true, all datasets versions will be removed. Defaults to False.
         studio: If True, removes dataset from Studio only, otherwise removes local
             dataset. Defaults to False.

datachain/lib/dc/hf.py

@@ -1,8 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 from datachain.lib.data_model import dict_to_data_model
 from datachain.query import Session
@@ -20,28 +16,28 @@ if TYPE_CHECKING:
 
 def read_hf(
     dataset: Union[str, "HFDatasetType"],
-    *args,
+    *args: Any,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     column: str = "",
     model_name: str = "",
     limit: int = 0,
-    **kwargs,
+    **kwargs: Any,
 ) -> "DataChain":
     """Generate chain from Hugging Face Hub dataset.
 
     Parameters:
-        dataset : Path or name of the dataset to read from Hugging Face Hub,
+        dataset: Path or name of the dataset to read from Hugging Face Hub,
             or an instance of `datasets.Dataset`-like object.
-        args : Additional positional arguments to pass to `datasets.load_dataset`.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
-        column : Generated object column name.
-        model_name : Generated model name.
-        limit : Limit the number of items to read from the HF dataset.
-                Adds `take(limit)` to the `datasets.load_dataset`.
-                Defaults to 0 (no limit).
-        kwargs : Parameters to pass to `datasets.load_dataset`.
+        args: Additional positional arguments to pass to `datasets.load_dataset`.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
+        column: Generated object column name.
+        model_name: Generated model name.
+        limit: The maximum number of items to read from the HF dataset.
+            Applies `take(limit)` to `datasets.load_dataset`.
+            Defaults to 0 (no limit).
+        kwargs: Parameters to pass to `datasets.load_dataset`.
 
     Example:
         Load from Hugging Face Hub:

datachain/lib/dc/json.py

@@ -26,22 +26,22 @@ def read_json(
     column: Optional[str] = "",
     model_name: Optional[str] = None,
     format: Optional[str] = "json",
-    nrows=None,
+    nrows: Optional[int] = None,
     **kwargs,
 ) -> "DataChain":
     """Get data from JSON. It returns the chain itself.
 
     Parameters:
-        path : storage URI with directory. URI must start with storage prefix such
+        path: storage URI with directory. URI must start with storage prefix such
             as `s3://`, `gs://`, `az://` or "file:///"
-        type : read file as "binary", "text", or "image" data. Default is "text".
-        spec : optional Data Model
-        schema_from : path to sample to infer spec (if schema not provided)
-        column : generated column name
-        model_name : optional generated model name
+        type: read file as "binary", "text", or "image" data. Default is "text".
+        spec: optional Data Model
+        schema_from: path to sample to infer spec (if schema not provided)
+        column: generated column name
+        model_name: optional generated model name
         format: "json", "jsonl"
-        jmespath : optional JMESPATH expression to reduce JSON
-        nrows : optional row limit for jsonl and JSON arrays
+        jmespath: optional JMESPATH expression to reduce JSON
+        nrows: optional row limit for jsonl and JSON arrays
 
     Example:
         infer JSON schema from data, reduce using JMESPATH

datachain/lib/dc/parquet.py

@@ -1,8 +1,5 @@
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Optional,
-)
+import os
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 from datachain.lib.data_model import DataType
 from datachain.query import Session
@@ -16,7 +13,7 @@ if TYPE_CHECKING:
 
 
 def read_parquet(
-    path,
+    path: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
     partitioning: Any = "hive",
     output: Optional[dict[str, DataType]] = None,
     column: str = "",
@@ -29,15 +26,15 @@ def read_parquet(
     """Generate chain from parquet files.
 
     Parameters:
-        path : Storage URI with directory. URI must start with storage prefix such
+        path: Storage URI with directory. URI must start with storage prefix such
             as `s3://`, `gs://`, `az://` or "file:///".
-        partitioning : Any pyarrow partitioning schema.
-        output : Dictionary defining column names and their corresponding types.
-        column : Created column name.
-        model_name : Generated model name.
-        source : Whether to include info about the source file.
-        session : Session to use for the chain.
-        settings : Settings to use for the chain.
+        partitioning: Any pyarrow partitioning schema.
+        output: Dictionary defining column names and their corresponding types.
+        column: Created column name.
+        model_name: Generated model name.
+        source: Whether to include info about the source file.
+        session: Session to use for the chain.
+        settings: Settings to use for the chain.
 
     Example:
         Reading a single file:

datachain/lib/dc/records.py

@@ -30,9 +30,9 @@ def read_records(
     or other sources.
 
     Parameters:
-        to_insert : records (or a single record) to insert. Each record is
+        to_insert: records (or a single record) to insert. Each record is
                     a dictionary of signals and their values.
-        schema : describes chain signals and their corresponding types
+        schema: describes chain signals and their corresponding types
 
     Example:
         ```py

datachain/lib/dc/storage.py

@@ -1,22 +1,17 @@
-import os.path
+import os
 from collections.abc import Sequence
 from functools import reduce
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Optional, Union
 
-from datachain.lib.file import (
-    FileType,
-    get_file_type,
-)
-from datachain.lib.listing import (
-    get_file_info,
-    get_listing,
-    list_bucket,
-    ls,
+from datachain.lib.dc.storage_pattern import (
+    apply_glob_filter,
+    expand_brace_pattern,
+    should_use_recursion,
+    split_uri_pattern,
+    validate_cloud_bucket_name,
 )
+from datachain.lib.file import FileType, get_file_type
+from datachain.lib.listing import get_file_info, get_listing, list_bucket, ls
 from datachain.query import Session
 
 if TYPE_CHECKING:
@@ -50,15 +45,19 @@ def read_storage(
     It returns the chain itself as usual.
 
     Parameters:
-        uri : storage URI with directory or list of URIs.
-            URIs must start with storage prefix such
-            as `s3://`, `gs://`, `az://` or "file:///"
-        type : read file as "binary", "text", or "image" data. Default is "binary".
-        recursive : search recursively for the given path.
-        column : Created column name.
-        update : force storage reindexing. Default is False.
-        anon : If True, we will treat cloud bucket as public one
-        client_config : Optional client configuration for the storage client.
+        uri: Storage path(s) or URI(s). Can be a local path or start with a
+            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
+            Supports glob patterns:
+              - `*` : wildcard
+              - `**` : recursive wildcard
+              - `?` : single character
+              - `{a,b}` : brace expansion
+        type: read file as "binary", "text", or "image" data. Default is "binary".
+        recursive: search recursively for the given path.
+        column: Column name that will contain File objects. Default is "file".
+        update: force storage reindexing. Default is False.
+        anon: If True, we will treat cloud bucket as public one.
+        client_config: Optional client configuration for the storage client.
         delta: If True, only process new or changed files instead of reprocessing
             everything. This saves time by skipping files that were already processed in
             previous versions. The optimization is working when a new version of the
@@ -92,12 +91,19 @@ def read_storage(
         chain = dc.read_storage("s3://my-bucket/my-dir")
         ```
 
+        Match all .json files recursively using glob pattern
+        ```py
+        chain = dc.read_storage("gs://bucket/meta/**/*.json")
+        ```
+
+        Match image file extensions for directories with pattern
+        ```py
+        chain = dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")
+        ```
+
         Multiple URIs:
         ```python
-        chain = dc.read_storage([
-            "s3://bucket1/dir1",
-            "s3://bucket2/dir2"
-        ])
+        chain = dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])
         ```
 
         With AWS S3-compatible storage:
@@ -107,19 +113,6 @@ def read_storage(
             client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
         )
         ```
-
-        Pass existing session
-        ```py
-        session = Session.get()
-        chain = dc.read_storage([
-            "path/to/dir1",
-            "path/to/dir2"
-        ], session=session, recursive=True)
-        ```
-
-    Note:
-        When using multiple URIs with `update=True`, the function optimizes by
-        avoiding redundant updates for URIs pointing to the same storage location.
     """
     from .datachain import DataChain
     from .datasets import read_dataset
@@ -142,13 +135,36 @@ def read_storage(
     if not uris:
         raise ValueError("No URIs provided")
 
+    # Then expand all URIs that contain brace patterns
+    expanded_uris = []
+    for single_uri in uris:
+        uri_str = str(single_uri)
+        validate_cloud_bucket_name(uri_str)
+        expanded_uris.extend(expand_brace_pattern(uri_str))
+
+    # Now process each expanded URI
     chains = []
     listed_ds_name = set()
     file_values = []
 
-    for single_uri in uris:
+    updated_uris = set()
+
+    for single_uri in expanded_uris:
+        # Check if URI contains glob patterns and split them
+        base_uri, glob_pattern = split_uri_pattern(single_uri)
+
+        # If a pattern is found, use the base_uri for listing
+        # The pattern will be used for filtering later
+        list_uri_to_use = base_uri if glob_pattern else single_uri
+
+        # Avoid double updates for the same URI
+        update_single_uri = False
+        if update and (list_uri_to_use not in updated_uris):
+            updated_uris.add(list_uri_to_use)
+            update_single_uri = True
+
         list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
-            single_uri, session, update=update
+            list_uri_to_use, session, update=update_single_uri
         )
 
         # list_ds_name is None if object is a file, we don't want to use cache
@@ -197,7 +213,21 @@ def read_storage(
                 lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
             )
 
-        chains.append(ls(dc, list_path, recursive=recursive, column=column))
+        # If a glob pattern was detected, use it for filtering
+        # Otherwise, use the original list_path from get_listing
+        if glob_pattern:
+            # Determine if we should use recursive listing based on the pattern
+            use_recursive = should_use_recursion(glob_pattern, recursive or False)
+
+            # Apply glob filter - no need for brace expansion here as it's done above
+            chain = apply_glob_filter(
+                dc, glob_pattern, list_path, use_recursive, column
+            )
+            chains.append(chain)
+        else:
+            # No glob pattern detected, use normal ls behavior
+            chains.append(ls(dc, list_path, recursive=recursive, column=column))
+
         listed_ds_name.add(list_ds_name)
 
     storage_chain = None if not chains else reduce(lambda x, y: x.union(y), chains)

datachain/lib/dc/storage_pattern.py

@@ -0,0 +1,300 @@
+import glob
+from typing import TYPE_CHECKING, Union
+
+from datachain.client.fsspec import is_cloud_uri
+from datachain.lib.listing import ls
+
+if TYPE_CHECKING:
+    from .datachain import DataChain
+
+
+def validate_cloud_bucket_name(uri: str) -> None:
+    """
+    Validate that cloud storage bucket names don't contain glob patterns.
+
+    Args:
+        uri: URI to validate
+
+    Raises:
+        ValueError: If a cloud storage bucket name contains glob patterns
+    """
+    if not is_cloud_uri(uri):
+        return
+
+    # Extract bucket name (everything between :// and first /)
+    if "://" in uri:
+        scheme_end = uri.index("://") + 3
+        path_part = uri[scheme_end:]
+
+        # Get the bucket name (first segment)
+        if "/" in path_part:
+            bucket_name = path_part.split("/")[0]
+        else:
+            bucket_name = path_part
+
+        # Check if bucket name contains glob patterns
+        glob_chars = ["*", "?", "[", "]", "{", "}"]
+        if any(char in bucket_name for char in glob_chars):
+            raise ValueError(f"Glob patterns in bucket names are not supported: {uri}")
+
+
+def split_uri_pattern(uri: str) -> tuple[str, Union[str, None]]:
+    """
+    Split a URI into base path and glob pattern.
+
+    Args:
+        uri: URI that may contain glob patterns (*, **, ?, {})
+
+    Returns:
+        Tuple of (base_uri, pattern) where pattern is None if no glob pattern found
+
+    Examples:
+        "s3://bucket/dir/*.mp3" -> ("s3://bucket/dir", "*.mp3")
+        "s3://bucket/**/*.mp3" -> ("s3://bucket", "**/*.mp3")
+        "s3://bucket/dir" -> ("s3://bucket/dir", None)
+    """
+    if not any(char in uri for char in ["*", "?", "[", "{", "}"]):
+        return uri, None
+
+    # Handle different URI schemes
+    if "://" in uri:
+        # Split into scheme and path
+        scheme_end = uri.index("://") + 3
+        scheme_part = uri[:scheme_end]
+        path_part = uri[scheme_end:]
+
+        # Find where the glob pattern starts
+        path_segments = path_part.split("/")
+
+        # Find first segment with glob pattern
+        pattern_start_idx = None
+        for i, segment in enumerate(path_segments):
+            # Check for glob patterns including brace expansion
+            if glob.has_magic(segment) or "{" in segment:
+                pattern_start_idx = i
+                break
+
+        if pattern_start_idx is None:
+            return uri, None
+
+        # Split into base and pattern
+        if pattern_start_idx == 0:
+            # Pattern at root of bucket
+            base = scheme_part + path_segments[0]
+            pattern = "/".join(path_segments[1:]) if len(path_segments) > 1 else "*"
+        else:
+            base = scheme_part + "/".join(path_segments[:pattern_start_idx])
+            pattern = "/".join(path_segments[pattern_start_idx:])
+
+        return base, pattern
+    # Local path
+    path_segments = uri.split("/")
+
+    # Find first segment with glob pattern
+    pattern_start_idx = None
+    for i, segment in enumerate(path_segments):
+        # Check for glob patterns including brace expansion
+        if glob.has_magic(segment) or "{" in segment:
+            pattern_start_idx = i
+            break
+
+    if pattern_start_idx is None:
+        return uri, None
+
+    # Split into base and pattern
+    base = "/".join(path_segments[:pattern_start_idx]) if pattern_start_idx > 0 else "/"
+    pattern = "/".join(path_segments[pattern_start_idx:])
+
+    return base, pattern
+
+
+def should_use_recursion(pattern: str, user_recursive: bool) -> bool:
+    """
+    Determine if we should use recursive listing based on the pattern.
+
+    Args:
+        pattern: The glob pattern extracted from URI
+        user_recursive: User's recursive preference
+
+    Returns:
+        True if recursive listing should be used
+
+    Examples:
+        "*" -> False (single level only)
+        "*.mp3" -> False (single level only)
+        "**/*.mp3" -> True (globstar requires recursion)
+        "dir/*/file.txt" -> True (multi-level pattern)
+    """
+    if not user_recursive:
+        # If user explicitly wants non-recursive, respect that
+        return False
+
+    # If pattern contains globstar, definitely need recursion
+    if "**" in pattern:
+        return True
+
+    # If pattern contains path separators, it needs recursion
+    # Single-level patterns like "*", "*.txt", "file?" should not be recursive
+    return "/" in pattern
+
+
+def expand_brace_pattern(pattern: str) -> list[str]:
+    """
+    Recursively expand brace patterns like *.{mp3,wav} into multiple glob patterns.
+    Handles nested and multiple brace patterns.
+
+    Args:
+        pattern: Pattern that may contain brace expansion
+
+    Returns:
+        List of expanded patterns
+
+    Examples:
+        "*.{mp3,wav}" -> ["*.mp3", "*.wav"]
+        "{a,b}/{c,d}" -> ["a/c", "a/d", "b/c", "b/d"]
+        "*.txt" -> ["*.txt"]
+        "{{a,b}}" -> ["{a}", "{b}"]  # Handle double braces
+    """
+    if "{" not in pattern or "}" not in pattern:
+        return [pattern]
+
+    return _expand_single_braces(pattern)
+
+
+def _expand_single_braces(pattern: str) -> list[str]:
+    """Helper to expand single-level braces."""
+    if "{" not in pattern or "}" not in pattern:
+        return [pattern]
+
+    # Find the first complete brace pattern
+    start = pattern.index("{")
+    end = start
+    depth = 0
+    for i in range(start, len(pattern)):
+        if pattern[i] == "{":
+            depth += 1
+        elif pattern[i] == "}":
+            depth -= 1
+            if depth == 0:
+                end = i
+                break
+
+    if start >= end:
+        return [pattern]
+
+    prefix = pattern[:start]
+    suffix = pattern[end + 1 :]
+    options = pattern[start + 1 : end].split(",")
+
+    # Generate all combinations and recursively expand
+    expanded = []
+    for option in options:
+        combined = prefix + option.strip() + suffix
+        # Recursively expand any remaining braces
+        expanded.extend(_expand_single_braces(combined))
+
+    return expanded
+
+
+def convert_globstar_to_glob(filter_pattern: str) -> str:
+    """Convert globstar patterns to GLOB patterns.
+
+    Standard GLOB doesn't understand ** as recursive wildcard,
+    so we need to convert patterns appropriately.
+
+    Args:
+        filter_pattern: Pattern that may contain globstars (**)
+
+    Returns:
+        GLOB-compatible pattern
+    """
+    if "**" not in filter_pattern:
+        return filter_pattern
+
+    parts = filter_pattern.split("/")
+    globstar_positions = [i for i, p in enumerate(parts) if p == "**"]
+
+    # Handle different cases based on number of globstars
+    num_globstars = len(globstar_positions)
+
+    if num_globstars <= 1:
+        # Special case: pattern like **/* means zero or more directories
+        # This is tricky because GLOB can't express "zero or more"
+        # We need different handling based on the pattern structure
+
+        if filter_pattern == "**/*":
+            # Match everything
+            return "*"
+        if filter_pattern.startswith("**/"):
+            remaining = filter_pattern[3:]
+            if "/" not in remaining:
+                # Pattern like **/*.ext or **/temp?.*
+                # The ** means zero or more directories
+                # For zero directories: pattern should be just the filename pattern
+                # For one or more: pattern should be */filename
+                # Since we can't OR in GLOB, we choose the more permissive option
+                # that works with recursive listing
+                # Special handling: if it's a simple extension pattern, match broadly
+                if remaining.startswith("*."):
+                    # Pattern like **/*.ext - match any file with this extension
+                    # This matches *.ext at current level and deeper with recursion:
+                    return remaining
+                # Pattern like **/temp?.* - match as filename in subdirs
+                return f"*/{remaining}"
+
+        # Default: Zero or one globstar - simple replacement
+        return filter_pattern.replace("**", "*")
+
+    # Multiple globstars - need more careful handling
+    # For patterns like **/level?/backup/**/*.ext
+    # We want to match any path containing /level?/backup/ and ending with .ext
+
+    # Find middle directories (between first and last **)
+    middle_parts = []
+    start_idx = globstar_positions[0] + 1
+    end_idx = globstar_positions[-1]
+    for i in range(start_idx, end_idx):
+        if parts[i] != "**":
+            middle_parts.append(parts[i])
+
+    if not middle_parts:
+        # No fixed middle parts, just use wildcards
+        result = filter_pattern.replace("**", "*")
+    else:
+        # Create pattern that matches the middle parts
+        middle_pattern = "/".join(middle_parts)
+        # Get the file pattern at the end if any
+        last_part = parts[-1] if parts[-1] != "**" else "*"
+
+        # Match any path containing this pattern
+        if last_part != "*":
+            # Has specific file pattern
+            result = f"*{middle_pattern}*{last_part}"
+        else:
+            result = f"*{middle_pattern}*"
+
+    return result
+
+
+def apply_glob_filter(
+    dc: "DataChain",
+    pattern: str,
+    list_path: str,
+    use_recursive: bool,
+    column: str,
+) -> "DataChain":
+    from datachain.query.schema import Column
+
+    chain = ls(dc, list_path, recursive=use_recursive, column=column)
+
+    # If pattern doesn't contain path separator and list_path is not empty,
+    # prepend the list_path to make the pattern match correctly
+    if list_path and "/" not in pattern:
+        filter_pattern = f"{list_path.rstrip('/')}/{pattern}"
+    else:
+        filter_pattern = pattern
+
+    # Convert globstar patterns to GLOB-compatible patterns
+    glob_pattern = convert_globstar_to_glob(filter_pattern)
+
+    return chain.filter(Column(f"{column}.path").glob(glob_pattern))

{datachain-0.31.0.dist-info → datachain-0.31.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.0
+Version: 0.31.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.0.dist-info → datachain-0.31.2.dist-info}/RECORD

@@ -41,7 +41,7 @@ datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
 datachain/client/azure.py,sha256=7yyAgANHfu9Kfh187MKNTT1guvu9Q-WYsi4vYoY3aew,3270
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=kb_myMWcgGFClY5Rsv6fvHIRblg41dfH5knHJuDbW6w,14015
+datachain/client/fsspec.py,sha256=sChjxu931QgU2-n9MdXlmOrhGAiAckXoDVZTxKcNv6M,14336
 datachain/client/gcs.py,sha256=8hcFhEHp8qGRsJoyfCoawfuwb1Et-MSkyQoM9AnNuXI,5204
 datachain/client/hf.py,sha256=n5xJZdvNLS-SqokxuBCIPfGbhIeC_XfLm_BNYtEVvg4,2677
 datachain/client/local.py,sha256=0J52Wzvw25hSucVlzBvLuMRAZwrAHZAYDvD1mNBqf4c,4607
@@ -72,7 +72,7 @@ datachain/func/window.py,sha256=ImyRpc1QI8QUSPO7KdD60e_DPVo7Ja0G5kcm6BlyMcw,1584
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/arrow.py,sha256=aedsosbFNjIBa6LQIxR2zhIVcA4pVw1p5hCVmrDhWsQ,10781
 datachain/lib/audio.py,sha256=fQmIBq-9hrUZtkgeJdPHYA_D8Wfe9D4cQZk4_ijxpNc,7580
-datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
+datachain/lib/clip.py,sha256=ae6uoiymOl53rBXwIfqJkbHrk_IA21R1uJwXo5454C4,6145
 datachain/lib/data_model.py,sha256=Rjah76GHwIV6AZQk4rsdg6JLre5D8Kb9T4PS5SXzsPA,3740
 datachain/lib/dataset_info.py,sha256=7w-DoKOyIVoOtWGCgciMLcP5CiAWJB3rVI-vUDF80k0,3311
 datachain/lib/file.py,sha256=IGwpCwjsSOpZXlRsatcMKToMmuvYiX6_UtaTjUKAAdg,44511
@@ -102,17 +102,18 @@ datachain/lib/convert/sql_to_python.py,sha256=Gxc4FylWC_Pvvuawuc2MKZIiuAWI7wje8p
 datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
 datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUOzHUGPoyZXAB0,4360
 datachain/lib/dc/__init__.py,sha256=UrUzmDH6YyVl8fxM5iXTSFtl5DZTUzEYm1MaazK4vdQ,900
-datachain/lib/dc/csv.py,sha256=q6a9BpapGwP6nwy6c5cklxQumep2fUp9l2LAjtTJr6s,4411
+datachain/lib/dc/csv.py,sha256=wUsDPpLD4lts92yn0gejZHqTv8qQBbv8JYRwiIepj0o,4471
 datachain/lib/dc/database.py,sha256=sTpos1rE4BS5BTzzixykhWIO2JxVYKH1GTRncdpu4dU,14716
-datachain/lib/dc/datachain.py,sha256=AtsvBndqMyKrfW4yH8V0Nf__hfR0LN-NpA2munzfiPM,99888
-datachain/lib/dc/datasets.py,sha256=-Bvyyu4XXDXLiWa-bOnsp0Q11RSYXRO0j5DaX8ShaFs,15355
-datachain/lib/dc/hf.py,sha256=AP_MUHg6HJWae10PN9hD_beQVjrl0cleZ6Cvhtl1yoI,2901
-datachain/lib/dc/json.py,sha256=dNijfJ-H92vU3soyR7X1IiDrWhm6yZIGG3bSnZkPdAE,2733
+datachain/lib/dc/datachain.py,sha256=pDgUmvmf0ENngFepoD0AkxxqiqNIgoRueejfojyuURQ,100458
+datachain/lib/dc/datasets.py,sha256=pVRcrVEPVPHMf8sLqqhjXbilB3QuUqKE-byvZ-XlJNE,15347
+datachain/lib/dc/hf.py,sha256=B7pubDQTDmth9uILXyhpQNtOAT3UOLjR-peU__tpypk,2884
+datachain/lib/dc/json.py,sha256=-vJ-pUpp2JxK4_vOfznE09FIoEOrvCwoIZSLxM6pjmY,2742
 datachain/lib/dc/listings.py,sha256=V379Cb-7ZyquM0w7sWArQZkzInZy4GB7QQ1ZfowKzQY,4544
 datachain/lib/dc/pandas.py,sha256=ObueUXDUFKJGu380GmazdG02ARpKAHPhSaymfmOH13E,1489
-datachain/lib/dc/parquet.py,sha256=zYcSgrWwyEDW9UxGUSVdIVsCu15IGEf0xL8KfWQqK94,1782
-datachain/lib/dc/records.py,sha256=IKf5MArify-cI1P4NgbIvrAi0UQ5cvofTI3u6_zKBP8,3069
-datachain/lib/dc/storage.py,sha256=OMJE-9ob9Ku5le8W6O8J1W-XJ0pwHt2PsO-ZCcee1ZA,7950
+datachain/lib/dc/parquet.py,sha256=STgm19AM-etu7WmOUMJa5Z9GI6tPC-A0P3JO3ulfsKo,1839
+datachain/lib/dc/records.py,sha256=l7TKSKjT6boXGd05KA5vvax-Y-mLMOo46VWrlxPhmdQ,3067
+datachain/lib/dc/storage.py,sha256=pydeiGLMsmDvruVY_bC5GsV6VLpYpRf7szrD0S2pTmE,9688
+datachain/lib/dc/storage_pattern.py,sha256=QDLLSuBd1mdfkdRi3srGXXigs7rHw3vAnQedjE01_H8,9779
 datachain/lib/dc/utils.py,sha256=9OMiFu2kXIbtMqzJTEr1qbCoCBGpOmTnkWImVgFTKgo,4112
 datachain/lib/dc/values.py,sha256=7l1n352xWrEdql2NhBcZ3hj8xyPglWiY4qHjFPjn6iw,1428
 datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
@@ -160,9 +161,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.31.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.31.0.dist-info/METADATA,sha256=hY_KVFdUHZmZcxRiy5e-GY6CXI-sY0oKAtrvNakApdY,13898
-datachain-0.31.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.31.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.31.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.31.0.dist-info/RECORD,,
+datachain-0.31.2.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.31.2.dist-info/METADATA,sha256=ALo4Vp6w2VSanACVy1xv6aHWzbdasSKzD2U8_SybXBU,13898
+datachain-0.31.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.31.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.31.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.31.2.dist-info/RECORD,,