datachain 0.31.1__py3-none-any.whl → 0.31.2__py3-none-any.whl

datachain/client/fsspec.py

@@ -44,6 +44,7 @@ FETCH_WORKERS = 100
 DELIMITER = "/"  # Path delimiter.
 
 DATA_SOURCE_URI_PATTERN = re.compile(r"^[\w]+:\/\/.*$")
+CLOUD_STORAGE_PROTOCOLS = {"s3", "gs", "az", "hf"}
 
 ResultQueue = asyncio.Queue[Optional[Sequence["File"]]]
 
@@ -62,6 +63,16 @@ def _is_win_local_path(uri: str) -> bool:
     return False
 
 
+def is_cloud_uri(uri: str) -> bool:
+    protocol = urlparse(uri).scheme
+    return protocol in CLOUD_STORAGE_PROTOCOLS
+
+
+def get_cloud_schemes() -> list[str]:
+    """Get list of cloud storage scheme prefixes."""
+    return [f"{p}://" for p in CLOUD_STORAGE_PROTOCOLS]
+
+
 class Bucket(NamedTuple):
     name: str
     uri: "StorageURI"

datachain/lib/dc/storage.py

@@ -3,6 +3,13 @@ from collections.abc import Sequence
 from functools import reduce
 from typing import TYPE_CHECKING, Optional, Union
 
+from datachain.lib.dc.storage_pattern import (
+    apply_glob_filter,
+    expand_brace_pattern,
+    should_use_recursion,
+    split_uri_pattern,
+    validate_cloud_bucket_name,
+)
 from datachain.lib.file import FileType, get_file_type
 from datachain.lib.listing import get_file_info, get_listing, list_bucket, ls
 from datachain.query import Session
@@ -38,14 +45,18 @@ def read_storage(
     It returns the chain itself as usual.
 
     Parameters:
-        uri: storage URI with directory or list of URIs.
-            URIs must start with storage prefix such
-            as `s3://`, `gs://`, `az://` or "file:///"
+        uri: Storage path(s) or URI(s). Can be a local path or start with a
+            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
+            Supports glob patterns:
+              - `*` : wildcard
+              - `**` : recursive wildcard
+              - `?` : single character
+              - `{a,b}` : brace expansion
         type: read file as "binary", "text", or "image" data. Default is "binary".
         recursive: search recursively for the given path.
-        column: Created column name.
+        column: Column name that will contain File objects. Default is "file".
         update: force storage reindexing. Default is False.
-        anon: If True, we will treat cloud bucket as public one
+        anon: If True, we will treat cloud bucket as public one.
         client_config: Optional client configuration for the storage client.
         delta: If True, only process new or changed files instead of reprocessing
             everything. This saves time by skipping files that were already processed in
@@ -80,12 +91,19 @@ def read_storage(
         chain = dc.read_storage("s3://my-bucket/my-dir")
         ```
 
+        Match all .json files recursively using glob pattern
+        ```py
+        chain = dc.read_storage("gs://bucket/meta/**/*.json")
+        ```
+
+        Match image file extensions for directories with pattern
+        ```py
+        chain = dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")
+        ```
+
         Multiple URIs:
         ```python
-        chain = dc.read_storage([
-            "s3://bucket1/dir1",
-            "s3://bucket2/dir2"
-        ])
+        chain = dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])
         ```
 
         With AWS S3-compatible storage:
@@ -95,19 +113,6 @@ def read_storage(
             client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
         )
         ```
-
-        Pass existing session
-        ```py
-        session = Session.get()
-        chain = dc.read_storage([
-            "path/to/dir1",
-            "path/to/dir2"
-        ], session=session, recursive=True)
-        ```
-
-    Note:
-        When using multiple URIs with `update=True`, the function optimizes by
-        avoiding redundant updates for URIs pointing to the same storage location.
     """
     from .datachain import DataChain
     from .datasets import read_dataset
@@ -130,13 +135,36 @@ def read_storage(
     if not uris:
         raise ValueError("No URIs provided")
 
+    # Then expand all URIs that contain brace patterns
+    expanded_uris = []
+    for single_uri in uris:
+        uri_str = str(single_uri)
+        validate_cloud_bucket_name(uri_str)
+        expanded_uris.extend(expand_brace_pattern(uri_str))
+
+    # Now process each expanded URI
     chains = []
     listed_ds_name = set()
     file_values = []
 
-    for single_uri in uris:
+    updated_uris = set()
+
+    for single_uri in expanded_uris:
+        # Check if URI contains glob patterns and split them
+        base_uri, glob_pattern = split_uri_pattern(single_uri)
+
+        # If a pattern is found, use the base_uri for listing
+        # The pattern will be used for filtering later
+        list_uri_to_use = base_uri if glob_pattern else single_uri
+
+        # Avoid double updates for the same URI
+        update_single_uri = False
+        if update and (list_uri_to_use not in updated_uris):
+            updated_uris.add(list_uri_to_use)
+            update_single_uri = True
+
         list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
-            single_uri, session, update=update
+            list_uri_to_use, session, update=update_single_uri
         )
 
         # list_ds_name is None if object is a file, we don't want to use cache
@@ -185,7 +213,21 @@ def read_storage(
                 lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
             )
 
-        chains.append(ls(dc, list_path, recursive=recursive, column=column))
+        # If a glob pattern was detected, use it for filtering
+        # Otherwise, use the original list_path from get_listing
+        if glob_pattern:
+            # Determine if we should use recursive listing based on the pattern
+            use_recursive = should_use_recursion(glob_pattern, recursive or False)
+
+            # Apply glob filter - no need for brace expansion here as it's done above
+            chain = apply_glob_filter(
+                dc, glob_pattern, list_path, use_recursive, column
+            )
+            chains.append(chain)
+        else:
+            # No glob pattern detected, use normal ls behavior
+            chains.append(ls(dc, list_path, recursive=recursive, column=column))
+
         listed_ds_name.add(list_ds_name)
 
     storage_chain = None if not chains else reduce(lambda x, y: x.union(y), chains)

datachain/lib/dc/storage_pattern.py

@@ -0,0 +1,300 @@
+import glob
+from typing import TYPE_CHECKING, Union
+
+from datachain.client.fsspec import is_cloud_uri
+from datachain.lib.listing import ls
+
+if TYPE_CHECKING:
+    from .datachain import DataChain
+
+
+def validate_cloud_bucket_name(uri: str) -> None:
+    """
+    Validate that cloud storage bucket names don't contain glob patterns.
+
+    Args:
+        uri: URI to validate
+
+    Raises:
+        ValueError: If a cloud storage bucket name contains glob patterns
+    """
+    if not is_cloud_uri(uri):
+        return
+
+    # Extract bucket name (everything between :// and first /)
+    if "://" in uri:
+        scheme_end = uri.index("://") + 3
+        path_part = uri[scheme_end:]
+
+        # Get the bucket name (first segment)
+        if "/" in path_part:
+            bucket_name = path_part.split("/")[0]
+        else:
+            bucket_name = path_part
+
+        # Check if bucket name contains glob patterns
+        glob_chars = ["*", "?", "[", "]", "{", "}"]
+        if any(char in bucket_name for char in glob_chars):
+            raise ValueError(f"Glob patterns in bucket names are not supported: {uri}")
+
+
+def split_uri_pattern(uri: str) -> tuple[str, Union[str, None]]:
+    """
+    Split a URI into base path and glob pattern.
+
+    Args:
+        uri: URI that may contain glob patterns (*, **, ?, {})
+
+    Returns:
+        Tuple of (base_uri, pattern) where pattern is None if no glob pattern found
+
+    Examples:
+        "s3://bucket/dir/*.mp3" -> ("s3://bucket/dir", "*.mp3")
+        "s3://bucket/**/*.mp3" -> ("s3://bucket", "**/*.mp3")
+        "s3://bucket/dir" -> ("s3://bucket/dir", None)
+    """
+    if not any(char in uri for char in ["*", "?", "[", "{", "}"]):
+        return uri, None
+
+    # Handle different URI schemes
+    if "://" in uri:
+        # Split into scheme and path
+        scheme_end = uri.index("://") + 3
+        scheme_part = uri[:scheme_end]
+        path_part = uri[scheme_end:]
+
+        # Find where the glob pattern starts
+        path_segments = path_part.split("/")
+
+        # Find first segment with glob pattern
+        pattern_start_idx = None
+        for i, segment in enumerate(path_segments):
+            # Check for glob patterns including brace expansion
+            if glob.has_magic(segment) or "{" in segment:
+                pattern_start_idx = i
+                break
+
+        if pattern_start_idx is None:
+            return uri, None
+
+        # Split into base and pattern
+        if pattern_start_idx == 0:
+            # Pattern at root of bucket
+            base = scheme_part + path_segments[0]
+            pattern = "/".join(path_segments[1:]) if len(path_segments) > 1 else "*"
+        else:
+            base = scheme_part + "/".join(path_segments[:pattern_start_idx])
+            pattern = "/".join(path_segments[pattern_start_idx:])
+
+        return base, pattern
+    # Local path
+    path_segments = uri.split("/")
+
+    # Find first segment with glob pattern
+    pattern_start_idx = None
+    for i, segment in enumerate(path_segments):
+        # Check for glob patterns including brace expansion
+        if glob.has_magic(segment) or "{" in segment:
+            pattern_start_idx = i
+            break
+
+    if pattern_start_idx is None:
+        return uri, None
+
+    # Split into base and pattern
+    base = "/".join(path_segments[:pattern_start_idx]) if pattern_start_idx > 0 else "/"
+    pattern = "/".join(path_segments[pattern_start_idx:])
+
+    return base, pattern
+
+
+def should_use_recursion(pattern: str, user_recursive: bool) -> bool:
+    """
+    Determine if we should use recursive listing based on the pattern.
+
+    Args:
+        pattern: The glob pattern extracted from URI
+        user_recursive: User's recursive preference
+
+    Returns:
+        True if recursive listing should be used
+
+    Examples:
+        "*" -> False (single level only)
+        "*.mp3" -> False (single level only)
+        "**/*.mp3" -> True (globstar requires recursion)
+        "dir/*/file.txt" -> True (multi-level pattern)
+    """
+    if not user_recursive:
+        # If user explicitly wants non-recursive, respect that
+        return False
+
+    # If pattern contains globstar, definitely need recursion
+    if "**" in pattern:
+        return True
+
+    # If pattern contains path separators, it needs recursion
+    # Single-level patterns like "*", "*.txt", "file?" should not be recursive
+    return "/" in pattern
+
+
+def expand_brace_pattern(pattern: str) -> list[str]:
+    """
+    Recursively expand brace patterns like *.{mp3,wav} into multiple glob patterns.
+    Handles nested and multiple brace patterns.
+
+    Args:
+        pattern: Pattern that may contain brace expansion
+
+    Returns:
+        List of expanded patterns
+
+    Examples:
+        "*.{mp3,wav}" -> ["*.mp3", "*.wav"]
+        "{a,b}/{c,d}" -> ["a/c", "a/d", "b/c", "b/d"]
+        "*.txt" -> ["*.txt"]
+        "{{a,b}}" -> ["{a}", "{b}"]  # Handle double braces
+    """
+    if "{" not in pattern or "}" not in pattern:
+        return [pattern]
+
+    return _expand_single_braces(pattern)
+
+
+def _expand_single_braces(pattern: str) -> list[str]:
+    """Helper to expand single-level braces."""
+    if "{" not in pattern or "}" not in pattern:
+        return [pattern]
+
+    # Find the first complete brace pattern
+    start = pattern.index("{")
+    end = start
+    depth = 0
+    for i in range(start, len(pattern)):
+        if pattern[i] == "{":
+            depth += 1
+        elif pattern[i] == "}":
+            depth -= 1
+            if depth == 0:
+                end = i
+                break
+
+    if start >= end:
+        return [pattern]
+
+    prefix = pattern[:start]
+    suffix = pattern[end + 1 :]
+    options = pattern[start + 1 : end].split(",")
+
+    # Generate all combinations and recursively expand
+    expanded = []
+    for option in options:
+        combined = prefix + option.strip() + suffix
+        # Recursively expand any remaining braces
+        expanded.extend(_expand_single_braces(combined))
+
+    return expanded
+
+
+def convert_globstar_to_glob(filter_pattern: str) -> str:
+    """Convert globstar patterns to GLOB patterns.
+
+    Standard GLOB doesn't understand ** as recursive wildcard,
+    so we need to convert patterns appropriately.
+
+    Args:
+        filter_pattern: Pattern that may contain globstars (**)
+
+    Returns:
+        GLOB-compatible pattern
+    """
+    if "**" not in filter_pattern:
+        return filter_pattern
+
+    parts = filter_pattern.split("/")
+    globstar_positions = [i for i, p in enumerate(parts) if p == "**"]
+
+    # Handle different cases based on number of globstars
+    num_globstars = len(globstar_positions)
+
+    if num_globstars <= 1:
+        # Special case: pattern like **/* means zero or more directories
+        # This is tricky because GLOB can't express "zero or more"
+        # We need different handling based on the pattern structure
+
+        if filter_pattern == "**/*":
+            # Match everything
+            return "*"
+        if filter_pattern.startswith("**/"):
+            remaining = filter_pattern[3:]
+            if "/" not in remaining:
+                # Pattern like **/*.ext or **/temp?.*
+                # The ** means zero or more directories
+                # For zero directories: pattern should be just the filename pattern
+                # For one or more: pattern should be */filename
+                # Since we can't OR in GLOB, we choose the more permissive option
+                # that works with recursive listing
+                # Special handling: if it's a simple extension pattern, match broadly
+                if remaining.startswith("*."):
+                    # Pattern like **/*.ext - match any file with this extension
+                    # This matches *.ext at current level and deeper with recursion:
+                    return remaining
+                # Pattern like **/temp?.* - match as filename in subdirs
+                return f"*/{remaining}"
+
+        # Default: Zero or one globstar - simple replacement
+        return filter_pattern.replace("**", "*")
+
+    # Multiple globstars - need more careful handling
+    # For patterns like **/level?/backup/**/*.ext
+    # We want to match any path containing /level?/backup/ and ending with .ext
+
+    # Find middle directories (between first and last **)
+    middle_parts = []
+    start_idx = globstar_positions[0] + 1
+    end_idx = globstar_positions[-1]
+    for i in range(start_idx, end_idx):
+        if parts[i] != "**":
+            middle_parts.append(parts[i])
+
+    if not middle_parts:
+        # No fixed middle parts, just use wildcards
+        result = filter_pattern.replace("**", "*")
+    else:
+        # Create pattern that matches the middle parts
+        middle_pattern = "/".join(middle_parts)
+        # Get the file pattern at the end if any
+        last_part = parts[-1] if parts[-1] != "**" else "*"
+
+        # Match any path containing this pattern
+        if last_part != "*":
+            # Has specific file pattern
+            result = f"*{middle_pattern}*{last_part}"
+        else:
+            result = f"*{middle_pattern}*"
+
+    return result
+
+
+def apply_glob_filter(
+    dc: "DataChain",
+    pattern: str,
+    list_path: str,
+    use_recursive: bool,
+    column: str,
+) -> "DataChain":
+    from datachain.query.schema import Column
+
+    chain = ls(dc, list_path, recursive=use_recursive, column=column)
+
+    # If pattern doesn't contain path separator and list_path is not empty,
+    # prepend the list_path to make the pattern match correctly
+    if list_path and "/" not in pattern:
+        filter_pattern = f"{list_path.rstrip('/')}/{pattern}"
+    else:
+        filter_pattern = pattern
+
+    # Convert globstar patterns to GLOB-compatible patterns
+    glob_pattern = convert_globstar_to_glob(filter_pattern)
+
+    return chain.filter(Column(f"{column}.path").glob(glob_pattern))

{datachain-0.31.1.dist-info → datachain-0.31.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.1
+Version: 0.31.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.1.dist-info → datachain-0.31.2.dist-info}/RECORD

@@ -41,7 +41,7 @@ datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
 datachain/client/azure.py,sha256=7yyAgANHfu9Kfh187MKNTT1guvu9Q-WYsi4vYoY3aew,3270
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=kb_myMWcgGFClY5Rsv6fvHIRblg41dfH5knHJuDbW6w,14015
+datachain/client/fsspec.py,sha256=sChjxu931QgU2-n9MdXlmOrhGAiAckXoDVZTxKcNv6M,14336
 datachain/client/gcs.py,sha256=8hcFhEHp8qGRsJoyfCoawfuwb1Et-MSkyQoM9AnNuXI,5204
 datachain/client/hf.py,sha256=n5xJZdvNLS-SqokxuBCIPfGbhIeC_XfLm_BNYtEVvg4,2677
 datachain/client/local.py,sha256=0J52Wzvw25hSucVlzBvLuMRAZwrAHZAYDvD1mNBqf4c,4607
@@ -112,7 +112,8 @@ datachain/lib/dc/listings.py,sha256=V379Cb-7ZyquM0w7sWArQZkzInZy4GB7QQ1ZfowKzQY,
 datachain/lib/dc/pandas.py,sha256=ObueUXDUFKJGu380GmazdG02ARpKAHPhSaymfmOH13E,1489
 datachain/lib/dc/parquet.py,sha256=STgm19AM-etu7WmOUMJa5Z9GI6tPC-A0P3JO3ulfsKo,1839
 datachain/lib/dc/records.py,sha256=l7TKSKjT6boXGd05KA5vvax-Y-mLMOo46VWrlxPhmdQ,3067
-datachain/lib/dc/storage.py,sha256=asqug7UcW6qgyGGEm7b9ZxfbP2UDIeINF07Ngx06KDE,7887
+datachain/lib/dc/storage.py,sha256=pydeiGLMsmDvruVY_bC5GsV6VLpYpRf7szrD0S2pTmE,9688
+datachain/lib/dc/storage_pattern.py,sha256=QDLLSuBd1mdfkdRi3srGXXigs7rHw3vAnQedjE01_H8,9779
 datachain/lib/dc/utils.py,sha256=9OMiFu2kXIbtMqzJTEr1qbCoCBGpOmTnkWImVgFTKgo,4112
 datachain/lib/dc/values.py,sha256=7l1n352xWrEdql2NhBcZ3hj8xyPglWiY4qHjFPjn6iw,1428
 datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
@@ -160,9 +161,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.31.1.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.31.1.dist-info/METADATA,sha256=5VpADYBwb_LImU9zvCBSHmycEFwnAxf8EZPMziLp6sM,13898
-datachain-0.31.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.31.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.31.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.31.1.dist-info/RECORD,,
+datachain-0.31.2.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.31.2.dist-info/METADATA,sha256=ALo4Vp6w2VSanACVy1xv6aHWzbdasSKzD2U8_SybXBU,13898
+datachain-0.31.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.31.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.31.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.31.2.dist-info/RECORD,,