thds.adls 4.4.20251020213233__py3-none-any.whl → 4.4.20251021162747__py3-none-any.whl

thds/adls/__init__.py

@@ -2,6 +2,7 @@ from thds import core
 
 from . import (  # noqa: F401
     abfss,
+    blob_meta,
     defaults,
     etag,
     fqn,

thds/adls/blob_meta.py

@@ -0,0 +1,38 @@
+import typing as ty
+from dataclasses import dataclass
+
+from azure.storage.blob import BlobProperties, ContainerClient
+
+from thds.core import hashing
+
+from . import hashes
+
+
+@dataclass
+class BlobMeta:
+    path: str
+    size: int
+    hash: ty.Optional[hashing.Hash]
+    metadata: dict[str, str]
+
+
+def to_blob_meta(blob_props: BlobProperties) -> BlobMeta:
+    return BlobMeta(
+        blob_props.name,
+        blob_props.size,
+        next(iter(hashes.extract_hashes_from_props(blob_props).values()), None),
+        blob_props.metadata or {},
+    )
+
+
+def is_dir(blob_meta: BlobMeta) -> bool:
+    return blob_meta.metadata.get("hdi_isfolder", "false") == "true"
+
+
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.containerclient?view=azure-python#azure-storage-blob-containerclient-list-blobs
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobproperties?view=azure-python
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.contentsettings?view=azure-python
+def yield_blob_meta(container_client: ContainerClient, root_dir: str) -> ty.Iterator[BlobMeta]:
+    for blob_props in container_client.list_blobs(name_starts_with=root_dir, include=["metadata"]):
+        # `list_blobs` does not include metadata by default, so we need to explicitly specify including it
+        yield to_blob_meta(blob_props)

thds/adls/list_fast.py

@@ -6,11 +6,12 @@ client instead of the file system client.
 
 import typing as ty
 
-from thds.core import log, parallel, thunks
+from thds.core import log, parallel, source, thunks
 
-from . import global_client
+from . import blob_meta, global_client
+from . import source as adls_source
 from .fqn import AdlsFqn
-from .source_tree import BlobMeta, to_blob_meta, yield_blob_meta
+from .uri import UriIsh, parse_any
 
 R = ty.TypeVar("R")
 
@@ -27,7 +28,7 @@ def _failfast_parallel(thunks: ty.Iterable[ty.Callable[[], R]]) -> ty.Iterator[R
     )
 
 
-def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[BlobMeta]:
+def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[blob_meta.BlobMeta]:
     """A fast way to find all blobs in a directory tree; we do this in parallel on
     subdirs, with as much nesting as necessary (though 1 level should be enough for most cases).
 
@@ -37,7 +38,7 @@ def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[Blo
     """
     if layers <= 0:
         # directly yield the blobs
-        yield from yield_blob_meta(
+        yield from blob_meta.yield_blob_meta(
             global_client.get_global_blob_container_client(fqn.sa, fqn.container),
             fqn.path.rstrip("/") + "/",
         )
@@ -77,8 +78,10 @@ def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[Blo
 
     blob_container_client = global_client.get_global_blob_container_client(fqn.sa, fqn.container)
 
-    def _get_blob_meta(blob_name: str) -> BlobMeta:
-        return to_blob_meta(blob_container_client.get_blob_client(blob_name).get_blob_properties())
+    def _get_blob_meta(blob_name: str) -> blob_meta.BlobMeta:
+        return blob_meta.to_blob_meta(
+            blob_container_client.get_blob_client(blob_name).get_blob_properties()
+        )
 
     for blob_meta_iter in (
         _failfast_parallel((thunks.thunking(_get_blob_meta)(file) for file in files)),
@@ -94,10 +97,22 @@ def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[Blo
         yield from blob_meta_iter
 
 
-def is_dir(blob_meta: BlobMeta) -> bool:  # TODO move to blob_meta.py once it exists
-    return blob_meta.metadata.get("hdi_isfolder", "false") == "true"
-
-
-def _list_blob_meta(fqn: AdlsFqn, layers: int = 1) -> list[BlobMeta]:
+def _list_blob_meta(fqn: AdlsFqn, layers: int = 1) -> list[blob_meta.BlobMeta]:
     """Only for use within multi_layer_yield_blobs."""
     return list(multilayer_yield_blob_meta(fqn, layers))
+
+
+def multilayer_yield_sources(
+    fqn_or_uri: UriIsh,
+    layers: int = 1,
+    filter_: ty.Callable[[blob_meta.BlobMeta], bool] = lambda _: True,
+) -> ty.Iterator[source.Source]:
+    """
+    if you want to list directories and files, use `multilayer_yield_blob_meta` instead
+    """
+    fqn = parse_any(fqn_or_uri)
+    root = fqn.root()
+    for blob in multilayer_yield_blob_meta(fqn, layers):
+        if not blob_meta.is_dir(blob) and filter_(blob):
+            # ^ a "dir" Source would not make sense
+            yield adls_source.from_adls(root / blob.path, hash=blob.hash, size=blob.size)

thds/adls/source_tree.py

@@ -1,53 +1,6 @@
-import typing as ty
-from dataclasses import dataclass
-
-from azure.storage.blob import BlobProperties, ContainerClient
-
-from thds.core import hashing
 from thds.core.source.tree import SourceTree
 
-from . import fqn, global_client, hashes, source, uri
-
-# TODO refactor BlobMeta into its own module.
-
-
-@dataclass
-class BlobMeta:
-    path: str
-    size: int
-    hash: ty.Optional[hashing.Hash]
-    metadata: dict[str, str]
-
-
-def to_blob_meta(blob_props: BlobProperties) -> BlobMeta:
-    return BlobMeta(
-        blob_props.name,
-        blob_props.size,
-        next(iter(hashes.extract_hashes_from_props(blob_props).values()), None),
-        blob_props.metadata or {},
-    )
-
-
-def yield_blob_meta(container_client: ContainerClient, root_dir: str) -> ty.Iterator[BlobMeta]:
-    for blob_props in container_client.list_blobs(name_starts_with=root_dir, include=["metadata"]):
-        # `list_blobs` does not include metadata by default, so we need to explicitly specify including it
-        yield to_blob_meta(blob_props)
-
-
-# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.containerclient?view=azure-python#azure-storage-blob-containerclient-list-blobs
-# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobproperties?view=azure-python
-# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.contentsettings?view=azure-python
-def list_blob_meta(
-    container_client: ContainerClient, root_dir: str, match_suffix: str = ""
-) -> ty.List[BlobMeta]:
-    """Gets the path (relative to the SA/container root), size, and _a_ hash (if available) of all blobs in a directory."""
-    return [
-        blob_meta
-        for blob_meta in yield_blob_meta(container_client, root_dir)
-        if blob_meta.size > 0
-        # container client lists directories as blobs with size 0
-        and blob_meta.path.endswith(match_suffix)
-    ]
+from . import fqn, list_fast, uri
 
 
 def from_path(adls_path: uri.UriIsh, match_suffix: str = "") -> SourceTree:
@@ -56,12 +9,12 @@ def from_path(adls_path: uri.UriIsh, match_suffix: str = "") -> SourceTree:
     """
     root_fqn = uri.parse_any(adls_path)
 
-    container_client = global_client.get_global_blob_container_client(root_fqn.sa, root_fqn.container)
-    container_root = root_fqn.root()
     return SourceTree(
-        sources=[
-            source.from_adls(container_root / blob_meta.path, hash=blob_meta.hash, size=blob_meta.size)
-            for blob_meta in list_blob_meta(container_client, root_fqn.path, match_suffix=match_suffix)
-        ],
+        sources=sorted(
+            list_fast.multilayer_yield_sources(
+                root_fqn, layers=0, filter_=lambda blob: blob.path.endswith(match_suffix)
+            ),
+            key=lambda src: src.uri,
+        ),
         higher_logical_root=fqn.split(root_fqn)[-1],
     )

{thds_adls-4.4.20251020213233.dist-info → thds_adls-4.4.20251021162747.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.4.20251020213233
+Version: 4.4.20251021162747
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.4.20251020213233.dist-info → thds_adls-4.4.20251021162747.dist-info}/RECORD

@@ -1,8 +1,9 @@
-thds/adls/__init__.py,sha256=Z450uhM5BIxgve9qkz_Lw7biVWFKp3Uz_iKaSPYEf1Y,1073
+thds/adls/__init__.py,sha256=CzNcmTdP5RHrD-9K1boDU4Z3pbJAvEAFfFhXGFwpEXw,1088
 thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
 thds/adls/_progress.py,sha256=0cmFZ2sOEogQY25AkMAhcunXO69FdlbkD1rU_zmLAp0,6648
 thds/adls/_upload.py,sha256=rN-nuRZur9VvSUywaQezUi_qedj-xU30fTBG6rasuIA,4853
 thds/adls/abfss.py,sha256=nCFfcdwLiwtz_MdbrpLJ9WOhoz0zHIVxsf-tarorEwM,727
+thds/adls/blob_meta.py,sha256=DDhRK3pNeMaDUjiClXZ9g8nRD-ECZp21gLZGCKSxmfk,1404
 thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
 thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=aD6AquUR8r5W9SXd6Nm1qPrFH_fYpLC5dZk6HjPJnSQ,6611
@@ -17,7 +18,7 @@ thds/adls/fqn.py,sha256=pIfEw25SjZNGmtzOLwrYCblciK35VQ35ZWb_mRaZKK0,5915
 thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
 thds/adls/hashes.py,sha256=2x1zcT_87en_vqFrLFs6F_EZCGpn7hk_81dYAwcypm8,5459
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
-thds/adls/list_fast.py,sha256=t6rIk0ub-YO7Uo--7ml2zx0_L-AYknCeALwA-JGi95s,4321
+thds/adls/list_fast.py,sha256=VTQ0C-78ucnywfCWRWSZ8Kfqhi2jt9VXArhpbiK0cII,4857
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
 thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,7 +26,7 @@ thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
 thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=5t9bc3tUxdh4QdbRvvVd9xF9_Rv4v7CDAqE09M3rE1g,2657
-thds/adls/source_tree.py,sha256=FqVXgvfYPiowrWhRsXBItjvB7t41JRI3sCFVAHxjwgI,2610
+thds/adls/source_tree.py,sha256=gJVpFUI3pjXRM5VmppyAcsmhJwtTl8tGLhed0TvlDOM,622
 thds/adls/upload.py,sha256=XJvygYzn1r6pZAl460s5j-kY9dgfQ6qeoddID7R6OQ0,6621
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
@@ -38,8 +39,8 @@ thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/ls_fast.py,sha256=Nowc-efAL_Y4ybPwZzKIeh7KGIjfecRzdWvJZcBzq_8,585
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.4.20251020213233.dist-info/METADATA,sha256=xlrF_oJgsTBFz5lbfwG1Lu-LAlAveyCqkcMdw6fCr50,587
-thds_adls-4.4.20251020213233.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.4.20251020213233.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
-thds_adls-4.4.20251020213233.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.4.20251020213233.dist-info/RECORD,,
+thds_adls-4.4.20251021162747.dist-info/METADATA,sha256=oYeTcXzql2mAFAUzfgYfHkNOi5PSLxcVgT8HKo7gZkw,587
+thds_adls-4.4.20251021162747.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.4.20251021162747.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
+thds_adls-4.4.20251021162747.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.4.20251021162747.dist-info/RECORD,,