thds.adls 3.1.20250217180321__py3-none-any.whl → 3.1.20250218201547__py3-none-any.whl

thds/adls/__init__.py

@@ -1,6 +1,6 @@
 from thds.core import meta
 
-from . import abfss, defaults, etag, fqn, named_roots, resource, source, uri  # noqa: F401
+from . import abfss, defaults, etag, fqn, named_roots, resource, source, source_tree, uri  # noqa: F401
 from .cached_up_down import download_directory, download_to_cache, upload_through_cache  # noqa: F401
 from .copy import copy_file, copy_files, wait_for_copy  # noqa: F401
 from .errors import BlobNotFoundError  # noqa: F401

thds/adls/fqn.py

@@ -1,10 +1,9 @@
 import re
 from functools import reduce
-from typing import NamedTuple, TypeVar, Union
+from typing import NamedTuple, Tuple, TypeVar, Union
 
-ADLS_SCHEME = (
-    "adls://"  # this is our invention, but ADLS does not appear to define one suitable for general use.
-)
+ADLS_SCHEME = "adls://"
+# this is our invention, but ADLS does not appear to define one suitable for general use.
 
 
 def join(*parts: str) -> str:
@@ -104,6 +103,13 @@ def parent(fqn: FR) -> FR:
     return AdlsFqn(fqn.sa, fqn.container, join(*fqn.path.split("/")[:-1]))  # type: ignore
 
 
+def split(fqn: FR) -> Tuple[str, ...]:
+    if isinstance(fqn, AdlsRoot):
+        return fqn.sa, fqn.container
+    assert isinstance(fqn, AdlsFqn)
+    return tuple(filter(None, (fqn.sa, fqn.container, *fqn.path.split("/"))))
+
+
 SA_REGEX = re.compile(r"^[\w]{3,24}$")
 # https://github.com/MicrosoftDocs/azure-docs/issues/64497#issuecomment-714380739
 CONT_REGEX = re.compile(r"^\w[\w\-]{2,63}$")

thds/adls/meta.json

@@ -1,8 +1,8 @@
 {
-  "git_commit": "d1dbeed714e4b8a66fc91ae65b2f721ae3c7211b",
+  "git_commit": "6aa6700b782d76b65b65256eaa49bd70e8217015",
   "git_branch": "main",
   "git_is_clean": true,
-  "pyproject_version": "3.1.20250217180321",
+  "pyproject_version": "3.1.20250218201547",
   "thds_user": "runner",
   "misc": {}
 }

thds/adls/resource/core.py

@@ -46,7 +46,7 @@ def from_source(source: source.Source) -> AdlsHashedResource:
 def to_source(resource: AdlsHashedResource) -> source.Source:
     return source.from_uri(
         str(resource.fqn),
-        hash=source.Hash("md5", hashing.db64(resource.md5b64)),
+        hash=hashing.Hash("md5", hashing.db64(resource.md5b64)),
     )
 
 

thds/adls/source_tree.py

@@ -0,0 +1,57 @@
+import typing as ty
+from dataclasses import dataclass
+
+from azure.storage.blob import ContainerClient
+
+from thds.core import hashing
+from thds.core.source.tree import SourceTree
+
+from . import fqn, global_client, source, uri
+
+
+@dataclass
+class BlobMeta:
+    path: str
+    size: int
+    md5: ty.Optional[hashing.Hash]
+
+
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.containerclient?view=azure-python#azure-storage-blob-containerclient-list-blobs
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobproperties?view=azure-python
+# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.contentsettings?view=azure-python
+def list_blob_meta(
+    container_client: ContainerClient, root_dir: str, match_suffix: str = ""
+) -> ty.List[BlobMeta]:
+    """Gets the path (relative to the SA/container root), size, and MD5 hash of all blobs in a directory."""
+    return [
+        BlobMeta(
+            blob_props.name,  # type: ignore
+            blob_props.size,  # type: ignore
+            (
+                hashing.Hash("md5", bytes(blob_props.content_settings.content_md5))
+                if blob_props.content_settings.content_md5
+                else None
+            ),
+        )
+        for blob_props in container_client.list_blobs(name_starts_with=root_dir)
+        if blob_props.size > 0  # type: ignore
+        # container client lists directories as blobs with size 0
+        and blob_props.name.endswith(match_suffix)  # type: ignore
+    ]
+
+
+def from_path(adls_path: uri.UriIsh, match_suffix: str = "") -> SourceTree:
+    """Creates a SourceTree object where the logical root is the final piece of the
+    provided adls path.
+    """
+    root_fqn = uri.parse_any(adls_path)
+
+    container_client = global_client.get_global_blob_container_client(root_fqn.sa, root_fqn.container)
+    container_root = root_fqn.root()
+    return SourceTree(
+        sources=[
+            source.from_adls(container_root / blob_meta.path, hash=blob_meta.md5)
+            for blob_meta in list_blob_meta(container_client, root_fqn.path, match_suffix=match_suffix)
+        ],
+        higher_logical_root=fqn.split(root_fqn)[-1],
+    )

{thds.adls-3.1.20250217180321.dist-info → thds.adls-3.1.20250218201547.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: thds.adls
-Version: 3.1.20250217180321
+Version: 3.1.20250218201547
 Summary: ADLS tools
 Author: Trilliant Health
 Description-Content-Type: text/markdown
@@ -9,7 +9,7 @@ Requires-Dist: aiostream>=0.4.5
 Requires-Dist: azure-identity>=1.9
 Requires-Dist: azure-storage-file-datalake>=12.6
 Requires-Dist: filelock>=3.0
-Requires-Dist: thds.core>=1.31
+Requires-Dist: thds.core>=1.32
 
 # adls Library
 

{thds.adls-3.1.20250217180321.dist-info → thds.adls-3.1.20250218201547.dist-info}/RECORD

@@ -1,4 +1,4 @@
-thds/adls/__init__.py,sha256=Tc23Gy292GO7hC8qwxiAnVE3jGqLcEjHed2MKlSSpvI,784
+thds/adls/__init__.py,sha256=er14MoCC9PlJMxWVS4G1hAeMJItaJj4EAsrTZlvlb0M,797
 thds/adls/_progress.py,sha256=SVOiVRvfvOfUjp-y-UUgA6n0FhAnpKesxltaQ16kmOk,6394
 thds/adls/_upload.py,sha256=q6Sk0CRnNcAjUOUPiBj4CfO4tJD196SQY0lT25CTSE4,4364
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
@@ -12,27 +12,28 @@ thds/adls/download_lock.py,sha256=_JZj-kjCUfHk9FvrmEuYpJYknmbam5eReFhGNDgzdLQ,25
 thds/adls/errors.py,sha256=IQfkdLVg0feAUd-xQzkS4-3Lp5_Ld9V9-SqiXeWLRtA,1335
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
 thds/adls/file_properties.py,sha256=JFkobkxcAaRGAh3TJ9mfc3X872gqrGuw4Xk_HSTj1sY,1582
-thds/adls/fqn.py,sha256=pGPBAWKrXorXs1DGkH-XXeDgwwQk3S_heQE7VkPAbW4,5598
+thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
 thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
 thds/adls/impl.py,sha256=x1nSqc8W4NeuX8-JGOp2MRkK8ff6GnelTWedGxPs-qY,42494
 thds/adls/md5.py,sha256=qOX4_7WUj1QkbH_IwREcQNHvvZccOj-HpHZBfsKn1gY,1846
-thds/adls/meta.json,sha256=86WmgGgPKi8ZL0dYW19JTjs0rK58zLr2A_bAlvpkaBc,195
+thds/adls/meta.json,sha256=9mFILKWuoXbJo2n0GOcOki9KkeCM84bOwctQogfI-Jo,195
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
 thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/adls/ro_cache.py,sha256=F0uXol0t95mcRuBukNg3A7wt7XXQxpD5Sy09d9sl8f0,4825
 thds/adls/sas_tokens.py,sha256=tO7uUh5EVp2F_NITzz1Vks78KCZwVxT7C0faLj8UJqA,1889
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=1JYliDqafqpLewOQ2XMKp4T2WD7b5Rl7XYViyLYtzX8,2437
+thds/adls/source_tree.py,sha256=CiyCwgItF186aOoYiICkWt2oMLGMD6v526HRi8XWHtM,2219
 thds/adls/uri.py,sha256=pDH956p_VEHnjLLUnjWY6sGgRqksp9gdpc9KOW4gEP0,1205
 thds/adls/resource/__init__.py,sha256=IZ7_aRf1b3jEp7wXOxqHop0gV2gUcf9SOLeEEjIWlCU,1669
-thds/adls/resource/core.py,sha256=uU31tAVkwDcLC_TerZWq58xi4D791vNLDUfP0Y1BAoo,2716
+thds/adls/resource/core.py,sha256=BVM91xsZ_B_CoGTc9DDD3FnGy8g6X-9eFpa86ZCzuZI,2717
 thds/adls/resource/file_pointers.py,sha256=PLru_3lwut_ZvrX5Keu-wJkPOt5o7UGf-OOT4ixaXME,2049
 thds/adls/resource/up_down.py,sha256=3uNlTvm2gVhSyYdQTBwsGecOgwtINQfINckR-awwV0Y,9907
 thds/adls/tools/download.py,sha256=ZZ8t1g9MRRy-aENGjr10E4aAqyVkKq2OqpyH70pCD_8,965
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=eMk4pdug1aCMPDDWpIE3Zoq77i5APp9Uuh-sVCCDNJE,493
-thds.adls-3.1.20250217180321.dist-info/METADATA,sha256=FLx2KCf8yGNlmlXASxJzjIcQdbh4itJZ90OE7qPBN-M,397
-thds.adls-3.1.20250217180321.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-thds.adls-3.1.20250217180321.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds.adls-3.1.20250217180321.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds.adls-3.1.20250217180321.dist-info/RECORD,,
+thds.adls-3.1.20250218201547.dist-info/METADATA,sha256=EDR-b7AK1xPKvEq7rNK5xJgcWPhz32aL0mr5IwL645k,397
+thds.adls-3.1.20250218201547.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+thds.adls-3.1.20250218201547.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds.adls-3.1.20250218201547.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds.adls-3.1.20250218201547.dist-info/RECORD,,