thds.adls 3.1.20250227181712__py3-none-any.whl → 3.1.20250227213257__py3-none-any.whl

thds/adls/_upload.py

@@ -26,7 +26,7 @@ def _get_checksum_content_settings(data: AnyStrSrc) -> ty.Optional[ContentSettin
     """
     md5 = try_md5(data)
     if md5:
-        return ContentSettings(content_md5=md5)
+        return ContentSettings(content_md5=bytearray(md5))
     return None
 
 

thds/adls/azcopy/download.py

@@ -178,7 +178,8 @@ async def async_fastpath(
         except (subprocess.SubprocessError, FileNotFoundError):
             logger.warning("Falling back to Python SDK for download")
 
-    reader = await dl_file_client.download_file(
+    reader = await dl_file_client.download_file(  # type: ignore[misc]
+        # TODO - check above type ignore
         max_concurrency=conf.DOWNLOAD_FILE_MAX_CONCURRENCY(),
         connection_timeout=conf.CONNECTION_TIMEOUT(),
     )

thds/adls/cached_up_down.py

@@ -1,5 +1,10 @@
+import typing as ty
 from pathlib import Path
 
+from thds.adls import source
+from thds.core.source import Source
+from thds.core.source.tree import SourceTree
+
 from .download import download_or_use_verified
 from .fqn import AdlsFqn
 from .global_client import get_global_fs_client
@@ -46,3 +51,20 @@ def download_directory(fqn: AdlsFqn) -> Path:
     fs.fetch_directory(fqn.path, cached_dir_root)
     assert cached_dir_root.is_dir(), "Directory should have been downloaded to the cache."
     return cached_dir_root
+
+
+def upload_directory_through_cache(dest: UriIsh, src_path: Path) -> SourceTree:
+    if not src_path.is_dir():
+        raise ValueError(f"If you want to upload a file, use {upload_through_cache.__name__} instead")
+
+    dest = parse_any(dest)
+
+    def _upload_directory(dir_path: Path) -> ty.Iterable[Source]:
+        for item in dir_path.iterdir():
+            if item.is_dir():  # recur
+                yield from _upload_directory(item)
+            elif item.is_file():  # upload
+                file_dest = dest / str(item.relative_to(src_path))
+                yield source.from_adls(upload_through_cache(file_dest, item))
+
+    return SourceTree(sources=list(_upload_directory(src_path)))

thds/adls/download.py

@@ -208,7 +208,8 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
         # we don't know what we expect, so attempt to retrieve an
         # expectation from ADLS itself.
         file_properties = yield _IoRequest.FILE_PROPERTIES
-        md5b64 = _remote_md5b64(file_properties)
+        md5b64 = _remote_md5b64(file_properties)  # type: ignore[arg-type]
+        # TODO - check above type ignore
 
     def attempt_cache_hit() -> ty.Optional[_FileResult]:
         if not md5b64:
@@ -259,7 +260,8 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
     file_properties = yield _IoRequest.FILE_PROPERTIES
     # no point in downloading if we've asked for hash X but ADLS only has hash Y.
     with _verify_md5s_before_and_after_download(
-        _remote_md5b64(file_properties),
+        _remote_md5b64(file_properties),  # type: ignore[arg-type]
+        # TODO - check above type ignore
         md5b64,
         fqn,
         local_path,
@@ -293,7 +295,7 @@ def _prep_download_coroutine(
     DataLakeFileClient,
 ]:
     co = _download_or_use_verified_cached_coroutine(
-        AdlsFqn(fs_client.account_name, fs_client.file_system_name, remote_key),
+        AdlsFqn(ty.cast(str, fs_client.account_name), fs_client.file_system_name, remote_key),
         local_path,
         md5b64=md5b64,
         cache=cache,
@@ -306,7 +308,8 @@ def _set_md5_if_missing(
 ) -> ty.Optional[ContentSettings]:
     if not file_properties or file_properties.content_settings.content_md5:
         return None
-    file_properties.content_settings.content_md5 = b64decode(md5b64)
+    file_properties.content_settings.content_md5 = b64decode(md5b64)  # type: ignore[assignment]
+    # TODO - check above type ignore
     return file_properties.content_settings
 
 
@@ -370,7 +373,8 @@ async def async_download_or_use_verified(
             if co_request == _IoRequest.FILE_PROPERTIES:
                 if not file_properties:
                     # only fetch these if they haven't already been requested
-                    file_properties = await dl_file_client.get_file_properties()
+                    file_properties = await dl_file_client.get_file_properties()  # type: ignore[misc]
+                    # TODO - check above type ignore
                 co_request = co.send(file_properties)
             elif isinstance(co_request, azcopy.download.DownloadRequest):
                 # coroutine is requesting download
@@ -384,7 +388,10 @@ async def async_download_or_use_verified(
             try:
                 logger.info(f"Setting missing MD5 for {remote_key}")
                 assert file_properties
-                await dl_file_client.set_http_headers(cs, **match_etag(file_properties))
+                await dl_file_client.set_http_headers(  # type: ignore[misc]
+                    cs, **match_etag(file_properties)
+                )
+                # TODO - check above type ignore
             except HttpResponseError as hre:
                 logger.info(f"Unable to set MD5 for {remote_key}: {hre}")
         return si.value.hit

thds/adls/file_properties.py

@@ -9,7 +9,7 @@ from .global_client import get_global_blob_container_client, get_global_fs_clien
 
 def is_directory(info: FileProperties) -> bool:
     # from https://github.com/Azure/azure-sdk-for-python/issues/24814#issuecomment-1159280840
-    return str(info.metadata.get("hdi_isfolder", "")).lower() == "true"
+    return str(info.get("metadata", dict()).get("hdi_isfolder", "")).lower() == "true"
 
 
 def get_file_properties(fqn: AdlsFqn) -> FileProperties:

thds/adls/impl.py

@@ -14,7 +14,6 @@ from typing import (
     AsyncIterator,
     Awaitable,
     Callable,
-    Dict,
     Iterable,
     List,
     Mapping,
@@ -457,7 +456,7 @@ class ADLSFileSystem:
         incl_subdirs: bool = False,
         batch_size: Optional[int] = None,
         recursive: bool = True,
-        path_filter: Optional[Callable[[FileProperties], bool]] = None,
+        path_filter: Optional[Callable[[PathProperties], bool]] = None,
     ) -> List[FileProperties]:
         """Returns a list of `FileProperties` for files in a remote directory.
 
@@ -492,7 +491,7 @@ class ADLSFileSystem:
         file_system_client: FileSystemClient,
         remote_paths: Iterable[str],
         batch_size: Optional[int] = None,
-    ) -> List[Dict[str, Any]]:
+    ) -> List[FileProperties]:
         """Returns a list of `FileProperties` for each file in a list of remote file paths.
 
         See :meth:`~ADLSFileSystem.get_files_info` for more details.
@@ -691,10 +690,11 @@ class ADLSFileSystem:
         self, it: AsyncIterable[T], size: Optional[int] = None
     ) -> AsyncIterator[List[T]]:
         """Async batch generator"""
+        # TODO - look at type ignores here
         batch_size = size if size is not None else self.default_batch_size
-        async with stream.chunks(it, batch_size).stream() as streamer:
+        async with stream.chunks(it, batch_size).stream() as streamer:  # type: ignore[arg-type,var-annotated]
             async for chunk in streamer:
-                yield chunk
+                yield chunk  # type: ignore[misc]
 
     def fetch_files(self, remote_paths: Union[Iterable[str], Mapping[str, Union[Path, str]]]):
         return self._run(self._fetch_files, remote_paths)

thds/adls/sas_tokens.py

@@ -43,6 +43,8 @@ def gen_blob_sas_token(
 
     `expiry` is in seconds.
     """
+    expiry_datetime: ty.Union[ty.Optional[str], datetime.datetime]
+
     if isinstance(account_key, UserDelegationKey):
         expiry_datetime = account_key.signed_expiry
     else:

{thds_adls-3.1.20250227181712.dist-info → thds_adls-3.1.20250227213257.dist-info}/METADATA

@@ -1,15 +1,18 @@
 Metadata-Version: 2.2
 Name: thds.adls
-Version: 3.1.20250227181712
+Version: 3.1.20250227213257
 Summary: ADLS tools
-Author: Trilliant Health
+Author-email: Trilliant Health <info@trillianthealth.com>
+License: MIT
+Project-URL: Repository, https://github.com/TrilliantHealth/trilliant-data-science
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp>=3.8.1
 Requires-Dist: aiostream>=0.4.5
 Requires-Dist: azure-identity>=1.9
 Requires-Dist: azure-storage-file-datalake>=12.6
 Requires-Dist: filelock>=3.0
-Requires-Dist: thds.core>=1.32
+Requires-Dist: thds-core
 
 # adls Library
 

{thds_adls-3.1.20250227181712.dist-info → thds_adls-3.1.20250227213257.dist-info}/RECORD

@@ -1,32 +1,31 @@
 thds/adls/__init__.py,sha256=er14MoCC9PlJMxWVS4G1hAeMJItaJj4EAsrTZlvlb0M,797
 thds/adls/_progress.py,sha256=ZzCHn_G7nHakioNFxdvoJZRr-jN6ymsp5JXf-iReROM,6580
-thds/adls/_upload.py,sha256=q6Sk0CRnNcAjUOUPiBj4CfO4tJD196SQY0lT25CTSE4,4364
+thds/adls/_upload.py,sha256=XyP6tDM7s-A3G0SPSVlXRT4IZYsPqpOE4TeqtxP5i-I,4375
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
-thds/adls/cached_up_down.py,sha256=qTMrsHHrDq3YZ0gaLMFIPnGNraz0ctG9yoFd3HCVit4,1896
+thds/adls/cached_up_down.py,sha256=CEqg-Q9FB_fvgCsN0C_dn9CJqnEpnF0lA7gCdd9s6gg,2721
 thds/adls/conf.py,sha256=q1SPrgb46NpobVzwt_Oyv71-BvsIbZLq9nRWS3LZjz0,1990
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=VxyaGz1gCcz2rGTR9acNRybbxGOYhTldvD_-SdGLBh0,652
-thds/adls/download.py,sha256=Vfd9DEHUmSKPKpKxf9WjkqAzZFZwIgjo4fgM5aIvW0g,16455
+thds/adls/download.py,sha256=JUF-J6bfaSzDM389errg8tHPssGt-K1yp3O_pznls3o,16852
 thds/adls/download_lock.py,sha256=_JZj-kjCUfHk9FvrmEuYpJYknmbam5eReFhGNDgzdLQ,2520
 thds/adls/errors.py,sha256=B_rMsQvQnNmP_sf-x8kmGsv2vIeOh4G9kVbdNVyk350,1469
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
-thds/adls/file_properties.py,sha256=JFkobkxcAaRGAh3TJ9mfc3X872gqrGuw4Xk_HSTj1sY,1582
+thds/adls/file_properties.py,sha256=V3VEjEG3PNyeQaJ159Kco5l7c2EiyjE0yygtmVVjd6E,1597
 thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
 thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
-thds/adls/impl.py,sha256=x1nSqc8W4NeuX8-JGOp2MRkK8ff6GnelTWedGxPs-qY,42494
+thds/adls/impl.py,sha256=4qa70w1sehzp60CI6lW82NLDK-lsM1uUfhPmZnYJaw0,42589
 thds/adls/md5.py,sha256=qOX4_7WUj1QkbH_IwREcQNHvvZccOj-HpHZBfsKn1gY,1846
-thds/adls/meta.json,sha256=MhZqsLGrHq1cqZB6OydUf5yDbfo_GySrRRtjdpImBQc,195
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
 thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/adls/ro_cache.py,sha256=F0uXol0t95mcRuBukNg3A7wt7XXQxpD5Sy09d9sl8f0,4825
-thds/adls/sas_tokens.py,sha256=tO7uUh5EVp2F_NITzz1Vks78KCZwVxT7C0faLj8UJqA,1889
+thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=1JYliDqafqpLewOQ2XMKp4T2WD7b5Rl7XYViyLYtzX8,2437
 thds/adls/source_tree.py,sha256=CiyCwgItF186aOoYiICkWt2oMLGMD6v526HRi8XWHtM,2219
 thds/adls/uri.py,sha256=pDH956p_VEHnjLLUnjWY6sGgRqksp9gdpc9KOW4gEP0,1205
 thds/adls/azcopy/__init__.py,sha256=nTNbgz2GcEiGeswYbAgy4oPhivnzl_5crF3HqCdWWiw,31
-thds/adls/azcopy/download.py,sha256=_I0smR2f9pu0xIiQC9C97Xb-GHMGwn5dqbEhz41H-qo,6548
+thds/adls/azcopy/download.py,sha256=Fig6q-bRZBeoA5L-URl5BACfMH9-9_sc4T2sa9yiZyQ,6611
 thds/adls/resource/__init__.py,sha256=IZ7_aRf1b3jEp7wXOxqHop0gV2gUcf9SOLeEEjIWlCU,1669
 thds/adls/resource/core.py,sha256=BVM91xsZ_B_CoGTc9DDD3FnGy8g6X-9eFpa86ZCzuZI,2717
 thds/adls/resource/file_pointers.py,sha256=PLru_3lwut_ZvrX5Keu-wJkPOt5o7UGf-OOT4ixaXME,2049
@@ -34,8 +33,8 @@ thds/adls/resource/up_down.py,sha256=3uNlTvm2gVhSyYdQTBwsGecOgwtINQfINckR-awwV0Y
 thds/adls/tools/download.py,sha256=vvBO8lSDl9oPugv75qpCkoemT9pOM9BV6yeExlkyG08,1594
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=eMk4pdug1aCMPDDWpIE3Zoq77i5APp9Uuh-sVCCDNJE,493
-thds_adls-3.1.20250227181712.dist-info/METADATA,sha256=O8W1_KMHFvwQZ--B8xPvFDMLL774NvfbhpxvEKkACWE,397
-thds_adls-3.1.20250227181712.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-thds_adls-3.1.20250227181712.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-3.1.20250227181712.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-3.1.20250227181712.dist-info/RECORD,,
+thds_adls-3.1.20250227213257.dist-info/METADATA,sha256=ITESaWWIR-e2DCqoKNHKVGd0kolDejghHQqZ82F5qOc,543
+thds_adls-3.1.20250227213257.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+thds_adls-3.1.20250227213257.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-3.1.20250227213257.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-3.1.20250227213257.dist-info/RECORD,,

thds/adls/meta.json

@@ -1,8 +0,0 @@
-{
-  "git_commit": "f4211224c05a5de6426bfd3efe4d62e3c0a7d64d",
-  "git_branch": "main",
-  "git_is_clean": true,
-  "pyproject_version": "3.1.20250227181712",
-  "thds_user": "runner",
-  "misc": {}
-}