lsst-resources 29.2025.2000__py3-none-any.whl → 29.2025.2400__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/resources/_resourceHandles/_fileResourceHandle.py +1 -1
- lsst/resources/_resourceHandles/_s3ResourceHandle.py +3 -17
- lsst/resources/_resourcePath.py +9 -1
- lsst/resources/http.py +152 -11
- lsst/resources/s3.py +81 -17
- lsst/resources/tests.py +26 -13
- lsst/resources/version.py +1 -1
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/METADATA +1 -1
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/RECORD +14 -14
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/WHEEL +1 -1
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/licenses/LICENSE +0 -0
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/top_level.txt +0 -0
- {lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/zip-safe +0 -0
|
@@ -14,14 +14,12 @@ from __future__ import annotations
|
|
|
14
14
|
__all__ = ("S3ResourceHandle",)
|
|
15
15
|
|
|
16
16
|
import logging
|
|
17
|
-
import warnings
|
|
18
17
|
from collections.abc import Iterable, Mapping
|
|
19
18
|
from io import SEEK_CUR, SEEK_END, SEEK_SET, BytesIO, UnsupportedOperation
|
|
20
19
|
from typing import TYPE_CHECKING
|
|
21
20
|
|
|
22
21
|
from botocore.exceptions import ClientError
|
|
23
22
|
|
|
24
|
-
from lsst.utils.introspection import find_outside_stacklevel
|
|
25
23
|
from lsst.utils.timer import time_this
|
|
26
24
|
|
|
27
25
|
from ..s3utils import all_retryable_errors, backoff, max_retry_time, translate_client_error
|
|
@@ -168,21 +166,9 @@ class S3ResourceHandle(BaseResourceHandle[bytes]):
|
|
|
168
166
|
# written to.
|
|
169
167
|
s3_min_bits = 5 * 1024 * 1024 # S3 flush threshold is 5 Mib.
|
|
170
168
|
if (
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
):
|
|
175
|
-
amount = s3_min_bits / (1024 * 1024)
|
|
176
|
-
warnings.warn(
|
|
177
|
-
f"S3 does not support flushing objects less than {amount} Mib, skipping",
|
|
178
|
-
stacklevel=find_outside_stacklevel(
|
|
179
|
-
"lsst.resources",
|
|
180
|
-
"backoff",
|
|
181
|
-
"contextlib",
|
|
182
|
-
allow_modules={"lsst.resources.tests"},
|
|
183
|
-
),
|
|
184
|
-
)
|
|
185
|
-
self._warned = True
|
|
169
|
+
self.tell() - (self._last_flush_position or 0)
|
|
170
|
+
) < s3_min_bits and self._closed != CloseStatus.CLOSING:
|
|
171
|
+
# Return until the buffer is big enough.
|
|
186
172
|
return
|
|
187
173
|
# nothing to write, don't create an empty upload
|
|
188
174
|
if self.tell() == 0:
|
lsst/resources/_resourcePath.py
CHANGED
|
@@ -649,9 +649,11 @@ class ResourcePath: # numpydoc ignore=PR02
|
|
|
649
649
|
# Disallow a change in scheme
|
|
650
650
|
if "scheme" in kwargs:
|
|
651
651
|
raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
|
|
652
|
-
|
|
652
|
+
result = self.__class__(
|
|
653
653
|
self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
|
|
654
654
|
)
|
|
655
|
+
result._copy_extra_attributes(self)
|
|
656
|
+
return result
|
|
655
657
|
|
|
656
658
|
def updatedFile(self, newfile: str) -> ResourcePath:
|
|
657
659
|
"""Return new URI with an updated final component of the path.
|
|
@@ -1903,6 +1905,12 @@ class ResourcePath: # numpydoc ignore=PR02
|
|
|
1903
1905
|
"""
|
|
1904
1906
|
raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
|
|
1905
1907
|
|
|
1908
|
+
def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
|
|
1909
|
+
# May be overridden by subclasses to transfer attributes when a
|
|
1910
|
+
# ResourcePath is constructed using the "clone" version of the
|
|
1911
|
+
# ResourcePath constructor by passing in a ResourcePath object.
|
|
1912
|
+
pass
|
|
1913
|
+
|
|
1906
1914
|
|
|
1907
1915
|
ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
|
|
1908
1916
|
"""Type-annotation alias for objects that can be coerced to ResourcePath.
|
lsst/resources/http.py
CHANGED
|
@@ -759,6 +759,42 @@ class HttpResourcePath(ResourcePath):
|
|
|
759
759
|
a HTTP URL. The value of the variable is not inspected.
|
|
760
760
|
"""
|
|
761
761
|
|
|
762
|
+
@staticmethod
|
|
763
|
+
def create_http_resource_path(
|
|
764
|
+
path: str, *, extra_headers: dict[str, str] | None = None
|
|
765
|
+
) -> HttpResourcePath:
|
|
766
|
+
"""Create an instance of `HttpResourcePath` with additional
|
|
767
|
+
HTTP-specific configuration.
|
|
768
|
+
|
|
769
|
+
Parameters
|
|
770
|
+
----------
|
|
771
|
+
path : `str`
|
|
772
|
+
HTTP URL to be wrapped in a `ResourcePath` instance.
|
|
773
|
+
extra_headers : `dict` [ `str`, `str` ], optional
|
|
774
|
+
Additional headers that will be sent with every HTTP request made
|
|
775
|
+
by this `ResourcePath`. These override any headers that may be
|
|
776
|
+
generated internally by `HttpResourcePath` (e.g. authentication
|
|
777
|
+
headers).
|
|
778
|
+
|
|
779
|
+
Return
|
|
780
|
+
------
|
|
781
|
+
instance : `ResourcePath`
|
|
782
|
+
Newly-created `HttpResourcePath` instance.
|
|
783
|
+
|
|
784
|
+
Notes
|
|
785
|
+
-----
|
|
786
|
+
Most users should use the `ResourcePath` constructor, instead.
|
|
787
|
+
"""
|
|
788
|
+
# Make sure we instantiate ResourcePath using a string to guarantee we
|
|
789
|
+
# get a new ResourcePath. If we accidentally provided a ResourcePath
|
|
790
|
+
# instance instead, the ResourcePath constructor sometimes returns
|
|
791
|
+
# the original object and we would be modifying an object that is
|
|
792
|
+
# supposed to be immutable.
|
|
793
|
+
instance = ResourcePath(str(path))
|
|
794
|
+
assert isinstance(instance, HttpResourcePath)
|
|
795
|
+
instance._extra_headers = extra_headers
|
|
796
|
+
return instance
|
|
797
|
+
|
|
762
798
|
# WebDAV servers known to be able to sign URLs. The values are lowercased
|
|
763
799
|
# server identifiers retrieved from the 'Server' header included in
|
|
764
800
|
# the response to a HTTP OPTIONS request.
|
|
@@ -805,39 +841,48 @@ class HttpResourcePath(ResourcePath):
|
|
|
805
841
|
# and is shared by all instances of this class.
|
|
806
842
|
_tcp_connector: TCPConnector | None = None
|
|
807
843
|
|
|
844
|
+
# Additional headers added to every request.
|
|
845
|
+
_extra_headers: dict[str, str] | None = None
|
|
846
|
+
|
|
808
847
|
@property
|
|
809
|
-
def metadata_session(self) ->
|
|
848
|
+
def metadata_session(self) -> _SessionWrapper:
|
|
810
849
|
"""Client session to send requests which do not require upload or
|
|
811
850
|
download of data, i.e. mostly metadata requests.
|
|
812
851
|
"""
|
|
852
|
+
session = None
|
|
813
853
|
if hasattr(self, "_metadata_session"):
|
|
814
854
|
if HttpResourcePath._pid == os.getpid():
|
|
815
|
-
|
|
855
|
+
session = self._metadata_session
|
|
816
856
|
else:
|
|
817
857
|
# The metadata session we have in cache was likely created by
|
|
818
858
|
# a parent process. Discard all the sessions in that store.
|
|
819
859
|
self._metadata_session_store.clear()
|
|
820
860
|
|
|
821
861
|
# Retrieve a new metadata session.
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
862
|
+
if session is None:
|
|
863
|
+
HttpResourcePath._pid = os.getpid()
|
|
864
|
+
session = self._metadata_session_store.get(self)
|
|
865
|
+
self._metadata_session: requests.Session = session
|
|
866
|
+
return _SessionWrapper(session, extra_headers=self._extra_headers)
|
|
825
867
|
|
|
826
868
|
@property
|
|
827
|
-
def data_session(self) ->
|
|
869
|
+
def data_session(self) -> _SessionWrapper:
|
|
828
870
|
"""Client session for uploading and downloading data."""
|
|
871
|
+
session = None
|
|
829
872
|
if hasattr(self, "_data_session"):
|
|
830
873
|
if HttpResourcePath._pid == os.getpid():
|
|
831
|
-
|
|
874
|
+
session = self._data_session
|
|
832
875
|
else:
|
|
833
876
|
# The data session we have in cache was likely created by
|
|
834
877
|
# a parent process. Discard all the sessions in that store.
|
|
835
878
|
self._data_session_store.clear()
|
|
836
879
|
|
|
837
880
|
# Retrieve a new data session.
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
881
|
+
if session is None:
|
|
882
|
+
HttpResourcePath._pid = os.getpid()
|
|
883
|
+
session = self._data_session_store.get(self)
|
|
884
|
+
self._data_session: requests.Session = session
|
|
885
|
+
return _SessionWrapper(session, extra_headers=self._extra_headers)
|
|
841
886
|
|
|
842
887
|
def _clear_sessions(self) -> None:
|
|
843
888
|
"""Close the socket connections that are still open.
|
|
@@ -1562,7 +1607,7 @@ class HttpResourcePath(ResourcePath):
|
|
|
1562
1607
|
url: str | None = None,
|
|
1563
1608
|
headers: dict[str, str] | None = None,
|
|
1564
1609
|
body: str | None = None,
|
|
1565
|
-
session:
|
|
1610
|
+
session: _SessionWrapper | None = None,
|
|
1566
1611
|
timeout: tuple[float, float] | None = None,
|
|
1567
1612
|
) -> requests.Response:
|
|
1568
1613
|
"""Send a webDAV request and correctly handle redirects.
|
|
@@ -1983,6 +2028,10 @@ class HttpResourcePath(ResourcePath):
|
|
|
1983
2028
|
with super()._openImpl(mode, encoding=encoding) as http_handle:
|
|
1984
2029
|
yield http_handle
|
|
1985
2030
|
|
|
2031
|
+
def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
|
|
2032
|
+
assert isinstance(original_uri, HttpResourcePath)
|
|
2033
|
+
self._extra_headers = original_uri._extra_headers
|
|
2034
|
+
|
|
1986
2035
|
|
|
1987
2036
|
def _dump_response(resp: requests.Response) -> None:
|
|
1988
2037
|
"""Log the contents of a HTTP or webDAV request and its response.
|
|
@@ -2193,3 +2242,95 @@ class DavProperty:
|
|
|
2193
2242
|
@property
|
|
2194
2243
|
def href(self) -> str:
|
|
2195
2244
|
return self._href
|
|
2245
|
+
|
|
2246
|
+
|
|
2247
|
+
class _SessionWrapper(contextlib.AbstractContextManager):
|
|
2248
|
+
"""Wraps a `requests.Session` to allow header values to be injected with
|
|
2249
|
+
all requests.
|
|
2250
|
+
|
|
2251
|
+
Notes
|
|
2252
|
+
-----
|
|
2253
|
+
`requests.Session` already has a feature for setting headers globally, but
|
|
2254
|
+
our session objects are global and authorization headers can vary for each
|
|
2255
|
+
HttpResourcePath instance.
|
|
2256
|
+
"""
|
|
2257
|
+
|
|
2258
|
+
def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
|
|
2259
|
+
self._session = session
|
|
2260
|
+
self._extra_headers = extra_headers
|
|
2261
|
+
|
|
2262
|
+
def __enter__(self) -> _SessionWrapper:
|
|
2263
|
+
self._session.__enter__()
|
|
2264
|
+
return self
|
|
2265
|
+
|
|
2266
|
+
def __exit__(
|
|
2267
|
+
self,
|
|
2268
|
+
exc_type: Any,
|
|
2269
|
+
exc_value: Any,
|
|
2270
|
+
traceback: Any,
|
|
2271
|
+
) -> None:
|
|
2272
|
+
return self._session.__exit__(exc_type, exc_value, traceback)
|
|
2273
|
+
|
|
2274
|
+
def get(
|
|
2275
|
+
self,
|
|
2276
|
+
url: str,
|
|
2277
|
+
*,
|
|
2278
|
+
timeout: tuple[float, float],
|
|
2279
|
+
allow_redirects: bool = True,
|
|
2280
|
+
stream: bool,
|
|
2281
|
+
headers: dict[str, str] | None = None,
|
|
2282
|
+
) -> requests.Response:
|
|
2283
|
+
return self._session.get(
|
|
2284
|
+
url,
|
|
2285
|
+
timeout=timeout,
|
|
2286
|
+
allow_redirects=allow_redirects,
|
|
2287
|
+
stream=stream,
|
|
2288
|
+
headers=self._augment_headers(headers),
|
|
2289
|
+
)
|
|
2290
|
+
|
|
2291
|
+
def head(
|
|
2292
|
+
self,
|
|
2293
|
+
url: str,
|
|
2294
|
+
*,
|
|
2295
|
+
timeout: tuple[float, float],
|
|
2296
|
+
allow_redirects: bool,
|
|
2297
|
+
stream: bool,
|
|
2298
|
+
headers: dict[str, str] | None = None,
|
|
2299
|
+
) -> requests.Response:
|
|
2300
|
+
return self._session.head(
|
|
2301
|
+
url,
|
|
2302
|
+
timeout=timeout,
|
|
2303
|
+
allow_redirects=allow_redirects,
|
|
2304
|
+
stream=stream,
|
|
2305
|
+
headers=self._augment_headers(headers),
|
|
2306
|
+
)
|
|
2307
|
+
|
|
2308
|
+
def request(
|
|
2309
|
+
self,
|
|
2310
|
+
method: str,
|
|
2311
|
+
url: str,
|
|
2312
|
+
*,
|
|
2313
|
+
data: str | bytes | BinaryIO | None,
|
|
2314
|
+
timeout: tuple[float, float],
|
|
2315
|
+
allow_redirects: bool,
|
|
2316
|
+
stream: bool,
|
|
2317
|
+
headers: dict[str, str] | None = None,
|
|
2318
|
+
) -> requests.Response:
|
|
2319
|
+
return self._session.request(
|
|
2320
|
+
method,
|
|
2321
|
+
url,
|
|
2322
|
+
data=data,
|
|
2323
|
+
timeout=timeout,
|
|
2324
|
+
allow_redirects=allow_redirects,
|
|
2325
|
+
stream=stream,
|
|
2326
|
+
headers=self._augment_headers(headers),
|
|
2327
|
+
)
|
|
2328
|
+
|
|
2329
|
+
def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
|
|
2330
|
+
if headers is None:
|
|
2331
|
+
headers = {}
|
|
2332
|
+
|
|
2333
|
+
if self._extra_headers is not None:
|
|
2334
|
+
headers = headers | self._extra_headers
|
|
2335
|
+
|
|
2336
|
+
return headers
|
lsst/resources/s3.py
CHANGED
|
@@ -13,6 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
__all__ = ("S3ResourcePath",)
|
|
15
15
|
|
|
16
|
+
import concurrent.futures
|
|
16
17
|
import contextlib
|
|
17
18
|
import io
|
|
18
19
|
import logging
|
|
@@ -32,7 +33,7 @@ from lsst.utils.timer import time_this
|
|
|
32
33
|
|
|
33
34
|
from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
|
|
34
35
|
from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
|
|
35
|
-
from ._resourcePath import MBulkResult, ResourcePath
|
|
36
|
+
from ._resourcePath import _EXECUTOR_TYPE, MBulkResult, ResourcePath, _get_executor_class, _patch_environ
|
|
36
37
|
from .s3utils import (
|
|
37
38
|
_get_s3_connection_parameters,
|
|
38
39
|
_s3_disable_bucket_validation,
|
|
@@ -46,6 +47,7 @@ from .s3utils import (
|
|
|
46
47
|
s3CheckFileExists,
|
|
47
48
|
translate_client_error,
|
|
48
49
|
)
|
|
50
|
+
from .utils import _get_num_workers
|
|
49
51
|
|
|
50
52
|
try:
|
|
51
53
|
from boto3.s3.transfer import TransferConfig # type: ignore
|
|
@@ -247,34 +249,96 @@ class S3ResourcePath(ResourcePath):
|
|
|
247
249
|
|
|
248
250
|
results: dict[ResourcePath, MBulkResult] = {}
|
|
249
251
|
for related_uris in grouped_uris.values():
|
|
250
|
-
# The client and bucket are the same for each of the remaining
|
|
251
|
-
# URIs.
|
|
252
|
-
first_uri = related_uris[0]
|
|
253
252
|
# API requires no more than 1000 per call.
|
|
254
253
|
chunk_num = 0
|
|
254
|
+
chunks: list[tuple[ResourcePath, ...]] = []
|
|
255
|
+
key_to_uri: dict[str, ResourcePath] = {}
|
|
255
256
|
for chunk in chunk_iterable(related_uris, chunk_size=1_000):
|
|
256
|
-
key_to_uri: dict[str, ResourcePath] = {}
|
|
257
|
-
keys: list[dict[str, str]] = []
|
|
258
257
|
for uri in chunk:
|
|
259
258
|
key = uri.relativeToPathRoot
|
|
260
259
|
key_to_uri[key] = uri
|
|
261
|
-
keys.append({"Key": key})
|
|
262
260
|
# Default to assuming everything worked.
|
|
263
261
|
results[uri] = MBulkResult(True, None)
|
|
264
262
|
chunk_num += 1
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
263
|
+
chunks.append(chunk)
|
|
264
|
+
|
|
265
|
+
# Bulk remove.
|
|
266
|
+
with time_this(
|
|
267
|
+
log,
|
|
268
|
+
msg="Bulk delete; %d chunk%s; totalling %d dataset%s",
|
|
269
|
+
args=(
|
|
270
|
+
len(chunks),
|
|
271
|
+
"s" if len(chunks) != 1 else "",
|
|
272
|
+
len(related_uris),
|
|
273
|
+
"s" if len(related_uris) != 1 else "",
|
|
274
|
+
),
|
|
275
|
+
):
|
|
276
|
+
errored = cls._mremove_select(chunks)
|
|
277
|
+
|
|
278
|
+
# Update with error information.
|
|
279
|
+
results.update(errored)
|
|
271
280
|
|
|
272
|
-
|
|
273
|
-
for key, bulk_result in errored.items():
|
|
274
|
-
results[key_to_uri[key]] = bulk_result
|
|
281
|
+
return results
|
|
275
282
|
|
|
283
|
+
@classmethod
|
|
284
|
+
def _mremove_select(cls, chunks: list[tuple[ResourcePath, ...]]) -> dict[ResourcePath, MBulkResult]:
|
|
285
|
+
if len(chunks) == 1:
|
|
286
|
+
# Do the removal directly without futures.
|
|
287
|
+
return cls._delete_objects_wrapper(chunks[0])
|
|
288
|
+
pool_executor_class = _get_executor_class()
|
|
289
|
+
if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
|
|
290
|
+
# Patch the environment to make it think there is only one worker
|
|
291
|
+
# for each subprocess.
|
|
292
|
+
with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
|
|
293
|
+
return cls._mremove_with_pool(pool_executor_class, chunks)
|
|
294
|
+
else:
|
|
295
|
+
return cls._mremove_with_pool(pool_executor_class, chunks)
|
|
296
|
+
|
|
297
|
+
@classmethod
|
|
298
|
+
def _mremove_with_pool(
|
|
299
|
+
cls,
|
|
300
|
+
pool_executor_class: _EXECUTOR_TYPE,
|
|
301
|
+
chunks: list[tuple[ResourcePath, ...]],
|
|
302
|
+
*,
|
|
303
|
+
num_workers: int | None = None,
|
|
304
|
+
) -> dict[ResourcePath, MBulkResult]:
|
|
305
|
+
# Different name because different API to base class.
|
|
306
|
+
# No need to make more workers than we have chunks.
|
|
307
|
+
max_workers = num_workers if num_workers is not None else min(len(chunks), _get_num_workers())
|
|
308
|
+
results: dict[ResourcePath, MBulkResult] = {}
|
|
309
|
+
with pool_executor_class(max_workers=max_workers) as remove_executor:
|
|
310
|
+
future_remove = {
|
|
311
|
+
remove_executor.submit(cls._delete_objects_wrapper, chunk): i
|
|
312
|
+
for i, chunk in enumerate(chunks)
|
|
313
|
+
}
|
|
314
|
+
for future in concurrent.futures.as_completed(future_remove):
|
|
315
|
+
try:
|
|
316
|
+
results.update(future.result())
|
|
317
|
+
except Exception as e:
|
|
318
|
+
# The chunk utterly failed.
|
|
319
|
+
chunk = chunks[future_remove[future]]
|
|
320
|
+
for uri in chunk:
|
|
321
|
+
results[uri] = MBulkResult(False, e)
|
|
276
322
|
return results
|
|
277
323
|
|
|
324
|
+
@classmethod
|
|
325
|
+
def _delete_objects_wrapper(cls, uris: tuple[ResourcePath, ...]) -> dict[ResourcePath, MBulkResult]:
|
|
326
|
+
"""Convert URIs to keys and call low-level API."""
|
|
327
|
+
if not uris:
|
|
328
|
+
return {}
|
|
329
|
+
keys: list[dict[str, str]] = []
|
|
330
|
+
key_to_uri: dict[str, ResourcePath] = {}
|
|
331
|
+
for uri in uris:
|
|
332
|
+
key = uri.relativeToPathRoot
|
|
333
|
+
key_to_uri[key] = uri
|
|
334
|
+
keys.append({"Key": key})
|
|
335
|
+
|
|
336
|
+
first_uri = cast(S3ResourcePath, uris[0])
|
|
337
|
+
results = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
|
|
338
|
+
|
|
339
|
+
# Remap error object keys to uris.
|
|
340
|
+
return {key_to_uri[key]: result for key, result in results.items()}
|
|
341
|
+
|
|
278
342
|
@classmethod
|
|
279
343
|
@backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
|
|
280
344
|
def _delete_related_objects(
|
|
@@ -478,7 +542,7 @@ class S3ResourcePath(ResourcePath):
|
|
|
478
542
|
try:
|
|
479
543
|
self.client.copy_object(CopySource=copy_source, Bucket=self._bucket, Key=self.relativeToPathRoot)
|
|
480
544
|
except (self.client.exceptions.NoSuchKey, self.client.exceptions.NoSuchBucket) as err:
|
|
481
|
-
raise FileNotFoundError("No such resource to transfer: {self}") from err
|
|
545
|
+
raise FileNotFoundError(f"No such resource to transfer: {self}") from err
|
|
482
546
|
except ClientError as err:
|
|
483
547
|
translate_client_error(err, self)
|
|
484
548
|
raise
|
lsst/resources/tests.py
CHANGED
|
@@ -61,18 +61,18 @@ def _check_open(
|
|
|
61
61
|
"""
|
|
62
62
|
text_content = "abcdefghijklmnopqrstuvwxyz🙂"
|
|
63
63
|
bytes_content = uuid.uuid4().bytes
|
|
64
|
-
content_by_mode_suffix = {
|
|
64
|
+
content_by_mode_suffix: dict[str, str | bytes] = {
|
|
65
65
|
"": text_content,
|
|
66
66
|
"t": text_content,
|
|
67
67
|
"b": bytes_content,
|
|
68
68
|
}
|
|
69
|
-
empty_content_by_mode_suffix = {
|
|
69
|
+
empty_content_by_mode_suffix: dict[str, str | bytes] = {
|
|
70
70
|
"": "",
|
|
71
71
|
"t": "",
|
|
72
72
|
"b": b"",
|
|
73
73
|
}
|
|
74
74
|
# To appease mypy
|
|
75
|
-
double_content_by_mode_suffix = {
|
|
75
|
+
double_content_by_mode_suffix: dict[str, str | bytes] = {
|
|
76
76
|
"": text_content + text_content,
|
|
77
77
|
"t": text_content + text_content,
|
|
78
78
|
"b": bytes_content + bytes_content,
|
|
@@ -143,6 +143,16 @@ def _check_open(
|
|
|
143
143
|
content_read = read_buffer.read()
|
|
144
144
|
test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
|
|
145
145
|
|
|
146
|
+
# Write multiple chunks with flushing to ensure that any handles that
|
|
147
|
+
# cache without flushing work properly.
|
|
148
|
+
n = 3
|
|
149
|
+
with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
|
|
150
|
+
for _ in range(n):
|
|
151
|
+
write_buffer.write(content)
|
|
152
|
+
write_buffer.flush()
|
|
153
|
+
with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
|
|
154
|
+
test_case.assertEqual(read_buffer.read(), content * n)
|
|
155
|
+
|
|
146
156
|
# Write two copies of the content, overwriting the single copy there.
|
|
147
157
|
with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
|
|
148
158
|
write_buffer.write(double_content)
|
|
@@ -557,6 +567,10 @@ class GenericReadWriteTestCase(_GenericTestCase):
|
|
|
557
567
|
|
|
558
568
|
transfer_modes: tuple[str, ...] = ("copy", "move")
|
|
559
569
|
testdir: str | None = None
|
|
570
|
+
# Number of files to use for mremove() testing to ensure difference code
|
|
571
|
+
# paths are hit. Do not want to generically use many files for schemes
|
|
572
|
+
# where it makes no difference.
|
|
573
|
+
n_mremove_files: int = 15
|
|
560
574
|
|
|
561
575
|
def setUp(self) -> None:
|
|
562
576
|
if self.scheme is None:
|
|
@@ -1023,19 +1037,14 @@ class GenericReadWriteTestCase(_GenericTestCase):
|
|
|
1023
1037
|
# A file that is not there.
|
|
1024
1038
|
file = root.join("config/basic/butler.yaml")
|
|
1025
1039
|
|
|
1026
|
-
# Create some files.
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
"dir1/c.yaml",
|
|
1031
|
-
"dir1/d.yaml",
|
|
1032
|
-
"dir2/e.yaml",
|
|
1033
|
-
}
|
|
1034
|
-
expected_uris = {root.join(f) for f in expected_files}
|
|
1040
|
+
# Create some files. Most schemes the code paths do not change for 10
|
|
1041
|
+
# vs 1000 files but in some schemes it does.
|
|
1042
|
+
expected_files = [f"dir1/f{n}.yaml" for n in range(self.n_mremove_files)]
|
|
1043
|
+
expected_uris = [root.join(f) for f in expected_files]
|
|
1035
1044
|
for uri in expected_uris:
|
|
1036
1045
|
uri.write(b"")
|
|
1037
1046
|
self.assertTrue(uri.exists())
|
|
1038
|
-
expected_uris.
|
|
1047
|
+
expected_uris.append(file)
|
|
1039
1048
|
|
|
1040
1049
|
# Force to run with fewer workers than there are files.
|
|
1041
1050
|
multi = ResourcePath.mexists(expected_uris, num_workers=3)
|
|
@@ -1054,3 +1063,7 @@ class GenericReadWriteTestCase(_GenericTestCase):
|
|
|
1054
1063
|
multi = ResourcePath.mexists(expected_uris, num_workers=3)
|
|
1055
1064
|
for uri, is_there in multi.items():
|
|
1056
1065
|
self.assertFalse(is_there)
|
|
1066
|
+
|
|
1067
|
+
# Clean up a subset of files that are already gone, but this can
|
|
1068
|
+
# trigger a different code path.
|
|
1069
|
+
ResourcePath.mremove(expected_uris[:5], do_raise=False)
|
lsst/resources/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "29.2025.
|
|
2
|
+
__version__ = "29.2025.2400"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-resources
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.2400
|
|
4
4
|
Summary: An abstraction layer for reading and writing from URI file resources.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
lsst/__init__.py,sha256=9I6UQ9gj-ZcPlvsa0OPBo76UujxXVehVzw9yMAOQvyM,466
|
|
2
2
|
lsst/resources/__init__.py,sha256=BDj6uokvd0ZQNGl-Xgz5gZd83Z0L2gFqGSk0KJpylP8,778
|
|
3
|
-
lsst/resources/_resourcePath.py,sha256=
|
|
3
|
+
lsst/resources/_resourcePath.py,sha256=NLjOjz0ARFJxFwE9fw4HowFV0FrdYBX_jiyXODPFSVY,74273
|
|
4
4
|
lsst/resources/dav.py,sha256=XQMAZgthvtJ4hUXZ0avBAHnAYjMYU3m2DPHDyQVT7a8,31547
|
|
5
5
|
lsst/resources/davutils.py,sha256=xALuMRSvYroqn_Jz6bjnj43b4OgOgCJtNW49kyTtuiw,97983
|
|
6
6
|
lsst/resources/file.py,sha256=-jPuoHvTEtx5tnDyNkfwhWAyX0cTwkuMd-JvJn9EGdE,23226
|
|
7
7
|
lsst/resources/gs.py,sha256=Lpo5GAzH7R7HG8E5RMGOdP4j4hjWJn-k6M3OXj0nHQM,12783
|
|
8
|
-
lsst/resources/http.py,sha256=
|
|
8
|
+
lsst/resources/http.py,sha256=gQNlmCyx9z_0GwrfBaOQ4WfV3Psz6BTfJYdTyolIntA,92441
|
|
9
9
|
lsst/resources/location.py,sha256=x3Tq0x5o1OXYmZDxYBenUG1N71wtDhnjVAr3s2ZEiu8,7937
|
|
10
10
|
lsst/resources/mem.py,sha256=VOWh7XxJPfqKcFdLZSjKEAfORQ2AHZHpxmjT8LniV60,1008
|
|
11
11
|
lsst/resources/packageresource.py,sha256=vnfeRlpVwpC5cDQZE6Lnh8EH6oZy1sH2vLz9ONYjJ4k,6817
|
|
12
12
|
lsst/resources/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
-
lsst/resources/s3.py,sha256=
|
|
13
|
+
lsst/resources/s3.py,sha256=u2rZkg12C3JgIehIJOx69NeipdN6fawhvdZKffhdAF8,29573
|
|
14
14
|
lsst/resources/s3utils.py,sha256=ojWf9BPrK9mhGQ8jvs4_8Nsqf9360e79U5FnPTxe24A,14576
|
|
15
15
|
lsst/resources/schemeless.py,sha256=GfJcKzZ0XIeepfQdW4HPZWiZlSp_ej0SEtSiJTrDUQs,10666
|
|
16
|
-
lsst/resources/tests.py,sha256=
|
|
16
|
+
lsst/resources/tests.py,sha256=G43eaajzIaA6D-zz63vsgYJTeSy9Bso-my_Dh4X34CE,44966
|
|
17
17
|
lsst/resources/utils.py,sha256=6O3Mq7JbPEtqyD2lM77pRpwcPMfV5SxiNMknw-F2vNs,8097
|
|
18
|
-
lsst/resources/version.py,sha256=
|
|
18
|
+
lsst/resources/version.py,sha256=AyivLkwA4FcYO6eiFsfTA_EuqMhydgvzBupNZlQZL-E,55
|
|
19
19
|
lsst/resources/_resourceHandles/__init__.py,sha256=zOcZ8gVEBdAWcHJaZabA8Vdq-wAVcxjbmA_1b1IWM6M,76
|
|
20
20
|
lsst/resources/_resourceHandles/_baseResourceHandle.py,sha256=lQwxDOmFUNJndTxsjpz-HxrQBL0L-z4aXQocHdOEI7c,4676
|
|
21
21
|
lsst/resources/_resourceHandles/_davResourceHandle.py,sha256=12X5-K5KqzG4EV78ZkIIrjcZcFroXy3Y2JQ_N-SDqF0,6616
|
|
22
|
-
lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=
|
|
22
|
+
lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=2nC8tfP_ynAfjpzrtkw_1ahx1CuMEFpZ5mLmofSShUk,3676
|
|
23
23
|
lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=Yami8IVGeru4bLQCag-OvGG0ltz1qyEg57FY4IEB87Y,10995
|
|
24
|
-
lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=
|
|
25
|
-
lsst_resources-29.2025.
|
|
26
|
-
lsst_resources-29.2025.
|
|
27
|
-
lsst_resources-29.2025.
|
|
28
|
-
lsst_resources-29.2025.
|
|
29
|
-
lsst_resources-29.2025.
|
|
30
|
-
lsst_resources-29.2025.
|
|
31
|
-
lsst_resources-29.2025.
|
|
24
|
+
lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=Cp-eBtptskbmthy3DwLKPpYPLvU_lrqtK10X37inHt0,12406
|
|
25
|
+
lsst_resources-29.2025.2400.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
|
|
26
|
+
lsst_resources-29.2025.2400.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
|
|
27
|
+
lsst_resources-29.2025.2400.dist-info/METADATA,sha256=ba8Zw1ARVEELnFMBRB-4pn_WOBtDdS-7GKIqlONe-yQ,2237
|
|
28
|
+
lsst_resources-29.2025.2400.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
29
|
+
lsst_resources-29.2025.2400.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
30
|
+
lsst_resources-29.2025.2400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
31
|
+
lsst_resources-29.2025.2400.dist-info/RECORD,,
|
{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/licenses/COPYRIGHT
RENAMED
|
File without changes
|
{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2400.dist-info}/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|