fsspec 2023.6.0__py3-none-any.whl → 2023.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fsspec/_version.py +3 -3
- fsspec/asyn.py +154 -92
- fsspec/caching.py +1 -1
- fsspec/compression.py +7 -2
- fsspec/core.py +16 -8
- fsspec/generic.py +111 -17
- fsspec/gui.py +4 -2
- fsspec/implementations/cache_mapper.py +80 -0
- fsspec/implementations/cache_metadata.py +232 -0
- fsspec/implementations/cached.py +74 -157
- fsspec/implementations/dirfs.py +3 -1
- fsspec/implementations/http.py +36 -19
- fsspec/implementations/local.py +4 -21
- fsspec/implementations/memory.py +8 -9
- fsspec/implementations/reference.py +8 -8
- fsspec/implementations/sftp.py +6 -2
- fsspec/implementations/smb.py +39 -23
- fsspec/mapping.py +8 -0
- fsspec/registry.py +22 -0
- fsspec/spec.py +164 -96
- fsspec/tests/abstract/__init__.py +147 -0
- fsspec/tests/abstract/common.py +175 -0
- fsspec/tests/abstract/copy.py +250 -56
- fsspec/tests/abstract/get.py +248 -38
- fsspec/tests/abstract/put.py +246 -66
- fsspec/utils.py +25 -8
- {fsspec-2023.6.0.dist-info → fsspec-2023.9.1.dist-info}/METADATA +1 -1
- fsspec-2023.9.1.dist-info/RECORD +54 -0
- fsspec-2023.6.0.dist-info/RECORD +0 -51
- {fsspec-2023.6.0.dist-info → fsspec-2023.9.1.dist-info}/LICENSE +0 -0
- {fsspec-2023.6.0.dist-info → fsspec-2023.9.1.dist-info}/WHEEL +0 -0
- {fsspec-2023.6.0.dist-info → fsspec-2023.9.1.dist-info}/top_level.txt +0 -0
fsspec/tests/abstract/put.py
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
from hashlib import md5
|
|
2
|
+
from itertools import product
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
|
|
7
|
+
|
|
8
|
+
|
|
1
9
|
class AbstractPutTests:
|
|
2
10
|
def test_put_file_to_existing_directory(
|
|
3
11
|
self,
|
|
@@ -6,13 +14,14 @@ class AbstractPutTests:
|
|
|
6
14
|
fs_target,
|
|
7
15
|
local_join,
|
|
8
16
|
local_bulk_operations_scenario_0,
|
|
17
|
+
supports_empty_directories,
|
|
9
18
|
):
|
|
10
19
|
# Copy scenario 1a
|
|
11
20
|
source = local_bulk_operations_scenario_0
|
|
12
21
|
|
|
13
22
|
target = fs_target
|
|
14
23
|
fs.mkdir(target)
|
|
15
|
-
if not
|
|
24
|
+
if not supports_empty_directories:
|
|
16
25
|
# Force target directory to exist by adding a dummy file
|
|
17
26
|
fs.touch(fs_join(target, "dummy"))
|
|
18
27
|
assert fs.isdir(target)
|
|
@@ -58,13 +67,23 @@ class AbstractPutTests:
|
|
|
58
67
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
59
68
|
|
|
60
69
|
def test_put_file_to_file_in_existing_directory(
|
|
61
|
-
self,
|
|
70
|
+
self,
|
|
71
|
+
fs,
|
|
72
|
+
fs_join,
|
|
73
|
+
fs_target,
|
|
74
|
+
local_join,
|
|
75
|
+
supports_empty_directories,
|
|
76
|
+
local_bulk_operations_scenario_0,
|
|
62
77
|
):
|
|
63
78
|
# Copy scenario 1c
|
|
64
79
|
source = local_bulk_operations_scenario_0
|
|
65
80
|
|
|
66
81
|
target = fs_target
|
|
67
82
|
fs.mkdir(target)
|
|
83
|
+
if not supports_empty_directories:
|
|
84
|
+
# Force target directory to exist by adding a dummy file
|
|
85
|
+
fs.touch(fs_join(target, "dummy"))
|
|
86
|
+
assert fs.isdir(target)
|
|
68
87
|
|
|
69
88
|
fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile"))
|
|
70
89
|
assert fs.isfile(fs_join(target, "newfile"))
|
|
@@ -86,14 +105,19 @@ class AbstractPutTests:
|
|
|
86
105
|
assert fs.isfile(fs_join(target, "newdir", "newfile"))
|
|
87
106
|
|
|
88
107
|
def test_put_directory_to_existing_directory(
|
|
89
|
-
self,
|
|
108
|
+
self,
|
|
109
|
+
fs,
|
|
110
|
+
fs_join,
|
|
111
|
+
fs_target,
|
|
112
|
+
local_bulk_operations_scenario_0,
|
|
113
|
+
supports_empty_directories,
|
|
90
114
|
):
|
|
91
115
|
# Copy scenario 1e
|
|
92
116
|
source = local_bulk_operations_scenario_0
|
|
93
117
|
|
|
94
118
|
target = fs_target
|
|
95
119
|
fs.mkdir(target)
|
|
96
|
-
if not
|
|
120
|
+
if not supports_empty_directories:
|
|
97
121
|
# Force target directory to exist by adding a dummy file
|
|
98
122
|
dummy = fs_join(target, "dummy")
|
|
99
123
|
fs.touch(dummy)
|
|
@@ -107,7 +131,7 @@ class AbstractPutTests:
|
|
|
107
131
|
|
|
108
132
|
# Without recursive does nothing
|
|
109
133
|
fs.put(s, t)
|
|
110
|
-
assert fs.ls(target) == [] if
|
|
134
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
111
135
|
|
|
112
136
|
# With recursive
|
|
113
137
|
fs.put(s, t, recursive=True)
|
|
@@ -118,7 +142,14 @@ class AbstractPutTests:
|
|
|
118
142
|
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
|
|
119
143
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
120
144
|
|
|
121
|
-
fs.rm(
|
|
145
|
+
fs.rm(
|
|
146
|
+
[
|
|
147
|
+
fs_join(target, "subfile1"),
|
|
148
|
+
fs_join(target, "subfile2"),
|
|
149
|
+
fs_join(target, "nesteddir"),
|
|
150
|
+
],
|
|
151
|
+
recursive=True,
|
|
152
|
+
)
|
|
122
153
|
else:
|
|
123
154
|
assert fs.isdir(fs_join(target, "subdir"))
|
|
124
155
|
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
|
|
@@ -127,7 +158,7 @@ class AbstractPutTests:
|
|
|
127
158
|
assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile"))
|
|
128
159
|
|
|
129
160
|
fs.rm(fs_join(target, "subdir"), recursive=True)
|
|
130
|
-
assert fs.ls(target) == [] if
|
|
161
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
131
162
|
|
|
132
163
|
# Limit recursive by maxdepth
|
|
133
164
|
fs.put(s, t, recursive=True, maxdepth=1)
|
|
@@ -137,7 +168,13 @@ class AbstractPutTests:
|
|
|
137
168
|
assert not fs.exists(fs_join(target, "nesteddir"))
|
|
138
169
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
139
170
|
|
|
140
|
-
fs.rm(
|
|
171
|
+
fs.rm(
|
|
172
|
+
[
|
|
173
|
+
fs_join(target, "subfile1"),
|
|
174
|
+
fs_join(target, "subfile2"),
|
|
175
|
+
],
|
|
176
|
+
recursive=True,
|
|
177
|
+
)
|
|
141
178
|
else:
|
|
142
179
|
assert fs.isdir(fs_join(target, "subdir"))
|
|
143
180
|
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
|
|
@@ -145,21 +182,21 @@ class AbstractPutTests:
|
|
|
145
182
|
assert not fs.exists(fs_join(target, "subdir", "nesteddir"))
|
|
146
183
|
|
|
147
184
|
fs.rm(fs_join(target, "subdir"), recursive=True)
|
|
148
|
-
assert fs.ls(target) == [] if
|
|
185
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
149
186
|
|
|
150
187
|
def test_put_directory_to_new_directory(
|
|
151
|
-
self,
|
|
188
|
+
self,
|
|
189
|
+
fs,
|
|
190
|
+
fs_join,
|
|
191
|
+
fs_target,
|
|
192
|
+
local_bulk_operations_scenario_0,
|
|
193
|
+
supports_empty_directories,
|
|
152
194
|
):
|
|
153
195
|
# Copy scenario 1f
|
|
154
196
|
source = local_bulk_operations_scenario_0
|
|
155
197
|
|
|
156
198
|
target = fs_target
|
|
157
199
|
fs.mkdir(target)
|
|
158
|
-
if not self.supports_empty_directories():
|
|
159
|
-
# Force target directory to exist by adding a dummy file
|
|
160
|
-
dummy = fs_join(target, "dummy")
|
|
161
|
-
fs.touch(dummy)
|
|
162
|
-
assert fs.isdir(target)
|
|
163
200
|
|
|
164
201
|
for source_slash, target_slash in zip([False, True], [False, True]):
|
|
165
202
|
s = fs_join(source, "subdir")
|
|
@@ -171,7 +208,11 @@ class AbstractPutTests:
|
|
|
171
208
|
|
|
172
209
|
# Without recursive does nothing
|
|
173
210
|
fs.put(s, t)
|
|
174
|
-
|
|
211
|
+
if supports_empty_directories:
|
|
212
|
+
assert fs.ls(target) == []
|
|
213
|
+
else:
|
|
214
|
+
with pytest.raises(FileNotFoundError):
|
|
215
|
+
fs.ls(target)
|
|
175
216
|
|
|
176
217
|
# With recursive
|
|
177
218
|
fs.put(s, t, recursive=True)
|
|
@@ -197,14 +238,20 @@ class AbstractPutTests:
|
|
|
197
238
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
198
239
|
|
|
199
240
|
def test_put_glob_to_existing_directory(
|
|
200
|
-
self,
|
|
241
|
+
self,
|
|
242
|
+
fs,
|
|
243
|
+
fs_join,
|
|
244
|
+
fs_target,
|
|
245
|
+
local_join,
|
|
246
|
+
supports_empty_directories,
|
|
247
|
+
local_bulk_operations_scenario_0,
|
|
201
248
|
):
|
|
202
249
|
# Copy scenario 1g
|
|
203
250
|
source = local_bulk_operations_scenario_0
|
|
204
251
|
|
|
205
252
|
target = fs_target
|
|
206
253
|
fs.mkdir(target)
|
|
207
|
-
if not
|
|
254
|
+
if not supports_empty_directories:
|
|
208
255
|
# Force target directory to exist by adding a dummy file
|
|
209
256
|
dummy = fs_join(target, "dummy")
|
|
210
257
|
fs.touch(dummy)
|
|
@@ -221,29 +268,54 @@ class AbstractPutTests:
|
|
|
221
268
|
assert not fs.exists(fs_join(target, "nesteddir", "nestedfile"))
|
|
222
269
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
223
270
|
|
|
224
|
-
fs.rm(
|
|
225
|
-
|
|
271
|
+
fs.rm(
|
|
272
|
+
[
|
|
273
|
+
fs_join(target, "subfile1"),
|
|
274
|
+
fs_join(target, "subfile2"),
|
|
275
|
+
],
|
|
276
|
+
recursive=True,
|
|
277
|
+
)
|
|
278
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
226
279
|
|
|
227
280
|
# With recursive
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
fs.rm(fs.ls(target, detail=False), recursive=True)
|
|
236
|
-
assert fs.ls(target) == [] if self.supports_empty_directories() else [dummy]
|
|
281
|
+
for glob, recursive in zip(["*", "**"], [True, False]):
|
|
282
|
+
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
|
|
283
|
+
assert fs.isfile(fs_join(target, "subfile1"))
|
|
284
|
+
assert fs.isfile(fs_join(target, "subfile2"))
|
|
285
|
+
assert fs.isdir(fs_join(target, "nesteddir"))
|
|
286
|
+
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
|
|
287
|
+
assert not fs.exists(fs_join(target, "subdir"))
|
|
237
288
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
289
|
+
fs.rm(
|
|
290
|
+
[
|
|
291
|
+
fs_join(target, "subfile1"),
|
|
292
|
+
fs_join(target, "subfile2"),
|
|
293
|
+
fs_join(target, "nesteddir"),
|
|
294
|
+
],
|
|
295
|
+
recursive=True,
|
|
296
|
+
)
|
|
297
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
298
|
+
|
|
299
|
+
# Limit recursive by maxdepth
|
|
300
|
+
fs.put(
|
|
301
|
+
local_join(source, "subdir", glob),
|
|
302
|
+
t,
|
|
303
|
+
recursive=recursive,
|
|
304
|
+
maxdepth=1,
|
|
305
|
+
)
|
|
306
|
+
assert fs.isfile(fs_join(target, "subfile1"))
|
|
307
|
+
assert fs.isfile(fs_join(target, "subfile2"))
|
|
308
|
+
assert not fs.exists(fs_join(target, "nesteddir"))
|
|
309
|
+
assert not fs.exists(fs_join(target, "subdir"))
|
|
244
310
|
|
|
245
|
-
|
|
246
|
-
|
|
311
|
+
fs.rm(
|
|
312
|
+
[
|
|
313
|
+
fs_join(target, "subfile1"),
|
|
314
|
+
fs_join(target, "subfile2"),
|
|
315
|
+
],
|
|
316
|
+
recursive=True,
|
|
317
|
+
)
|
|
318
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
247
319
|
|
|
248
320
|
def test_put_glob_to_new_directory(
|
|
249
321
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
@@ -253,11 +325,6 @@ class AbstractPutTests:
|
|
|
253
325
|
|
|
254
326
|
target = fs_target
|
|
255
327
|
fs.mkdir(target)
|
|
256
|
-
if not self.supports_empty_directories():
|
|
257
|
-
# Force target directory to exist by adding a dummy file
|
|
258
|
-
dummy = fs_join(target, "dummy")
|
|
259
|
-
fs.touch(dummy)
|
|
260
|
-
assert fs.isdir(target)
|
|
261
328
|
|
|
262
329
|
for target_slash in [False, True]:
|
|
263
330
|
t = fs_join(target, "newdir")
|
|
@@ -278,29 +345,81 @@ class AbstractPutTests:
|
|
|
278
345
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
279
346
|
|
|
280
347
|
# With recursive
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
348
|
+
for glob, recursive in zip(["*", "**"], [True, False]):
|
|
349
|
+
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
|
|
350
|
+
assert fs.isdir(fs_join(target, "newdir"))
|
|
351
|
+
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
352
|
+
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
353
|
+
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
|
|
354
|
+
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
|
|
355
|
+
assert not fs.exists(fs_join(target, "subdir"))
|
|
356
|
+
assert not fs.exists(fs_join(target, "newdir", "subdir"))
|
|
357
|
+
|
|
358
|
+
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
359
|
+
assert not fs.exists(fs_join(target, "newdir"))
|
|
360
|
+
|
|
361
|
+
# Limit recursive by maxdepth
|
|
362
|
+
fs.put(
|
|
363
|
+
local_join(source, "subdir", glob),
|
|
364
|
+
t,
|
|
365
|
+
recursive=recursive,
|
|
366
|
+
maxdepth=1,
|
|
367
|
+
)
|
|
368
|
+
assert fs.isdir(fs_join(target, "newdir"))
|
|
369
|
+
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
370
|
+
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
371
|
+
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
|
|
372
|
+
assert not fs.exists(fs_join(target, "subdir"))
|
|
373
|
+
assert not fs.exists(fs_join(target, "newdir", "subdir"))
|
|
289
374
|
|
|
290
|
-
|
|
291
|
-
|
|
375
|
+
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
376
|
+
assert not fs.exists(fs_join(target, "newdir"))
|
|
292
377
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
378
|
+
@pytest.mark.parametrize(
|
|
379
|
+
GLOB_EDGE_CASES_TESTS["argnames"],
|
|
380
|
+
GLOB_EDGE_CASES_TESTS["argvalues"],
|
|
381
|
+
)
|
|
382
|
+
def test_put_glob_edge_cases(
|
|
383
|
+
self,
|
|
384
|
+
path,
|
|
385
|
+
recursive,
|
|
386
|
+
maxdepth,
|
|
387
|
+
expected,
|
|
388
|
+
fs,
|
|
389
|
+
fs_join,
|
|
390
|
+
fs_target,
|
|
391
|
+
local_glob_edge_cases_files,
|
|
392
|
+
local_join,
|
|
393
|
+
fs_sanitize_path,
|
|
394
|
+
):
|
|
395
|
+
# Copy scenario 1g
|
|
396
|
+
source = local_glob_edge_cases_files
|
|
301
397
|
|
|
302
|
-
|
|
303
|
-
|
|
398
|
+
target = fs_target
|
|
399
|
+
|
|
400
|
+
for new_dir, target_slash in product([True, False], [True, False]):
|
|
401
|
+
fs.mkdir(target)
|
|
402
|
+
|
|
403
|
+
t = fs_join(target, "newdir") if new_dir else target
|
|
404
|
+
t = t + "/" if target_slash else t
|
|
405
|
+
|
|
406
|
+
fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
|
|
407
|
+
|
|
408
|
+
output = fs.find(target)
|
|
409
|
+
if new_dir:
|
|
410
|
+
prefixed_expected = [
|
|
411
|
+
fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected
|
|
412
|
+
]
|
|
413
|
+
else:
|
|
414
|
+
prefixed_expected = [
|
|
415
|
+
fs_sanitize_path(fs_join(target, p)) for p in expected
|
|
416
|
+
]
|
|
417
|
+
assert sorted(output) == sorted(prefixed_expected)
|
|
418
|
+
|
|
419
|
+
try:
|
|
420
|
+
fs.rm(target, recursive=True)
|
|
421
|
+
except FileNotFoundError:
|
|
422
|
+
pass
|
|
304
423
|
|
|
305
424
|
def test_put_list_of_files_to_existing_directory(
|
|
306
425
|
self,
|
|
@@ -309,14 +428,14 @@ class AbstractPutTests:
|
|
|
309
428
|
fs_target,
|
|
310
429
|
local_join,
|
|
311
430
|
local_bulk_operations_scenario_0,
|
|
312
|
-
|
|
431
|
+
supports_empty_directories,
|
|
313
432
|
):
|
|
314
433
|
# Copy scenario 2a
|
|
315
434
|
source = local_bulk_operations_scenario_0
|
|
316
435
|
|
|
317
436
|
target = fs_target
|
|
318
437
|
fs.mkdir(target)
|
|
319
|
-
if not
|
|
438
|
+
if not supports_empty_directories:
|
|
320
439
|
# Force target directory to exist by adding a dummy file
|
|
321
440
|
dummy = fs_join(target, "dummy")
|
|
322
441
|
fs.touch(dummy)
|
|
@@ -336,8 +455,15 @@ class AbstractPutTests:
|
|
|
336
455
|
assert fs.isfile(fs_join(target, "file2"))
|
|
337
456
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
338
457
|
|
|
339
|
-
fs.rm(
|
|
340
|
-
|
|
458
|
+
fs.rm(
|
|
459
|
+
[
|
|
460
|
+
fs_join(target, "file1"),
|
|
461
|
+
fs_join(target, "file2"),
|
|
462
|
+
fs_join(target, "subfile1"),
|
|
463
|
+
],
|
|
464
|
+
recursive=True,
|
|
465
|
+
)
|
|
466
|
+
assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
|
|
341
467
|
|
|
342
468
|
def test_put_list_of_files_to_new_directory(
|
|
343
469
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
@@ -395,3 +521,57 @@ class AbstractPutTests:
|
|
|
395
521
|
assert fs.isdir(target)
|
|
396
522
|
assert fs.isfile(fs_join(target, "file"))
|
|
397
523
|
assert not fs.exists(fs_join(target, "src"))
|
|
524
|
+
|
|
525
|
+
def test_put_directory_without_files_with_same_name_prefix(
|
|
526
|
+
self,
|
|
527
|
+
fs,
|
|
528
|
+
fs_join,
|
|
529
|
+
fs_target,
|
|
530
|
+
local_join,
|
|
531
|
+
local_dir_and_file_with_same_name_prefix,
|
|
532
|
+
supports_empty_directories,
|
|
533
|
+
):
|
|
534
|
+
# Create the test dirs
|
|
535
|
+
source = local_dir_and_file_with_same_name_prefix
|
|
536
|
+
target = fs_target
|
|
537
|
+
|
|
538
|
+
# Test without glob
|
|
539
|
+
fs.put(local_join(source, "subdir"), fs_target, recursive=True)
|
|
540
|
+
|
|
541
|
+
assert fs.isfile(fs_join(fs_target, "subfile.txt"))
|
|
542
|
+
assert not fs.isfile(fs_join(fs_target, "subdir.txt"))
|
|
543
|
+
|
|
544
|
+
fs.rm([fs_join(target, "subfile.txt")])
|
|
545
|
+
if supports_empty_directories:
|
|
546
|
+
assert fs.ls(target) == []
|
|
547
|
+
else:
|
|
548
|
+
assert not fs.exists(target)
|
|
549
|
+
|
|
550
|
+
# Test with glob
|
|
551
|
+
fs.put(local_join(source, "subdir*"), fs_target, recursive=True)
|
|
552
|
+
|
|
553
|
+
assert fs.isdir(fs_join(fs_target, "subdir"))
|
|
554
|
+
assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt"))
|
|
555
|
+
assert fs.isfile(fs_join(fs_target, "subdir.txt"))
|
|
556
|
+
|
|
557
|
+
def test_copy_with_source_and_destination_as_list(
|
|
558
|
+
self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names
|
|
559
|
+
):
|
|
560
|
+
# Create the test dir
|
|
561
|
+
source = local_10_files_with_hashed_names
|
|
562
|
+
target = fs_target
|
|
563
|
+
|
|
564
|
+
# Create list of files for source and destination
|
|
565
|
+
source_files = []
|
|
566
|
+
destination_files = []
|
|
567
|
+
for i in range(10):
|
|
568
|
+
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
|
|
569
|
+
source_files.append(local_join(source, f"{hashed_i}.txt"))
|
|
570
|
+
destination_files.append(fs_join(target, f"{hashed_i}.txt"))
|
|
571
|
+
|
|
572
|
+
# Copy and assert order was kept
|
|
573
|
+
fs.put(lpath=source_files, rpath=destination_files)
|
|
574
|
+
|
|
575
|
+
for i in range(10):
|
|
576
|
+
file_content = fs.cat(destination_files[i]).decode("utf-8")
|
|
577
|
+
assert file_content == str(i)
|
fsspec/utils.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import contextlib
|
|
3
4
|
import logging
|
|
4
5
|
import math
|
|
5
6
|
import os
|
|
6
7
|
import pathlib
|
|
7
8
|
import re
|
|
8
9
|
import sys
|
|
9
|
-
|
|
10
|
+
import tempfile
|
|
10
11
|
from functools import partial
|
|
11
12
|
from hashlib import md5
|
|
12
13
|
from importlib.metadata import version
|
|
@@ -343,7 +344,7 @@ def common_prefix(paths):
|
|
|
343
344
|
return "/".join(parts[0][:i])
|
|
344
345
|
|
|
345
346
|
|
|
346
|
-
def other_paths(paths, path2,
|
|
347
|
+
def other_paths(paths, path2, exists=False, flatten=False):
|
|
347
348
|
"""In bulk file operations, construct a new file tree from a list of files
|
|
348
349
|
|
|
349
350
|
Parameters
|
|
@@ -353,10 +354,6 @@ def other_paths(paths, path2, is_dir=None, exists=False, flatten=False):
|
|
|
353
354
|
path2: str or list of str
|
|
354
355
|
Root to construct the new list in. If this is already a list of str, we just
|
|
355
356
|
assert it has the right number of elements.
|
|
356
|
-
is_dir: bool (optional)
|
|
357
|
-
For the special case where the input in one element, whether to regard the value
|
|
358
|
-
as the target path, or as a directory to put a file path within. If None, a
|
|
359
|
-
directory is inferred if the path ends in '/'
|
|
360
357
|
exists: bool (optional)
|
|
361
358
|
For a str destination, it is already exists (and is a dir), files should
|
|
362
359
|
end up inside.
|
|
@@ -370,7 +367,6 @@ def other_paths(paths, path2, is_dir=None, exists=False, flatten=False):
|
|
|
370
367
|
"""
|
|
371
368
|
|
|
372
369
|
if isinstance(path2, str):
|
|
373
|
-
is_dir = is_dir or path2.endswith("/")
|
|
374
370
|
path2 = path2.rstrip("/")
|
|
375
371
|
|
|
376
372
|
if flatten:
|
|
@@ -481,7 +477,7 @@ def mirror_from(origin_name, methods):
|
|
|
481
477
|
return wrapper
|
|
482
478
|
|
|
483
479
|
|
|
484
|
-
@contextmanager
|
|
480
|
+
@contextlib.contextmanager
|
|
485
481
|
def nullcontext(obj):
|
|
486
482
|
yield obj
|
|
487
483
|
|
|
@@ -556,3 +552,24 @@ def file_size(filelike):
|
|
|
556
552
|
return filelike.seek(0, 2)
|
|
557
553
|
finally:
|
|
558
554
|
filelike.seek(pos)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
@contextlib.contextmanager
|
|
558
|
+
def atomic_write(path: str, mode: str = "wb"):
|
|
559
|
+
"""
|
|
560
|
+
A context manager that opens a temporary file next to `path` and, on exit,
|
|
561
|
+
replaces `path` with the temporary file, thereby updating `path`
|
|
562
|
+
atomically.
|
|
563
|
+
"""
|
|
564
|
+
fd, fn = tempfile.mkstemp(
|
|
565
|
+
dir=os.path.dirname(path), prefix=os.path.basename(path) + "-"
|
|
566
|
+
)
|
|
567
|
+
try:
|
|
568
|
+
with open(fd, mode) as fp:
|
|
569
|
+
yield fp
|
|
570
|
+
except BaseException:
|
|
571
|
+
with contextlib.suppress(FileNotFoundError):
|
|
572
|
+
os.unlink(fn)
|
|
573
|
+
raise
|
|
574
|
+
else:
|
|
575
|
+
os.replace(fn, path)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
fsspec/__init__.py,sha256=vbKs09bNlgkk-bfKZOzzvAVJ2-cYNvrmN07R48344F4,1800
|
|
2
|
+
fsspec/_version.py,sha256=xzm3AbPA2trvGGqc2MTnWuPLVLxQMDXP482xNuytWYU,500
|
|
3
|
+
fsspec/archive.py,sha256=FSbEj8w26fcCfuQjQZ_GMLQy5-O5M_KrZkrEdsksato,2406
|
|
4
|
+
fsspec/asyn.py,sha256=6djXIMTdRmke1AxcFsw8QXJWrmYflaSUOq4i_NBnS_M,36593
|
|
5
|
+
fsspec/caching.py,sha256=19qWNHl1dCh0zZSRhqFYIO7Qqco_KOW93L2QPWUnzOc,26373
|
|
6
|
+
fsspec/callbacks.py,sha256=qmD1v-WWxWmTmcUkEadq-_F_n3OGp9JYarjupUq_j3o,6358
|
|
7
|
+
fsspec/compression.py,sha256=-A_TjrGys1jSBQ1IPoPWOdE6X8HJHOSUv94rEvBj9_4,4905
|
|
8
|
+
fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
|
|
9
|
+
fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
|
|
10
|
+
fsspec/core.py,sha256=RrybaRFicCV42YkljakksQp9ZkPgOqLE2lpN6UtvblE,21694
|
|
11
|
+
fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
|
|
12
|
+
fsspec/exceptions.py,sha256=s5eA2wIwzj-aeV0i_KDXsBaIhJJRKzmMGUGwuBHTnS4,348
|
|
13
|
+
fsspec/fuse.py,sha256=VnXybwERLONVTgqTTi2ZLVwa2_CzORkRPJG3aHZgAEw,10187
|
|
14
|
+
fsspec/generic.py,sha256=DfRoESTne-DejjV_4VwYJlTqFECYikNIf9brwWp9wsU,13269
|
|
15
|
+
fsspec/gui.py,sha256=XM8ni7YxXC4gjK14Uf_b4HrJIh73WOZot9893B_kzf0,13888
|
|
16
|
+
fsspec/mapping.py,sha256=xr0JkYBf-wwzJcrBi7QIi2dyuZKFD6NWo5UVUHa61BU,8166
|
|
17
|
+
fsspec/parquet.py,sha256=i4H3EU3K1Q6jp8sqjFji6a6gKnlOEZufaa7DRNE5X-4,19516
|
|
18
|
+
fsspec/registry.py,sha256=cOxTT6MpRBif2AeW1x3tWsxYvZBHBsxEwYGNADBBxSM,11012
|
|
19
|
+
fsspec/spec.py,sha256=KqH2YhgleKcBE7WMg1W0QOvK8biFICEYIzidQH4TQao,66742
|
|
20
|
+
fsspec/transaction.py,sha256=KzJVfZxQZ1HeXiBbwC7ba3QtHMRy00yyThXZYP-vMQU,2195
|
|
21
|
+
fsspec/utils.py,sha256=8ste5YfMzOx-DEgKo8jAiGnrFWZkpaboarpMlBGzB3Y,17434
|
|
22
|
+
fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
+
fsspec/implementations/arrow.py,sha256=1d-c5KceQJxm8QXML8fFXHvQx0wstG-tNJNsrgMX_CI,8240
|
|
24
|
+
fsspec/implementations/cache_mapper.py,sha256=nE_sY3vw-jJbeBcAP6NGtacP3jHW_7EcG3yUSf0A-4Y,2502
|
|
25
|
+
fsspec/implementations/cache_metadata.py,sha256=ZvyA7Y3KK-5Ct4E5pELzD6mH_5T03XqaKVT96qYDADU,8576
|
|
26
|
+
fsspec/implementations/cached.py,sha256=ei5j6Ll5NKwWw_XN8L50kGdmIg63jKGXBYl6j51JfFk,27427
|
|
27
|
+
fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
|
|
28
|
+
fsspec/implementations/dbfs.py,sha256=0ndCE2OQqrWv6Y8ETufxOQ9ymIIO2JA_Q82bnilqTaw,14660
|
|
29
|
+
fsspec/implementations/dirfs.py,sha256=8EEgKin5JgFBqzHaKig7ipiFAZJvbChUX_vpC_jagoY,11136
|
|
30
|
+
fsspec/implementations/ftp.py,sha256=4oAragG80N75O78vVBWVIE1bPB1SGPnw_c7jFNOljsk,11463
|
|
31
|
+
fsspec/implementations/git.py,sha256=Sn96xSjXAl1aA-Gf1DByTd23tiObqv-uAQGc_mryiEw,4034
|
|
32
|
+
fsspec/implementations/github.py,sha256=1d7L22BococztJHKFZvzwvUQZ2cqcBHLhH-4tEDoL0A,7328
|
|
33
|
+
fsspec/implementations/http.py,sha256=bKbA3CiXeyeQGIBPZZVon5IxR5QmpHjWaOmU-jlj7MA,30134
|
|
34
|
+
fsspec/implementations/http_sync.py,sha256=mWHICWqhAqX-WSfdQeWKY5xfhekkVlx3qEl9bY1Y_yA,28555
|
|
35
|
+
fsspec/implementations/jupyter.py,sha256=maOaSOcu3lXI45KCfS3KWkVcM2EZJJjNSICQCXupLK0,3816
|
|
36
|
+
fsspec/implementations/libarchive.py,sha256=lqvLmqG_pSEhkirTEsk1guwtU9_WZ-eOwzzBrPFz84U,7248
|
|
37
|
+
fsspec/implementations/local.py,sha256=dXnZgjjc4wz5ndIRWREh_DzwY_m5qWhPXFFlRhHdWt8,13146
|
|
38
|
+
fsspec/implementations/memory.py,sha256=rtUWe5ooaSs0AWegpZ5DMdkcshGu9-GEQPwavbv1YE0,9741
|
|
39
|
+
fsspec/implementations/reference.py,sha256=3rFxJjLaybO2d7hK8G9K3z2wze9hihLO_dWpTp5ewUs,40765
|
|
40
|
+
fsspec/implementations/sftp.py,sha256=ZsFx4XlyMOx2fT57Pz-_fEz5qLfdPD4ljS3ihuDquAo,5570
|
|
41
|
+
fsspec/implementations/smb.py,sha256=1JeZb_34iqgefEBiJwloa991-4gOKIHxkpS0g6wX43w,10633
|
|
42
|
+
fsspec/implementations/tar.py,sha256=5ZpUp4E3SYbqrwAX2ezvZJqUoZO74Pjb9FpF8o1YBGs,4071
|
|
43
|
+
fsspec/implementations/webhdfs.py,sha256=kJ_gya30beO5fEwlNi8G8CyuZJpfIQ1oe1jOxgF6RjQ,15391
|
|
44
|
+
fsspec/implementations/zip.py,sha256=rQHoOCtT1-Xx3AvNFn5iJ9UVF7Wyiv9wj5MUTIks_nw,4216
|
|
45
|
+
fsspec/tests/abstract/__init__.py,sha256=i1wcFixV6QhOwdoB24c8oXjzobISNqiKVz9kl2DvAY8,10028
|
|
46
|
+
fsspec/tests/abstract/common.py,sha256=X1ijH_pdMc9uVpZtgGj1P-2Zj9VIY-Y0tG3u1vTGcpE,4963
|
|
47
|
+
fsspec/tests/abstract/copy.py,sha256=nyCp1Q9apHzti2_UPDh3HzVhRmV7dciD-3dq-wM7JuU,19643
|
|
48
|
+
fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755
|
|
49
|
+
fsspec/tests/abstract/put.py,sha256=hEf-yuMWBOT7B6eWcck3tMyJWzdVXtxkY-O6LUt1KAE,20877
|
|
50
|
+
fsspec-2023.9.1.dist-info/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
|
|
51
|
+
fsspec-2023.9.1.dist-info/METADATA,sha256=_OX1UMDXGpI_wzg2avUBO3X-AJq5HR_Y0Cz1CQJManU,6710
|
|
52
|
+
fsspec-2023.9.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
53
|
+
fsspec-2023.9.1.dist-info/top_level.txt,sha256=blt2pDrQDwN3Gklcw13CSPLQRd6aaOgJ8AxqrW395MI,7
|
|
54
|
+
fsspec-2023.9.1.dist-info/RECORD,,
|
fsspec-2023.6.0.dist-info/RECORD
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
fsspec/__init__.py,sha256=vbKs09bNlgkk-bfKZOzzvAVJ2-cYNvrmN07R48344F4,1800
|
|
2
|
-
fsspec/_version.py,sha256=NH6Y4s5Yi4eBruHt22MKbm1HyvY1UF6vdxcCMqS-pyA,500
|
|
3
|
-
fsspec/archive.py,sha256=FSbEj8w26fcCfuQjQZ_GMLQy5-O5M_KrZkrEdsksato,2406
|
|
4
|
-
fsspec/asyn.py,sha256=UerJX18MeZPub5hnYm8KrTPC3nX40-fnZ4bQ2yjTvKw,33978
|
|
5
|
-
fsspec/caching.py,sha256=17xqzc46i0YS_Moyu4JySEoge9TAbzJiLjhK_ieeO3A,26379
|
|
6
|
-
fsspec/callbacks.py,sha256=qmD1v-WWxWmTmcUkEadq-_F_n3OGp9JYarjupUq_j3o,6358
|
|
7
|
-
fsspec/compression.py,sha256=wvC1jmsmo6C6zZiStfBkaTaFBAABMHr0wgJ6WWjavPM,4856
|
|
8
|
-
fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
|
|
9
|
-
fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
|
|
10
|
-
fsspec/core.py,sha256=fKaEhCUAUiA4EgpB4rRXB4wBCmhG4_FGlkWcjkYa5nE,21578
|
|
11
|
-
fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
|
|
12
|
-
fsspec/exceptions.py,sha256=s5eA2wIwzj-aeV0i_KDXsBaIhJJRKzmMGUGwuBHTnS4,348
|
|
13
|
-
fsspec/fuse.py,sha256=VnXybwERLONVTgqTTi2ZLVwa2_CzORkRPJG3aHZgAEw,10187
|
|
14
|
-
fsspec/generic.py,sha256=TXcN_hSgZkYS2IGyt-QVSIw_unJkFtReuaBrgOCbFko,10211
|
|
15
|
-
fsspec/gui.py,sha256=b06FSpOgMrSe00dHZr7oBrY5nmL_3QOhrHimLQzfCfU,13860
|
|
16
|
-
fsspec/mapping.py,sha256=tWBRfnQTxXdX4r0j9CDREd3ZkEu7EPRvwRMMvVKE5ig,7879
|
|
17
|
-
fsspec/parquet.py,sha256=i4H3EU3K1Q6jp8sqjFji6a6gKnlOEZufaa7DRNE5X-4,19516
|
|
18
|
-
fsspec/registry.py,sha256=b8r-wlESPPfV-yFxkBHW_n-kvHWuYG5-0swJS4v6lCE,10255
|
|
19
|
-
fsspec/spec.py,sha256=LQ440aSvDovaWsVMIn2JkklZbCAKxKiF4O_NKbAq9SQ,63999
|
|
20
|
-
fsspec/transaction.py,sha256=KzJVfZxQZ1HeXiBbwC7ba3QtHMRy00yyThXZYP-vMQU,2195
|
|
21
|
-
fsspec/utils.py,sha256=np10Ju6YQ-67mpcFlhssQaL5r1ACeKsCcisk_14QGfQ,17155
|
|
22
|
-
fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
-
fsspec/implementations/arrow.py,sha256=1d-c5KceQJxm8QXML8fFXHvQx0wstG-tNJNsrgMX_CI,8240
|
|
24
|
-
fsspec/implementations/cached.py,sha256=kjva-cnkPwSFzo1FmTeJQiJ5-ZtWYCH2Wk5Ay1SOfJU,30534
|
|
25
|
-
fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
|
|
26
|
-
fsspec/implementations/dbfs.py,sha256=0ndCE2OQqrWv6Y8ETufxOQ9ymIIO2JA_Q82bnilqTaw,14660
|
|
27
|
-
fsspec/implementations/dirfs.py,sha256=GRplS-Dh6ETJQHSrEFuU3s72JR-JIwWyszu1SrCk8wI,11092
|
|
28
|
-
fsspec/implementations/ftp.py,sha256=4oAragG80N75O78vVBWVIE1bPB1SGPnw_c7jFNOljsk,11463
|
|
29
|
-
fsspec/implementations/git.py,sha256=Sn96xSjXAl1aA-Gf1DByTd23tiObqv-uAQGc_mryiEw,4034
|
|
30
|
-
fsspec/implementations/github.py,sha256=1d7L22BococztJHKFZvzwvUQZ2cqcBHLhH-4tEDoL0A,7328
|
|
31
|
-
fsspec/implementations/http.py,sha256=GttNyetwvL9cFJa5616ZuGwYzqKenfbdZw7xk6qYqEw,29225
|
|
32
|
-
fsspec/implementations/http_sync.py,sha256=mWHICWqhAqX-WSfdQeWKY5xfhekkVlx3qEl9bY1Y_yA,28555
|
|
33
|
-
fsspec/implementations/jupyter.py,sha256=maOaSOcu3lXI45KCfS3KWkVcM2EZJJjNSICQCXupLK0,3816
|
|
34
|
-
fsspec/implementations/libarchive.py,sha256=lqvLmqG_pSEhkirTEsk1guwtU9_WZ-eOwzzBrPFz84U,7248
|
|
35
|
-
fsspec/implementations/local.py,sha256=4h6C4H6vnAQ4hVJKjjbgEDT_lEUZaTkpmjaUz0RDGCc,13797
|
|
36
|
-
fsspec/implementations/memory.py,sha256=je-2VUT98PV9DRgeHZo1wtTMY2TieKLWvP3n_mCoMfM,9703
|
|
37
|
-
fsspec/implementations/reference.py,sha256=hZW68dAxHjutxoTrbpvak4WQp2xq3-v79qbEm6WrKKM,40766
|
|
38
|
-
fsspec/implementations/sftp.py,sha256=yYD2niFPv9dFnnSyn0Wcp3qklT_wYO2b09zISxygLOc,5464
|
|
39
|
-
fsspec/implementations/smb.py,sha256=DmiwQL7sg1RXM8ko9BqGaJjVrSDxmhBj9RNPJZt-SBs,10023
|
|
40
|
-
fsspec/implementations/tar.py,sha256=5ZpUp4E3SYbqrwAX2ezvZJqUoZO74Pjb9FpF8o1YBGs,4071
|
|
41
|
-
fsspec/implementations/webhdfs.py,sha256=kJ_gya30beO5fEwlNi8G8CyuZJpfIQ1oe1jOxgF6RjQ,15391
|
|
42
|
-
fsspec/implementations/zip.py,sha256=rQHoOCtT1-Xx3AvNFn5iJ9UVF7Wyiv9wj5MUTIks_nw,4216
|
|
43
|
-
fsspec/tests/abstract/__init__.py,sha256=WnY2HJC3dKWpuXEmyzBQyA2QnfaU-HXpK5jT9kuXxDQ,4702
|
|
44
|
-
fsspec/tests/abstract/copy.py,sha256=HB-mApp4ye34IeswDCkM8alcXQd63wtif6Vgj30Gcrw,13844
|
|
45
|
-
fsspec/tests/abstract/get.py,sha256=EyclZXxZNWo_IpZlDrrLg_GTkSGYvIQPWI9DofSVQDk,12310
|
|
46
|
-
fsspec/tests/abstract/put.py,sha256=Sg4Go5mI3rB4HC43s--yERnhZ3v5ezw9bP93ODcMuw0,15792
|
|
47
|
-
fsspec-2023.6.0.dist-info/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
|
|
48
|
-
fsspec-2023.6.0.dist-info/METADATA,sha256=jXoTGDuStQuJANkFAWuqOr_UQBDD0_zcWdHpLkz1OFk,6710
|
|
49
|
-
fsspec-2023.6.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
50
|
-
fsspec-2023.6.0.dist-info/top_level.txt,sha256=blt2pDrQDwN3Gklcw13CSPLQRd6aaOgJ8AxqrW395MI,7
|
|
51
|
-
fsspec-2023.6.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|