dazzle-filekit 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/PKG-INFO +1 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/__init__.py +34 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/links.py +131 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/metadata.py +152 -20
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/operations.py +165 -9
- dazzle_filekit-0.3.4/dazzle_filekit/pathenv.py +123 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/paths.py +14 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/utils/compat.py +12 -2
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/utils/validation.py +18 -3
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/verification.py +231 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit.egg-info/PKG-INFO +1 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit.egg-info/SOURCES.txt +8 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/pyproject.toml +1 -1
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/setup.py +1 -1
- dazzle_filekit-0.3.4/tests/test_atomic_stream_writer.py +130 -0
- dazzle_filekit-0.3.4/tests/test_atomic_write_retry.py +30 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_import_stability.py +9 -0
- dazzle_filekit-0.3.4/tests/test_junction_detection_improvements.py +147 -0
- dazzle_filekit-0.3.4/tests/test_link_fidelity_v034.py +182 -0
- dazzle_filekit-0.3.4/tests/test_native_hash_backend.py +183 -0
- dazzle_filekit-0.3.4/tests/test_pathenv.py +152 -0
- dazzle_filekit-0.3.4/tests/test_pathenv_edgecases.py +188 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/LICENSE +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/README.md +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/_fallback.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/content.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/platform/__init__.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/platform/unix.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/platform/windows.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/utils/__init__.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/utils/disk.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit/utils/logger.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit.egg-info/dependency_links.txt +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit.egg-info/requires.txt +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/dazzle_filekit.egg-info/top_level.txt +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/setup.cfg +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_ads_and_junction_v024.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_auto_naming.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_compat.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_content_and_process.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_disk.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_fallback.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_link_primitives.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_metadata_v024_roundtrip.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_operations_primitives_v024.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_paths_platform_simulation.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_paths_v024_enrichments.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_phase_b_absorb_and_v9.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_phase_c_renames.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_phase_d_typeddicts.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_resolver_ops.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_smoke.py +0 -0
- {dazzle_filekit-0.3.2 → dazzle_filekit-0.3.4}/tests/test_symlink_timestamps_v031.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dazzle-filekit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Cross-platform file operations toolkit with path handling, verification, and metadata preservation
|
|
5
5
|
Author: Dustin
|
|
6
6
|
Author-email: Dustin Darcy <6962246+djdarcy@users.noreply.github.com>
|
|
@@ -76,10 +76,15 @@ from .operations import (
|
|
|
76
76
|
atomic_write_text,
|
|
77
77
|
atomic_write_json,
|
|
78
78
|
copy_tree_preserving_links,
|
|
79
|
+
# Streaming atomic writer (from dazzlesum's MonolithicWriter)
|
|
80
|
+
AtomicStreamWriter,
|
|
79
81
|
)
|
|
80
82
|
|
|
81
83
|
from .verification import (
|
|
82
84
|
calculate_file_hash,
|
|
85
|
+
# Native checksum-tool backend (from dazzlesum)
|
|
86
|
+
detect_native_hash_tool,
|
|
87
|
+
calculate_file_hash_native,
|
|
83
88
|
verify_file_hash,
|
|
84
89
|
verify_files_with_manifest,
|
|
85
90
|
calculate_directory_hashes,
|
|
@@ -115,11 +120,25 @@ from .links import (
|
|
|
115
120
|
detect_link_type,
|
|
116
121
|
read_link_target,
|
|
117
122
|
create_junction,
|
|
123
|
+
create_junction_raw,
|
|
118
124
|
create_hardlink,
|
|
119
125
|
remove_link,
|
|
120
126
|
)
|
|
121
127
|
|
|
122
|
-
|
|
128
|
+
# v0.3.3 declared-platform PATH-value helpers (dazzlecmd#103 homing) --
|
|
129
|
+
# the provenance-typed sibling of the host-directional normalizers.
|
|
130
|
+
from . import pathenv # noqa: F401
|
|
131
|
+
from .pathenv import (
|
|
132
|
+
PLATFORM_WINDOWS,
|
|
133
|
+
PLATFORM_POSIX,
|
|
134
|
+
host_path_platform,
|
|
135
|
+
split_path_value,
|
|
136
|
+
normalize_path_entry,
|
|
137
|
+
path_value_contains,
|
|
138
|
+
append_path_value,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
__version__ = '0.3.4'
|
|
123
142
|
|
|
124
143
|
def configure_logging(level=logging.INFO, log_file=None):
|
|
125
144
|
"""
|
|
@@ -236,6 +255,7 @@ __all__ = [
|
|
|
236
255
|
'atomic_write_text',
|
|
237
256
|
'atomic_write_json',
|
|
238
257
|
'copy_tree_preserving_links',
|
|
258
|
+
'AtomicStreamWriter',
|
|
239
259
|
|
|
240
260
|
# Rich metadata module (v0.2.4)
|
|
241
261
|
'metadata',
|
|
@@ -256,11 +276,24 @@ __all__ = [
|
|
|
256
276
|
'detect_link_type',
|
|
257
277
|
'read_link_target',
|
|
258
278
|
'create_junction',
|
|
279
|
+
'create_junction_raw',
|
|
259
280
|
'create_hardlink',
|
|
260
281
|
'remove_link',
|
|
261
282
|
|
|
283
|
+
# Declared-platform PATH-value helpers (v0.3.3)
|
|
284
|
+
'pathenv',
|
|
285
|
+
'PLATFORM_WINDOWS',
|
|
286
|
+
'PLATFORM_POSIX',
|
|
287
|
+
'host_path_platform',
|
|
288
|
+
'split_path_value',
|
|
289
|
+
'normalize_path_entry',
|
|
290
|
+
'path_value_contains',
|
|
291
|
+
'append_path_value',
|
|
292
|
+
|
|
262
293
|
# Verification functions
|
|
263
294
|
'calculate_file_hash',
|
|
295
|
+
'detect_native_hash_tool',
|
|
296
|
+
'calculate_file_hash_native',
|
|
264
297
|
'verify_file_hash',
|
|
265
298
|
'verify_files_with_manifest',
|
|
266
299
|
'calculate_directory_hashes',
|
|
@@ -97,10 +97,20 @@ def detect_link_type(path: Union[str, Path]) -> Optional[str]:
|
|
|
97
97
|
Returns ``'symlink'``, ``'junction'`` (Windows), or ``'hardlink'`` (a file
|
|
98
98
|
whose ``st_nlink > 1``). Hard links are otherwise indistinguishable from
|
|
99
99
|
regular files, so a single-name file is reported as ``None``.
|
|
100
|
+
|
|
101
|
+
Existence is checked with ``os.lstat`` (the link ITSELF), not
|
|
102
|
+
``Path.exists()`` (which follows the link): a junction whose target is
|
|
103
|
+
missing is still a junction and must be reported as one, so
|
|
104
|
+
``analyze_link`` can mark it broken. The previous ``exists()`` gate
|
|
105
|
+
special-cased broken symlinks but silently dropped broken junctions
|
|
106
|
+
(contributed from dazzlesum, whose walker must refuse to traverse
|
|
107
|
+
broken junctions rather than treat them as plain directories).
|
|
100
108
|
"""
|
|
101
109
|
p = Path(path)
|
|
102
110
|
|
|
103
|
-
|
|
111
|
+
try:
|
|
112
|
+
os.lstat(p)
|
|
113
|
+
except OSError:
|
|
104
114
|
return None
|
|
105
115
|
|
|
106
116
|
if p.is_symlink():
|
|
@@ -276,6 +286,126 @@ def create_junction(
|
|
|
276
286
|
return False
|
|
277
287
|
|
|
278
288
|
|
|
289
|
+
def create_junction_raw(
|
|
290
|
+
target: Union[str, Path],
|
|
291
|
+
link: Union[str, Path],
|
|
292
|
+
force: bool = False,
|
|
293
|
+
) -> bool:
|
|
294
|
+
"""Create an NTFS junction by writing the mount-point reparse buffer
|
|
295
|
+
directly (``FSCTL_SET_REPARSE_POINT``), no subprocess.
|
|
296
|
+
|
|
297
|
+
Differences from :func:`create_junction` (PowerShell ``New-Item``):
|
|
298
|
+
|
|
299
|
+
- The target does NOT need to exist -- ``New-Item`` refuses a missing
|
|
300
|
+
target, which makes intentionally-broken junctions impossible to
|
|
301
|
+
recreate when mirroring a tree.
|
|
302
|
+
- The target string is stored VERBATIM: ``\\\\?\\C:\\x`` (as returned by
|
|
303
|
+
``os.readlink`` on another junction) and plain ``C:\\x`` are both
|
|
304
|
+
accepted; the NT-namespace SubstituteName (``\\??\\...``) and the
|
|
305
|
+
user-facing PrintName are derived without other normalization, so
|
|
306
|
+
``os.readlink`` round-trips byte-identically.
|
|
307
|
+
|
|
308
|
+
Unprivileged (junctions never require SeCreateSymbolicLinkPrivilege).
|
|
309
|
+
Windows-only, directory-only. Returns True on success.
|
|
310
|
+
"""
|
|
311
|
+
raw_target = os.fspath(target)
|
|
312
|
+
link_path = Path(link)
|
|
313
|
+
|
|
314
|
+
if os.name != "nt":
|
|
315
|
+
logger.error("Junctions are only supported on Windows")
|
|
316
|
+
return False
|
|
317
|
+
|
|
318
|
+
if link_path.exists() or link_path.is_symlink():
|
|
319
|
+
if not force:
|
|
320
|
+
logger.warning(f"Link path already exists: {link_path}")
|
|
321
|
+
return False
|
|
322
|
+
if not _remove_existing(link_path):
|
|
323
|
+
return False
|
|
324
|
+
|
|
325
|
+
# SubstituteName lives in the NT namespace (\??\); PrintName is the
|
|
326
|
+
# user-facing form. os.readlink() renders \??\ as \\?\, so accept both.
|
|
327
|
+
if raw_target.startswith("\\??\\"):
|
|
328
|
+
substitute = raw_target
|
|
329
|
+
print_name = raw_target[4:]
|
|
330
|
+
elif raw_target.startswith("\\\\?\\"):
|
|
331
|
+
substitute = "\\??\\" + raw_target[4:]
|
|
332
|
+
print_name = raw_target[4:]
|
|
333
|
+
else:
|
|
334
|
+
substitute = "\\??\\" + raw_target
|
|
335
|
+
print_name = raw_target
|
|
336
|
+
|
|
337
|
+
try:
|
|
338
|
+
link_path.parent.mkdir(parents=True, exist_ok=True)
|
|
339
|
+
os.mkdir(link_path)
|
|
340
|
+
except OSError as e:
|
|
341
|
+
logger.error(f"Failed to create junction directory {link_path}: {e}")
|
|
342
|
+
return False
|
|
343
|
+
|
|
344
|
+
try:
|
|
345
|
+
import ctypes
|
|
346
|
+
from ctypes import wintypes
|
|
347
|
+
|
|
348
|
+
kernel32 = ctypes.windll.kernel32
|
|
349
|
+
|
|
350
|
+
GENERIC_WRITE = 0x40000000
|
|
351
|
+
OPEN_EXISTING = 3
|
|
352
|
+
FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
|
|
353
|
+
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
|
354
|
+
FILE_SHARE_READ_WRITE_DELETE = 0x01 | 0x02 | 0x04
|
|
355
|
+
FSCTL_SET_REPARSE_POINT = 0x000900A4
|
|
356
|
+
IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
|
|
357
|
+
|
|
358
|
+
subst_b = substitute.encode("utf-16-le")
|
|
359
|
+
print_b = print_name.encode("utf-16-le")
|
|
360
|
+
# MountPointReparseBuffer: 4 offset/length WORDs + PathBuffer holding
|
|
361
|
+
# SubstituteName \0 PrintName \0 (UTF-16LE, lengths exclude the NULs).
|
|
362
|
+
path_buffer = subst_b + b"\x00\x00" + print_b + b"\x00\x00"
|
|
363
|
+
reparse_data_length = 8 + len(path_buffer)
|
|
364
|
+
header = (
|
|
365
|
+
IO_REPARSE_TAG_MOUNT_POINT.to_bytes(4, "little")
|
|
366
|
+
+ reparse_data_length.to_bytes(2, "little")
|
|
367
|
+
+ b"\x00\x00" # Reserved
|
|
368
|
+
+ (0).to_bytes(2, "little") # SubstituteNameOffset
|
|
369
|
+
+ len(subst_b).to_bytes(2, "little") # SubstituteNameLength
|
|
370
|
+
+ (len(subst_b) + 2).to_bytes(2, "little") # PrintNameOffset
|
|
371
|
+
+ len(print_b).to_bytes(2, "little") # PrintNameLength
|
|
372
|
+
)
|
|
373
|
+
buf = header + path_buffer
|
|
374
|
+
|
|
375
|
+
handle = kernel32.CreateFileW(
|
|
376
|
+
str(link_path), GENERIC_WRITE,
|
|
377
|
+
FILE_SHARE_READ_WRITE_DELETE,
|
|
378
|
+
None, OPEN_EXISTING,
|
|
379
|
+
FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
|
|
380
|
+
None,
|
|
381
|
+
)
|
|
382
|
+
if handle == wintypes.HANDLE(-1).value:
|
|
383
|
+
raise OSError(f"CreateFileW failed for {link_path}")
|
|
384
|
+
try:
|
|
385
|
+
bytes_returned = wintypes.DWORD(0)
|
|
386
|
+
ok = kernel32.DeviceIoControl(
|
|
387
|
+
handle, FSCTL_SET_REPARSE_POINT,
|
|
388
|
+
buf, len(buf), None, 0,
|
|
389
|
+
ctypes.byref(bytes_returned), None,
|
|
390
|
+
)
|
|
391
|
+
if not ok:
|
|
392
|
+
raise OSError(
|
|
393
|
+
f"FSCTL_SET_REPARSE_POINT failed "
|
|
394
|
+
f"(winerror={ctypes.get_last_error()})"
|
|
395
|
+
)
|
|
396
|
+
finally:
|
|
397
|
+
kernel32.CloseHandle(handle)
|
|
398
|
+
logger.debug(f"Created raw junction: {link_path} -> {raw_target}")
|
|
399
|
+
return True
|
|
400
|
+
except Exception as e: # noqa: BLE001 - report and clean up the stub dir
|
|
401
|
+
logger.error(f"Raw junction creation failed for {link_path}: {e}")
|
|
402
|
+
try:
|
|
403
|
+
os.rmdir(link_path)
|
|
404
|
+
except OSError:
|
|
405
|
+
pass
|
|
406
|
+
return False
|
|
407
|
+
|
|
408
|
+
|
|
279
409
|
def create_hardlink(
|
|
280
410
|
target: Union[str, Path],
|
|
281
411
|
link: Union[str, Path],
|
|
@@ -194,8 +194,93 @@ def _collect_windows_metadata(path: Path) -> Dict[str, Any]:
|
|
|
194
194
|
logger.error(f"Error collecting Windows metadata for {path}: {e}")
|
|
195
195
|
return windows_metadata
|
|
196
196
|
|
|
197
|
+
_FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
|
198
|
+
|
|
199
|
+
# Nanoseconds between 1601-01-01 (FILETIME epoch) and 1970-01-01 (Unix epoch)
|
|
200
|
+
_FILETIME_EPOCH_OFFSET_NS = 11644473600 * 10**9
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _set_link_times_exact_windows(
|
|
204
|
+
path_str: str,
|
|
205
|
+
created_ns: Optional[int],
|
|
206
|
+
accessed_ns: Optional[int],
|
|
207
|
+
modified_ns: Optional[int],
|
|
208
|
+
) -> bool:
|
|
209
|
+
"""Set timestamps on a link node with exact 100ns precision.
|
|
210
|
+
|
|
211
|
+
Opens the node with ``FILE_FLAG_OPEN_REPARSE_POINT`` (never follows to
|
|
212
|
+
the target) and calls ``SetFileTime`` with FILETIMEs computed directly
|
|
213
|
+
from integer nanoseconds -- the float/datetime/pywintypes chain rounds
|
|
214
|
+
to microseconds or worse, which a mirror's verify pass would flag.
|
|
215
|
+
Pure ctypes; works without pywin32. Returns True on success.
|
|
216
|
+
"""
|
|
217
|
+
try:
|
|
218
|
+
import ctypes
|
|
219
|
+
from ctypes import wintypes
|
|
220
|
+
|
|
221
|
+
kernel32 = ctypes.windll.kernel32
|
|
222
|
+
|
|
223
|
+
FILE_WRITE_ATTRIBUTES = 0x100
|
|
224
|
+
OPEN_EXISTING = 3
|
|
225
|
+
FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
|
|
226
|
+
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
|
227
|
+
FILE_SHARE_ALL = 0x01 | 0x02 | 0x04
|
|
228
|
+
|
|
229
|
+
def _ft(ns: Optional[int]):
|
|
230
|
+
if ns is None:
|
|
231
|
+
return None
|
|
232
|
+
ticks = (ns + _FILETIME_EPOCH_OFFSET_NS) // 100
|
|
233
|
+
return wintypes.FILETIME(
|
|
234
|
+
ticks & 0xFFFFFFFF, (ticks >> 32) & 0xFFFFFFFF
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
handle = kernel32.CreateFileW(
|
|
238
|
+
path_str, FILE_WRITE_ATTRIBUTES, FILE_SHARE_ALL, None,
|
|
239
|
+
OPEN_EXISTING,
|
|
240
|
+
FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
|
|
241
|
+
None,
|
|
242
|
+
)
|
|
243
|
+
if handle == wintypes.HANDLE(-1).value:
|
|
244
|
+
return False
|
|
245
|
+
try:
|
|
246
|
+
ft_c = _ft(created_ns)
|
|
247
|
+
ft_a = _ft(accessed_ns)
|
|
248
|
+
ft_m = _ft(modified_ns)
|
|
249
|
+
ok = kernel32.SetFileTime(
|
|
250
|
+
handle,
|
|
251
|
+
ctypes.byref(ft_c) if ft_c else None,
|
|
252
|
+
ctypes.byref(ft_a) if ft_a else None,
|
|
253
|
+
ctypes.byref(ft_m) if ft_m else None,
|
|
254
|
+
)
|
|
255
|
+
return bool(ok)
|
|
256
|
+
finally:
|
|
257
|
+
kernel32.CloseHandle(handle)
|
|
258
|
+
except Exception as e: # noqa: BLE001 - caller falls back to pywin32
|
|
259
|
+
logger.debug(f"_set_link_times_exact_windows({path_str}) failed: {e}")
|
|
260
|
+
return False
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _is_link_node(path_obj: Path) -> bool:
|
|
264
|
+
"""True for any link NODE whose own metadata must be set without
|
|
265
|
+
following to the target: symlinks everywhere, plus junctions (and any
|
|
266
|
+
other reparse point) on Windows. ``Path.is_symlink()`` alone is False for
|
|
267
|
+
junctions (IO_REPARSE_TAG_MOUNT_POINT vs IO_REPARSE_TAG_SYMLINK), which
|
|
268
|
+
routed junctions to the target-following ``os.utime`` path before v0.3.4.
|
|
269
|
+
"""
|
|
270
|
+
if path_obj.is_symlink():
|
|
271
|
+
return True
|
|
272
|
+
if platform.system() != 'Windows':
|
|
273
|
+
return False
|
|
274
|
+
try:
|
|
275
|
+
attrs = os.lstat(str(path_obj)).st_file_attributes
|
|
276
|
+
except OSError:
|
|
277
|
+
return False
|
|
278
|
+
return bool(attrs & _FILE_ATTRIBUTE_REPARSE_POINT)
|
|
279
|
+
|
|
280
|
+
|
|
197
281
|
def _set_symlink_timestamps(path_obj: Path, timestamps: Dict[str, Any]) -> bool:
|
|
198
|
-
"""Apply timestamps to a
|
|
282
|
+
"""Apply timestamps to a link node ITSELF (symlink or junction), never
|
|
283
|
+
the target it points to.
|
|
199
284
|
|
|
200
285
|
``os.utime()`` and a default Win32 ``CreateFile`` both follow the reparse
|
|
201
286
|
point to the target -- so applying timestamps to a link the naive way
|
|
@@ -208,22 +293,58 @@ def _set_symlink_timestamps(path_obj: Path, timestamps: Dict[str, Any]) -> bool:
|
|
|
208
293
|
accessed = timestamps.get('accessed')
|
|
209
294
|
modified = timestamps.get('modified')
|
|
210
295
|
created = timestamps.get('created')
|
|
296
|
+
# Optional exact-precision variants: integer nanoseconds. The float-
|
|
297
|
+
# seconds path bottoms out in datetime/pywintypes at microsecond (or
|
|
298
|
+
# worse) resolution; mirroring wants the NTFS 100ns tick preserved, so
|
|
299
|
+
# *_ns keys, when present, take precedence and travel through an exact
|
|
300
|
+
# integer FILETIME conversion.
|
|
301
|
+
accessed_ns = timestamps.get('accessed_ns')
|
|
302
|
+
modified_ns = timestamps.get('modified_ns')
|
|
303
|
+
created_ns = timestamps.get('created_ns')
|
|
211
304
|
|
|
212
305
|
if platform.system() != 'Windows':
|
|
213
|
-
if
|
|
214
|
-
|
|
215
|
-
|
|
306
|
+
if os.utime not in os.supports_follow_symlinks:
|
|
307
|
+
logger.warning(
|
|
308
|
+
f"Cannot set link timestamps for {path_obj} on this platform "
|
|
309
|
+
f"without following to the target; skipped."
|
|
310
|
+
)
|
|
311
|
+
return False
|
|
312
|
+
try:
|
|
313
|
+
if accessed_ns is not None and modified_ns is not None:
|
|
314
|
+
os.utime(path_obj, ns=(accessed_ns, modified_ns),
|
|
315
|
+
follow_symlinks=False)
|
|
316
|
+
return True
|
|
317
|
+
if accessed is not None and modified is not None:
|
|
216
318
|
os.utime(path_obj, (accessed, modified), follow_symlinks=False)
|
|
217
319
|
return True
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
320
|
+
except Exception as e:
|
|
321
|
+
logger.warning(f"Error applying symlink timestamps to {path_obj}: {e}")
|
|
322
|
+
return False
|
|
221
323
|
logger.warning(
|
|
222
|
-
f"Cannot set link timestamps for {path_obj}
|
|
223
|
-
f"
|
|
324
|
+
f"Cannot set link timestamps for {path_obj}: no usable "
|
|
325
|
+
f"accessed/modified values; skipped."
|
|
224
326
|
)
|
|
225
327
|
return False
|
|
226
328
|
|
|
329
|
+
# Windows exact path: integer ns -> FILETIME (100ns ticks since 1601)
|
|
330
|
+
# via ctypes SetFileTime. No float/datetime rounding, no pywin32 needed.
|
|
331
|
+
if any(v is not None for v in (created_ns, accessed_ns, modified_ns)):
|
|
332
|
+
if _set_link_times_exact_windows(
|
|
333
|
+
str(path_obj), created_ns, accessed_ns, modified_ns
|
|
334
|
+
):
|
|
335
|
+
return True
|
|
336
|
+
logger.debug(
|
|
337
|
+
f"Exact ns SetFileTime failed for {path_obj}; falling back to "
|
|
338
|
+
f"pywin32 float path"
|
|
339
|
+
)
|
|
340
|
+
# Fall through to the float path with whatever float values exist.
|
|
341
|
+
if accessed is None and accessed_ns is not None:
|
|
342
|
+
accessed = accessed_ns / 1e9
|
|
343
|
+
if modified is None and modified_ns is not None:
|
|
344
|
+
modified = modified_ns / 1e9
|
|
345
|
+
if created is None and created_ns is not None:
|
|
346
|
+
created = created_ns / 1e9
|
|
347
|
+
|
|
227
348
|
# Windows: open the link itself (FILE_FLAG_OPEN_REPARSE_POINT) + SetFileTime.
|
|
228
349
|
if not is_win32_available():
|
|
229
350
|
logger.warning(
|
|
@@ -289,13 +410,13 @@ def apply_file_metadata(path: Union[str, Path], metadata: FileMetadataDict) -> b
|
|
|
289
410
|
logger.warning(f"Error applying permissions to {path}: {e}")
|
|
290
411
|
success = False
|
|
291
412
|
|
|
292
|
-
# Apply timestamps.
|
|
293
|
-
# default Win32 handle follow the reparse
|
|
294
|
-
# TARGET, so route
|
|
295
|
-
# three times on the
|
|
413
|
+
# Apply timestamps. Link nodes (symlinks AND junctions) need special
|
|
414
|
+
# handling: os.utime() and the default Win32 handle follow the reparse
|
|
415
|
+
# point and would write to the TARGET, so route them through a
|
|
416
|
+
# link-targeting helper that sets all three times on the node itself.
|
|
296
417
|
if 'timestamps' in metadata:
|
|
297
418
|
timestamps = metadata['timestamps']
|
|
298
|
-
if path_obj
|
|
419
|
+
if _is_link_node(path_obj):
|
|
299
420
|
if not _set_symlink_timestamps(path_obj, timestamps):
|
|
300
421
|
success = False
|
|
301
422
|
else:
|
|
@@ -932,15 +1053,26 @@ def restore_windows_creation_time(
|
|
|
932
1053
|
# Raw value for FILE_WRITE_ATTRIBUTES (0x100) -- not always in win32con
|
|
933
1054
|
FILE_WRITE_ATTRIBUTES = 0x100
|
|
934
1055
|
|
|
935
|
-
# Directories require FILE_FLAG_BACKUP_SEMANTICS to open
|
|
936
|
-
|
|
1056
|
+
# Directories require FILE_FLAG_BACKUP_SEMANTICS to open. For link
|
|
1057
|
+
# nodes (symlink/junction) decide dir-ness from lstat attributes --
|
|
1058
|
+
# is_dir() follows the reparse point and reports False for a BROKEN
|
|
1059
|
+
# directory link, whose handle still needs BACKUP_SEMANTICS.
|
|
1060
|
+
if _is_link_node(path_obj):
|
|
1061
|
+
try:
|
|
1062
|
+
node_attrs = os.lstat(path_str).st_file_attributes
|
|
1063
|
+
except OSError:
|
|
1064
|
+
node_attrs = 0
|
|
1065
|
+
if node_attrs & stat.FILE_ATTRIBUTE_DIRECTORY:
|
|
1066
|
+
flags = win32con.FILE_FLAG_BACKUP_SEMANTICS
|
|
1067
|
+
else:
|
|
1068
|
+
flags = win32con.FILE_ATTRIBUTE_NORMAL
|
|
1069
|
+
# Open the link itself, not the target it points to, so the
|
|
1070
|
+
# creation time lands on the link (otherwise CreateFile follows it).
|
|
1071
|
+
flags |= 0x00200000 # FILE_FLAG_OPEN_REPARSE_POINT
|
|
1072
|
+
elif path_obj.is_dir():
|
|
937
1073
|
flags = win32con.FILE_FLAG_BACKUP_SEMANTICS
|
|
938
1074
|
else:
|
|
939
1075
|
flags = win32con.FILE_ATTRIBUTE_NORMAL
|
|
940
|
-
# Symlinks: open the link itself, not the target it points to, so the
|
|
941
|
-
# creation time lands on the link (otherwise CreateFile follows it).
|
|
942
|
-
if path_obj.is_symlink():
|
|
943
|
-
flags |= 0x00200000 # FILE_FLAG_OPEN_REPARSE_POINT
|
|
944
1076
|
|
|
945
1077
|
# Handle readonly files: clear attribute, set time, restore attribute
|
|
946
1078
|
readonly_cleared = False
|
|
@@ -66,7 +66,20 @@ def atomic_write_text(
|
|
|
66
66
|
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
|
67
67
|
with open(tmp_path, "w", encoding=encoding, newline=newline) as f:
|
|
68
68
|
f.write(content)
|
|
69
|
-
|
|
69
|
+
# Windows: antivirus/indexers briefly lock freshly-written files, so a
|
|
70
|
+
# first os.replace can fail with PermissionError [WinError 5] even
|
|
71
|
+
# though nothing of ours holds the file (observed in the wild: one
|
|
72
|
+
# random victim test per dazzlecmd full-suite run). A short bounded
|
|
73
|
+
# retry absorbs the transient; a REAL permission problem still
|
|
74
|
+
# surfaces after ~0.3s total.
|
|
75
|
+
for attempt in range(6):
|
|
76
|
+
try:
|
|
77
|
+
os.replace(str(tmp_path), str(path))
|
|
78
|
+
break
|
|
79
|
+
except PermissionError:
|
|
80
|
+
if attempt == 5:
|
|
81
|
+
raise
|
|
82
|
+
time.sleep(0.01 * (2 ** attempt)) # 10..160ms backoff
|
|
70
83
|
|
|
71
84
|
|
|
72
85
|
def atomic_write_json(
|
|
@@ -107,6 +120,135 @@ def atomic_write_json(
|
|
|
107
120
|
atomic_write_text(path, text, encoding="utf-8")
|
|
108
121
|
|
|
109
122
|
|
|
123
|
+
class AtomicStreamWriter:
|
|
124
|
+
"""Streaming counterpart of :func:`atomic_write_text`.
|
|
125
|
+
|
|
126
|
+
For content too large -- or too incremental -- to build in memory:
|
|
127
|
+
writes go to a sibling ``.tmp`` file, and on successful close the tmp
|
|
128
|
+
is renamed over ``path`` via ``os.replace``. Readers observing ``path``
|
|
129
|
+
see either the old contents or the complete new contents, never a
|
|
130
|
+
partial write. On failure (an exception inside the ``with`` block, or
|
|
131
|
+
an explicit ``close(success=False)``) the destination is untouched and
|
|
132
|
+
the tmp is removed.
|
|
133
|
+
|
|
134
|
+
Options (all keyword-only):
|
|
135
|
+
|
|
136
|
+
* ``resume_from_existing``: seed the tmp with the current contents of
|
|
137
|
+
``path`` (``shutil.copy2``) and open it in append mode -- for
|
|
138
|
+
resuming an interrupted long-running write. When ``path`` does not
|
|
139
|
+
exist this degrades to a normal fresh write.
|
|
140
|
+
* ``fsync_on_flush``: make each :meth:`flush` also ``os.fsync``,
|
|
141
|
+
bounding data loss to the last flush for observers tailing the tmp
|
|
142
|
+
during very long runs (fsync failures are swallowed -- not every
|
|
143
|
+
platform/filesystem supports it).
|
|
144
|
+
* ``encoding`` / ``newline`` / ``buffering``: passed to ``open()``,
|
|
145
|
+
with the same defaults as :func:`atomic_write_text`.
|
|
146
|
+
|
|
147
|
+
Usage::
|
|
148
|
+
|
|
149
|
+
with AtomicStreamWriter(dest, fsync_on_flush=True) as w:
|
|
150
|
+
for chunk in produce():
|
|
151
|
+
w.write(chunk)
|
|
152
|
+
w.flush()
|
|
153
|
+
|
|
154
|
+
Provenance: generalized from dazzlesum's ``MonolithicWriter`` (its
|
|
155
|
+
streaming checksum-manifest writer), minus the domain concerns
|
|
156
|
+
(headers, footers, overwrite prompts). The one-shot sibling of this
|
|
157
|
+
class is :func:`atomic_write_text`.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
def __init__(
|
|
161
|
+
self,
|
|
162
|
+
path: Union[str, Path],
|
|
163
|
+
*,
|
|
164
|
+
encoding: str = "utf-8",
|
|
165
|
+
newline: Optional[str] = None,
|
|
166
|
+
resume_from_existing: bool = False,
|
|
167
|
+
fsync_on_flush: bool = False,
|
|
168
|
+
buffering: int = -1,
|
|
169
|
+
) -> None:
|
|
170
|
+
self.path = Path(path)
|
|
171
|
+
self.tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
|
|
172
|
+
self.encoding = encoding
|
|
173
|
+
self.newline = newline
|
|
174
|
+
self.resume_from_existing = resume_from_existing
|
|
175
|
+
self.fsync_on_flush = fsync_on_flush
|
|
176
|
+
self.buffering = buffering
|
|
177
|
+
self._file = None
|
|
178
|
+
self._closed = False
|
|
179
|
+
|
|
180
|
+
def __enter__(self) -> "AtomicStreamWriter":
|
|
181
|
+
self.open()
|
|
182
|
+
return self
|
|
183
|
+
|
|
184
|
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
185
|
+
self.close(success=exc_type is None)
|
|
186
|
+
|
|
187
|
+
def open(self) -> "AtomicStreamWriter":
|
|
188
|
+
"""Open the tmp file for writing (called automatically by ``with``)."""
|
|
189
|
+
if self._file is not None:
|
|
190
|
+
return self
|
|
191
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
if self.resume_from_existing and self.path.exists():
|
|
193
|
+
shutil.copy2(str(self.path), str(self.tmp_path))
|
|
194
|
+
mode = "a"
|
|
195
|
+
else:
|
|
196
|
+
mode = "w"
|
|
197
|
+
self._file = open(
|
|
198
|
+
self.tmp_path, mode,
|
|
199
|
+
encoding=self.encoding, newline=self.newline,
|
|
200
|
+
buffering=self.buffering,
|
|
201
|
+
)
|
|
202
|
+
self._closed = False
|
|
203
|
+
return self
|
|
204
|
+
|
|
205
|
+
def write(self, text: str) -> None:
|
|
206
|
+
"""Write ``text`` to the tmp file."""
|
|
207
|
+
if self._file is None:
|
|
208
|
+
raise ValueError("AtomicStreamWriter is not open")
|
|
209
|
+
self._file.write(text)
|
|
210
|
+
|
|
211
|
+
def flush(self) -> None:
|
|
212
|
+
"""Flush buffered writes to the tmp file (and fsync when enabled)."""
|
|
213
|
+
if self._file is None:
|
|
214
|
+
raise ValueError("AtomicStreamWriter is not open")
|
|
215
|
+
self._file.flush()
|
|
216
|
+
if self.fsync_on_flush:
|
|
217
|
+
try:
|
|
218
|
+
os.fsync(self._file.fileno())
|
|
219
|
+
except (OSError, AttributeError):
|
|
220
|
+
# fsync is not available on every platform/filesystem
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
def close(self, success: bool = True) -> None:
|
|
224
|
+
"""Finalize: rename tmp over ``path`` on success, discard it otherwise.
|
|
225
|
+
|
|
226
|
+
Idempotent; safe to call after the ``with`` block has already
|
|
227
|
+
closed the writer.
|
|
228
|
+
"""
|
|
229
|
+
if self._closed:
|
|
230
|
+
return
|
|
231
|
+
self._closed = True
|
|
232
|
+
try:
|
|
233
|
+
if self._file is not None:
|
|
234
|
+
self._file.close()
|
|
235
|
+
self._file = None
|
|
236
|
+
if success:
|
|
237
|
+
os.replace(str(self.tmp_path), str(self.path))
|
|
238
|
+
else:
|
|
239
|
+
self._discard_tmp()
|
|
240
|
+
except Exception:
|
|
241
|
+
self._discard_tmp()
|
|
242
|
+
raise
|
|
243
|
+
|
|
244
|
+
def _discard_tmp(self) -> None:
|
|
245
|
+
try:
|
|
246
|
+
if self.tmp_path.exists():
|
|
247
|
+
os.remove(str(self.tmp_path))
|
|
248
|
+
except OSError as e:
|
|
249
|
+
logger.warning(f"Could not remove temp file {self.tmp_path}: {e}")
|
|
250
|
+
|
|
251
|
+
|
|
110
252
|
# ---------------------------------------------------------------------------
|
|
111
253
|
# Link-safe tree copy (v0.2.4)
|
|
112
254
|
# ---------------------------------------------------------------------------
|
|
@@ -816,6 +958,11 @@ def create_symlink(
|
|
|
816
958
|
>>> create_symlink('C:\\data\\folder', 'C:\\links\\folder_link', target_is_directory=True)
|
|
817
959
|
True
|
|
818
960
|
"""
|
|
961
|
+
# Keep the caller's raw target string for creation: Path() round-trips
|
|
962
|
+
# normalize segments (``a\.\b`` -> ``a\b``, ``/`` -> ``\\``), but a link's
|
|
963
|
+
# stored target must be able to reproduce the caller's bytes exactly
|
|
964
|
+
# (mirroring reads it back via os.readlink and expects equality).
|
|
965
|
+
raw_target = os.fspath(target)
|
|
819
966
|
target_path = Path(target)
|
|
820
967
|
link_path = Path(link)
|
|
821
968
|
|
|
@@ -841,26 +988,34 @@ def create_symlink(
|
|
|
841
988
|
logger.error(f"Failed to create parent directories for {link_path}: {e}")
|
|
842
989
|
return False
|
|
843
990
|
|
|
844
|
-
# Auto-detect if target is a directory
|
|
991
|
+
# Auto-detect if target is a directory. Relative targets resolve against
|
|
992
|
+
# the LINK's parent directory (link semantics), not the process CWD. A
|
|
993
|
+
# broken target probes False -- callers recreating an existing link (e.g.
|
|
994
|
+
# mirroring) must pass target_is_directory explicitly from the source
|
|
995
|
+
# link's own kind, since a broken DIRECTORY symlink cannot be inferred.
|
|
845
996
|
if target_is_directory is None:
|
|
846
|
-
|
|
997
|
+
probe = (target_path if target_path.is_absolute()
|
|
998
|
+
else link_path.parent / raw_target)
|
|
999
|
+
target_is_directory = probe.is_dir()
|
|
847
1000
|
|
|
848
1001
|
# Try to create symlink
|
|
849
1002
|
if platform.system() != 'Windows':
|
|
850
1003
|
# Unix: straightforward symlink
|
|
851
1004
|
try:
|
|
852
|
-
os.symlink(
|
|
853
|
-
logger.debug(f"Created symlink: {link_path} -> {
|
|
1005
|
+
os.symlink(raw_target, str(link_path))
|
|
1006
|
+
logger.debug(f"Created symlink: {link_path} -> {raw_target}")
|
|
854
1007
|
return True
|
|
855
1008
|
except Exception as e:
|
|
856
1009
|
logger.error(f"Failed to create symlink on Unix: {e}")
|
|
857
1010
|
return False
|
|
858
1011
|
|
|
859
1012
|
# Windows: try multiple methods
|
|
860
|
-
return _create_windows_symlink(
|
|
1013
|
+
return _create_windows_symlink(raw_target, link_path, target_is_directory)
|
|
861
1014
|
|
|
862
1015
|
|
|
863
|
-
def _create_windows_symlink(
|
|
1016
|
+
def _create_windows_symlink(
|
|
1017
|
+
target: Union[str, Path], link: Path, is_directory: bool
|
|
1018
|
+
) -> bool:
|
|
864
1019
|
"""
|
|
865
1020
|
Create a symbolic link on Windows using an escalating chain of methods.
|
|
866
1021
|
|
|
@@ -879,14 +1034,15 @@ def _create_windows_symlink(target: Path, link: Path, is_directory: bool) -> boo
|
|
|
879
1034
|
returns False (dazzlelink raised).
|
|
880
1035
|
|
|
881
1036
|
Args:
|
|
882
|
-
target: Target path
|
|
1037
|
+
target: Target path. Passed through VERBATIM (no Path normalization)
|
|
1038
|
+
so the stored reparse target reproduces the caller's bytes.
|
|
883
1039
|
link: Link path to create
|
|
884
1040
|
is_directory: Whether target is a directory
|
|
885
1041
|
|
|
886
1042
|
Returns:
|
|
887
1043
|
True if successful, False otherwise
|
|
888
1044
|
"""
|
|
889
|
-
target_str =
|
|
1045
|
+
target_str = os.fspath(target)
|
|
890
1046
|
link_str = str(link)
|
|
891
1047
|
|
|
892
1048
|
# Method 1: os.symlink (Developer Mode enabled)
|