kisstdlib 0.0.9__tar.gz → 0.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/PKG-INFO +2 -2
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/__init__.py +22 -6
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/argparse_ext.py +6 -6
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/base.py +1 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/fs.py +143 -95
- kisstdlib-0.0.11/kisstdlib/io/adapter.py +140 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/io/stdio.py +8 -5
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/logging_ext.py +10 -17
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/signal_ext.py +31 -15
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/string_ext.py +7 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/PKG-INFO +2 -2
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/SOURCES.txt +1 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/pyproject.toml +1 -1
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/test/test_fs.py +32 -32
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/LICENSE.txt +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/README.md +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/compression.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/failure.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/getpass_ext.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/io/__init__.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/io/base.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/io/encoder.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/io/wrapper.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/itertools_ext.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/parsing.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/py.typed +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/pyrepr.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/sorted.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/sqlite3_ext.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/tariter.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib/time.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/dependency_links.txt +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/entry_points.txt +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/requires.txt +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib.egg-info/top_level.txt +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/kisstdlib_bin/describe_forest.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/setup.cfg +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/setup.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/test/test_basics.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/test/test_io.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/test/test_sorted.py +0 -0
- {kisstdlib-0.0.9 → kisstdlib-0.0.11}/test/test_time.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: kisstdlib
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.11
|
|
4
4
|
Summary: System programming with conceptually and algebraically simple primitives.
|
|
5
5
|
Author-email: Jan Malakhovski <oxij@oxij.org>
|
|
6
6
|
Project-URL: Homepage, https://oxij.org/software/kisstdlib/
|
|
@@ -36,11 +36,15 @@ from .io.stdio import *
|
|
|
36
36
|
from .logging_ext import *
|
|
37
37
|
from .signal_ext import *
|
|
38
38
|
|
|
39
|
+
from .fs import setup_fs as _setup_fs
|
|
40
|
+
|
|
39
41
|
|
|
40
42
|
def setup_kisstdlib(
|
|
41
43
|
prog: str | None = None,
|
|
44
|
+
*,
|
|
42
45
|
do_setup_stdio: bool = True,
|
|
43
|
-
|
|
46
|
+
do_setup_delay_signals: bool = True,
|
|
47
|
+
do_setup_fs: bool = True,
|
|
44
48
|
**kwargs: _t.Any,
|
|
45
49
|
) -> tuple[LogCounter, ANSILogHandler]:
|
|
46
50
|
"""Run all `setup_*` functions of all other `kisstdlib.*` modules and setup
|
|
@@ -50,9 +54,12 @@ def setup_kisstdlib(
|
|
|
50
54
|
prog = _op.basename(_sys.argv[0])
|
|
51
55
|
if do_setup_stdio:
|
|
52
56
|
setup_stdio()
|
|
53
|
-
res = setup_logging(prog, **kwargs)
|
|
54
57
|
setup_delay_signal_messages(prog + ": ")
|
|
55
|
-
|
|
58
|
+
if do_setup_delay_signals:
|
|
59
|
+
setup_delay_signals()
|
|
60
|
+
if do_setup_fs:
|
|
61
|
+
_setup_fs(prog)
|
|
62
|
+
res = setup_logging(prog, **kwargs)
|
|
56
63
|
return res
|
|
57
64
|
|
|
58
65
|
|
|
@@ -102,9 +109,18 @@ def run_kisstdlib_main(
|
|
|
102
109
|
for be in flat_exceptions(excs):
|
|
103
110
|
critical(gettext("Uncaught exception: %s"), str(be), exc_info=be)
|
|
104
111
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
112
|
+
try:
|
|
113
|
+
logger.flush()
|
|
114
|
+
except BrokenPipeError:
|
|
115
|
+
pass
|
|
116
|
+
try:
|
|
117
|
+
stdout.flush()
|
|
118
|
+
except BrokenPipeError:
|
|
119
|
+
pass
|
|
120
|
+
try:
|
|
121
|
+
stderr.flush()
|
|
122
|
+
except BrokenPipeError:
|
|
123
|
+
pass
|
|
108
124
|
|
|
109
125
|
num_warnings = counter.warnings
|
|
110
126
|
num_errors = counter.errors
|
|
@@ -159,16 +159,16 @@ class MarkdownBetterHelpFormatter(BetterHelpFormatter):
|
|
|
159
159
|
return join(["\n", heading, item_help, "\n"])
|
|
160
160
|
|
|
161
161
|
|
|
162
|
-
class
|
|
162
|
+
class _OptionallyMarkdownHelpAction(Action):
|
|
163
163
|
def __init__(
|
|
164
164
|
self,
|
|
165
165
|
option_strings: list[str],
|
|
166
166
|
dest: str = SUPPRESS,
|
|
167
167
|
default: _t.Any = SUPPRESS,
|
|
168
|
-
|
|
168
|
+
**kwargs: _t.Any,
|
|
169
169
|
) -> None:
|
|
170
170
|
super().__init__(
|
|
171
|
-
option_strings=option_strings, dest=dest, default=default, nargs=0,
|
|
171
|
+
option_strings=option_strings, dest=dest, default=default, nargs=0, **kwargs
|
|
172
172
|
)
|
|
173
173
|
|
|
174
174
|
def show_help(self, parser: "BetterArgumentParser", namespace: Namespace) -> None:
|
|
@@ -225,6 +225,8 @@ class BetterArgumentParser(ArgumentParser):
|
|
|
225
225
|
**kwargs,
|
|
226
226
|
)
|
|
227
227
|
|
|
228
|
+
self.register("action", "help_omd", _OptionallyMarkdownHelpAction)
|
|
229
|
+
|
|
228
230
|
self.add_version = add_version
|
|
229
231
|
self.add_help = add_help
|
|
230
232
|
self.additional_sections = additional_sections
|
|
@@ -249,12 +251,10 @@ class BetterArgumentParser(ArgumentParser):
|
|
|
249
251
|
)
|
|
250
252
|
|
|
251
253
|
if add_help:
|
|
252
|
-
self.register("action", "help", OptionallyMarkdownHelpAction)
|
|
253
|
-
|
|
254
254
|
self.add_argument(
|
|
255
255
|
default_prefix + "h",
|
|
256
256
|
default_prefix * 2 + "help",
|
|
257
|
-
action="
|
|
257
|
+
action="help_omd",
|
|
258
258
|
default=SUPPRESS,
|
|
259
259
|
help=_gettext("show this help message and exit"),
|
|
260
260
|
)
|
|
@@ -36,41 +36,39 @@ import stat as _stat
|
|
|
36
36
|
import sys as _sys
|
|
37
37
|
import typing as _t
|
|
38
38
|
|
|
39
|
+
from os import fsencode, fsdecode
|
|
40
|
+
|
|
39
41
|
from .base import POSIX as _POSIX, identity as _identity
|
|
40
|
-
from .string_ext import escaper as _escaper
|
|
41
42
|
from .io.base import *
|
|
43
|
+
from .string_ext import escape_path as _escape_path
|
|
42
44
|
from .time import Timestamp as _Timestamp
|
|
43
45
|
|
|
44
46
|
sep = _op.sep
|
|
45
47
|
sepb = _os.fsencode(_op.sep)
|
|
46
48
|
|
|
47
49
|
|
|
48
|
-
def
|
|
49
|
-
"""Apply `os.fsdecode` if `bytes`."""
|
|
50
|
-
if isinstance(x, bytes):
|
|
51
|
-
return _os.fsdecode(x)
|
|
52
|
-
return x
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def fsencode_maybe(x: str | bytes) -> bytes:
|
|
56
|
-
"""Apply `os.fsencode` if `str`."""
|
|
57
|
-
if isinstance(x, str):
|
|
58
|
-
return _os.fsencode(x)
|
|
59
|
-
return x
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def dirname_dot(x: _t.AnyStr) -> tuple[_t.AnyStr, bool]:
|
|
50
|
+
def dirname_dot(path: _t.AnyStr) -> tuple[_t.AnyStr, bool]:
|
|
63
51
|
"""Apply `os.path.dirname` to the given argument, but if it's empty, return "." instead.
|
|
64
52
|
The second element of the tuple is `False` if the above replacement was performed, `True` otherwise.
|
|
65
53
|
"""
|
|
66
|
-
|
|
67
|
-
if isinstance(
|
|
68
|
-
if
|
|
54
|
+
path = _os.path.dirname(path)
|
|
55
|
+
if isinstance(path, bytes):
|
|
56
|
+
if path == b"":
|
|
69
57
|
return b".", False
|
|
70
|
-
return
|
|
71
|
-
if
|
|
58
|
+
return path, True
|
|
59
|
+
if path == "":
|
|
72
60
|
return ".", False
|
|
73
|
-
return
|
|
61
|
+
return path, True
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def realdir(path: _t.AnyStr, strict: bool = False) -> _t.AnyStr:
|
|
65
|
+
"""Apply `os.path.realpath` to the `dirname` part of `path`.
|
|
66
|
+
|
|
67
|
+
Essentially, this returns the canonical path of the inode the `path` points to, even when that
|
|
68
|
+
inode is a symlink.
|
|
69
|
+
"""
|
|
70
|
+
dn, bn = _op.split(path)
|
|
71
|
+
return _op.join(_op.realpath(dn, strict=strict), bn)
|
|
74
72
|
|
|
75
73
|
|
|
76
74
|
def read_file_maybe(path: str | bytes) -> bytes | None:
|
|
@@ -100,8 +98,8 @@ def same_file_data(path1: str | bytes, path2: str | bytes) -> bool:
|
|
|
100
98
|
|
|
101
99
|
def same_symlink_data(path1: str | bytes, path2: str | bytes) -> bool:
|
|
102
100
|
"""Check if two given symlinks have exactly the same content."""
|
|
103
|
-
a = _os.readlink(
|
|
104
|
-
b = _os.readlink(
|
|
101
|
+
a = _os.readlink(fsencode(path1))
|
|
102
|
+
b = _os.readlink(fsencode(path2))
|
|
105
103
|
return a == b
|
|
106
104
|
|
|
107
105
|
|
|
@@ -204,9 +202,9 @@ def iter_subtree(
|
|
|
204
202
|
fstat = _os.stat(path, follow_symlinks=follow_symlinks)
|
|
205
203
|
except OSError as exc:
|
|
206
204
|
if handle_error is not None:
|
|
207
|
-
eno = exc.errno
|
|
205
|
+
eno = exc.errno or 0
|
|
208
206
|
handle_error(
|
|
209
|
-
"failed to stat
|
|
207
|
+
"failed to `stat`: [Errno %d, %s] %s: %s",
|
|
210
208
|
eno,
|
|
211
209
|
_errno.errorcode.get(eno, "?"),
|
|
212
210
|
_os.strerror(eno),
|
|
@@ -230,7 +228,7 @@ def iter_subtree(
|
|
|
230
228
|
scandir_it = _os.scandir(path)
|
|
231
229
|
except OSError as exc:
|
|
232
230
|
if handle_error is not None:
|
|
233
|
-
eno = exc.errno
|
|
231
|
+
eno = exc.errno or 0
|
|
234
232
|
handle_error(
|
|
235
233
|
"failed to `scandir`: [Errno %d, %s] %s: %s",
|
|
236
234
|
eno,
|
|
@@ -252,7 +250,7 @@ def iter_subtree(
|
|
|
252
250
|
break
|
|
253
251
|
except OSError as exc:
|
|
254
252
|
if handle_error is not None:
|
|
255
|
-
eno = exc.errno
|
|
253
|
+
eno = exc.errno or 0
|
|
256
254
|
handle_error(
|
|
257
255
|
"failed in `scandir`: [Errno %d, %s] %s: %s",
|
|
258
256
|
eno,
|
|
@@ -267,7 +265,7 @@ def iter_subtree(
|
|
|
267
265
|
entry_is_dir = entry.is_dir(follow_symlinks=follow_symlinks)
|
|
268
266
|
except OSError as exc:
|
|
269
267
|
if handle_error is not None:
|
|
270
|
-
eno = exc.errno
|
|
268
|
+
eno = exc.errno or 0
|
|
271
269
|
handle_error(
|
|
272
270
|
"failed to `stat`: [Errno %d, %s] %s: %s",
|
|
273
271
|
eno,
|
|
@@ -410,12 +408,12 @@ def describe_forest(
|
|
|
410
408
|
"""Produce a simple `find .`+`stat`-like description of walks of given `paths`.
|
|
411
409
|
See `describe-forest` script for more info.
|
|
412
410
|
"""
|
|
413
|
-
escape: _t.Callable[[str], str] = _identity if literal else
|
|
411
|
+
escape: _t.Callable[[str], str] = _identity if literal else _escape_path
|
|
414
412
|
seen: dict[tuple[int, int], tuple[_t.AnyStr, int, str]] = {}
|
|
415
413
|
for i, dirpath in enumerate(paths):
|
|
416
414
|
for fpath, _eps, _edir in iter_subtree(dirpath, follow_symlinks=follow_symlinks):
|
|
417
415
|
abs_path = _op.abspath(fpath)
|
|
418
|
-
rpath =
|
|
416
|
+
rpath = fsdecode(_op.relpath(fpath, dirpath))
|
|
419
417
|
epath: str
|
|
420
418
|
if numbers is False or numbers is None and len(paths) == 1:
|
|
421
419
|
epath = escape(rpath)
|
|
@@ -434,7 +432,7 @@ def describe_forest(
|
|
|
434
432
|
if relative_hardlinks and hi == i:
|
|
435
433
|
# within the same `dirpath`
|
|
436
434
|
dirname = _op.dirname(abs_path)
|
|
437
|
-
target = escape(
|
|
435
|
+
target = escape(fsdecode(_op.relpath(habs_path, dirname)))
|
|
438
436
|
yield [epath, "ref", "=>", target]
|
|
439
437
|
else:
|
|
440
438
|
yield [epath, "ref", "==>", hepath]
|
|
@@ -464,7 +462,7 @@ def describe_forest(
|
|
|
464
462
|
sha256 = sha256[:hash_length]
|
|
465
463
|
yield [epath, "reg"] + emode + emtime + esize + ["sha256", sha256] # fmt: skip
|
|
466
464
|
elif _stat.S_ISLNK(stat.st_mode):
|
|
467
|
-
esymlink = escape(
|
|
465
|
+
esymlink = escape(fsdecode(_os.readlink(abs_path)))
|
|
468
466
|
yield [epath, "sym"] + emode + emtime + ["->", esymlink]
|
|
469
467
|
else:
|
|
470
468
|
yield [epath, "???"] + emode + emtime + esize
|
|
@@ -588,6 +586,7 @@ class DeferredSync(_t.Generic[_t.AnyStr]):
|
|
|
588
586
|
Doing this can improve disk performance considerably.
|
|
589
587
|
"""
|
|
590
588
|
|
|
589
|
+
defer: bool
|
|
591
590
|
tmp_file: set[_t.AnyStr]
|
|
592
591
|
unlink_file: set[_t.AnyStr]
|
|
593
592
|
fsync_file: set[_t.AnyStr]
|
|
@@ -598,7 +597,8 @@ class DeferredSync(_t.Generic[_t.AnyStr]):
|
|
|
598
597
|
# if all of the above succeed, also do these
|
|
599
598
|
_after: _t.Optional["DeferredSync[_t.AnyStr]"]
|
|
600
599
|
|
|
601
|
-
def __init__(self) -> None:
|
|
600
|
+
def __init__(self, defer: bool) -> None:
|
|
601
|
+
self.defer = defer
|
|
602
602
|
self.reset()
|
|
603
603
|
|
|
604
604
|
def reset(self) -> None:
|
|
@@ -628,14 +628,14 @@ after="""
|
|
|
628
628
|
@property
|
|
629
629
|
def after(self) -> "DeferredSync[_t.AnyStr]":
|
|
630
630
|
if self._after is None:
|
|
631
|
-
self._after = DeferredSync()
|
|
631
|
+
self._after = DeferredSync(self.defer)
|
|
632
632
|
return self._after
|
|
633
633
|
|
|
634
634
|
def copy(self) -> "DeferredSync[_t.AnyStr]":
|
|
635
635
|
"""Return a shallow copy of this object (elements describing operations are not
|
|
636
636
|
copied, only the structure is).
|
|
637
637
|
"""
|
|
638
|
-
res: DeferredSync[_t.AnyStr] = DeferredSync()
|
|
638
|
+
res: DeferredSync[_t.AnyStr] = DeferredSync(self.defer)
|
|
639
639
|
res.tmp_file = set(self.tmp_file)
|
|
640
640
|
res.unlink_file = set(self.unlink_file)
|
|
641
641
|
res.fsync_file = set(self.fsync_file)
|
|
@@ -654,20 +654,18 @@ after="""
|
|
|
654
654
|
unlink_many(self.tmp_file)
|
|
655
655
|
self.reset()
|
|
656
656
|
|
|
657
|
-
def
|
|
658
|
-
self, strict: bool = True, simulate: list[list[str]] | None = None
|
|
659
|
-
) -> list[Exception]:
|
|
657
|
+
def sync(self, strict: bool = True, simulate: list[list[str]] | None = None) -> list[Exception]:
|
|
660
658
|
"""Perform all deferred operations and return a list of raised exceptions.
|
|
661
659
|
|
|
662
660
|
Operations that fail will be left sitting in the corresponding fields.
|
|
663
661
|
|
|
664
662
|
With `simulate` argument set, this function instead simulates the
|
|
665
|
-
`
|
|
663
|
+
`sync` and writes the log of things it would do into that argument.
|
|
666
664
|
"""
|
|
667
665
|
exceptions: list[Exception] = []
|
|
668
666
|
|
|
669
667
|
if simulate is not None:
|
|
670
|
-
simulate += [["unlink",
|
|
668
|
+
simulate += [["unlink", fsdecode(f)] for f in sorted(self.unlink_file)]
|
|
671
669
|
self.unlink_file = set()
|
|
672
670
|
elif self.unlink_file:
|
|
673
671
|
self.unlink_file = set(unlink_many(sorted(self.unlink_file), exceptions))
|
|
@@ -689,7 +687,7 @@ after="""
|
|
|
689
687
|
|
|
690
688
|
while self.fsync_file or self.fsync_dir or self.fsync_dir2 or self.rename_file:
|
|
691
689
|
if simulate is not None:
|
|
692
|
-
simulate += [["fsync",
|
|
690
|
+
simulate += [["fsync", fsdecode(f)] for f in sorted(self.fsync_file)]
|
|
693
691
|
self.fsync_file = set()
|
|
694
692
|
elif self.fsync_file:
|
|
695
693
|
fsync_file_left.update(fsync_many_files(sorted(self.fsync_file), 0, exceptions))
|
|
@@ -723,7 +721,7 @@ after="""
|
|
|
723
721
|
|
|
724
722
|
try:
|
|
725
723
|
if simulate is not None:
|
|
726
|
-
simulate.append(["replace" if allow_overwrites else "rename",
|
|
724
|
+
simulate.append(["replace" if allow_overwrites else "rename", fsdecode(src_path), fsdecode(dst_path)]) # fmt: skip
|
|
727
725
|
else:
|
|
728
726
|
rename(src_path, dst_path, allow_overwrites, makedirs=False, dst_dir=dst_dir) # fmt: skip
|
|
729
727
|
except Exception as exc:
|
|
@@ -751,14 +749,14 @@ after="""
|
|
|
751
749
|
return exceptions
|
|
752
750
|
|
|
753
751
|
if simulate is not None:
|
|
754
|
-
simulate += [["fsync_win",
|
|
752
|
+
simulate += [["fsync_win", fsdecode(f)] for f in sorted(self.fsync_file)]
|
|
755
753
|
self.fsync_file = set()
|
|
756
754
|
elif self.fsync_file:
|
|
757
755
|
fsync_file_left.update(fsync_many_files(sorted(self.fsync_file), 0, exceptions))
|
|
758
756
|
self.fsync_file = set()
|
|
759
757
|
|
|
760
758
|
if simulate is not None:
|
|
761
|
-
simulate += [["fsync_dir",
|
|
759
|
+
simulate += [["fsync_dir", fsdecode(f)] for f in sorted(self.fsync_dir)]
|
|
762
760
|
self.fsync_dir = set()
|
|
763
761
|
elif self.fsync_dir:
|
|
764
762
|
if _POSIX:
|
|
@@ -766,7 +764,7 @@ after="""
|
|
|
766
764
|
self.fsync_dir = set()
|
|
767
765
|
|
|
768
766
|
if simulate is not None:
|
|
769
|
-
simulate += [["fsync_dir",
|
|
767
|
+
simulate += [["fsync_dir", fsdecode(f)] for f in sorted(self.fsync_dir2)]
|
|
770
768
|
self.fsync_dir2 = set()
|
|
771
769
|
elif self.fsync_dir2:
|
|
772
770
|
if _POSIX:
|
|
@@ -785,7 +783,7 @@ after="""
|
|
|
785
783
|
return exceptions
|
|
786
784
|
|
|
787
785
|
if self._after is not None:
|
|
788
|
-
excs = self._after.
|
|
786
|
+
excs = self._after.sync(strict, simulate)
|
|
789
787
|
if len(excs) > 0:
|
|
790
788
|
exceptions += excs
|
|
791
789
|
else:
|
|
@@ -793,12 +791,12 @@ after="""
|
|
|
793
791
|
|
|
794
792
|
return exceptions
|
|
795
793
|
|
|
796
|
-
def
|
|
797
|
-
"""Like `
|
|
794
|
+
def flush(self, strict: bool = True) -> None:
|
|
795
|
+
"""Like `sync` but raises collected exceptions as an exception group and calls
|
|
798
796
|
`.clear` in that case.
|
|
799
797
|
"""
|
|
800
798
|
try:
|
|
801
|
-
excs = self.
|
|
799
|
+
excs = self.sync(strict)
|
|
802
800
|
if len(excs) > 0:
|
|
803
801
|
raise ExceptionGroup("failed to sync", excs)
|
|
804
802
|
finally:
|
|
@@ -811,7 +809,7 @@ def atomic_rename(
|
|
|
811
809
|
allow_overwrites: bool,
|
|
812
810
|
*,
|
|
813
811
|
makedirs: bool = True,
|
|
814
|
-
|
|
812
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
815
813
|
) -> None:
|
|
816
814
|
"""Atomically rename a file, performing all necesary `fsync`s."""
|
|
817
815
|
|
|
@@ -821,19 +819,24 @@ def atomic_rename(
|
|
|
821
819
|
if makedirs and nondot:
|
|
822
820
|
_os.makedirs(dst_dir, exist_ok=True)
|
|
823
821
|
|
|
824
|
-
if
|
|
825
|
-
|
|
822
|
+
if isinstance(sync, DeferredSync) and sync.defer:
|
|
823
|
+
sync.rename_file.append((src_path, dst_path, allow_overwrites, src_dir, dst_dir))
|
|
826
824
|
return
|
|
827
825
|
|
|
828
826
|
rename(src_path, dst_path, allow_overwrites, makedirs=False, dst_dir=dst_dir)
|
|
829
827
|
|
|
830
|
-
if
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
828
|
+
if isinstance(sync, DeferredSync):
|
|
829
|
+
sync.fsync_dir.add(dst_dir)
|
|
830
|
+
return
|
|
831
|
+
|
|
832
|
+
if sync:
|
|
833
|
+
if _POSIX:
|
|
834
|
+
fsync_file(dst_dir, _os.O_DIRECTORY)
|
|
835
|
+
if src_dir != dst_dir:
|
|
836
|
+
fsync_file(src_dir, _os.O_DIRECTORY)
|
|
837
|
+
else:
|
|
838
|
+
# on Windows, some docs claim, this helps
|
|
839
|
+
fsync_file(dst_path)
|
|
837
840
|
|
|
838
841
|
|
|
839
842
|
def make_file(
|
|
@@ -842,7 +845,7 @@ def make_file(
|
|
|
842
845
|
allow_overwrites: bool = False,
|
|
843
846
|
*,
|
|
844
847
|
makedirs: bool = True,
|
|
845
|
-
|
|
848
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
846
849
|
) -> None:
|
|
847
850
|
"""Create a file using a given `make_dst` function."""
|
|
848
851
|
|
|
@@ -854,24 +857,28 @@ def make_file(
|
|
|
854
857
|
|
|
855
858
|
if makedirs and nondot:
|
|
856
859
|
_os.makedirs(dst_dir, exist_ok=True)
|
|
857
|
-
make_dst(dst_path,
|
|
860
|
+
make_dst(dst_path, isinstance(sync, bool))
|
|
858
861
|
|
|
859
|
-
if
|
|
860
|
-
|
|
861
|
-
|
|
862
|
+
if isinstance(sync, DeferredSync):
|
|
863
|
+
sync.fsync_file.add(dst_path)
|
|
864
|
+
sync.fsync_dir.add(dst_dir)
|
|
862
865
|
return
|
|
863
866
|
|
|
864
|
-
if _POSIX:
|
|
867
|
+
if sync and _POSIX:
|
|
865
868
|
fsync_file(dst_dir, _os.O_DIRECTORY)
|
|
866
869
|
|
|
867
870
|
|
|
871
|
+
_dot_part = ".part"
|
|
872
|
+
_dot_partb = b".part"
|
|
873
|
+
|
|
874
|
+
|
|
868
875
|
def atomic_make_file(
|
|
869
876
|
make_dst: _t.Callable[[_t.AnyStr, bool], None],
|
|
870
877
|
dst_path: _t.AnyStr,
|
|
871
878
|
allow_overwrites: bool = False,
|
|
872
879
|
*,
|
|
873
880
|
makedirs: bool = True,
|
|
874
|
-
|
|
881
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
875
882
|
) -> None:
|
|
876
883
|
"""Atomically create a file using a given `make_dst` function. This
|
|
877
884
|
runs `make_dst` on a `.part` path first, `fsync`s it, then does
|
|
@@ -885,29 +892,35 @@ def atomic_make_file(
|
|
|
885
892
|
raise FileExistsError(_errno.EEXIST, _os.strerror(_errno.EEXIST), dst_path)
|
|
886
893
|
|
|
887
894
|
if isinstance(dst_path, str):
|
|
888
|
-
tmp_path = dst_path +
|
|
895
|
+
tmp_path = dst_path + _dot_part
|
|
889
896
|
else:
|
|
890
|
-
tmp_path = dst_path +
|
|
897
|
+
tmp_path = dst_path + _dot_partb
|
|
891
898
|
|
|
892
899
|
dst_dir, nondot = dirname_dot(dst_path)
|
|
893
900
|
|
|
894
901
|
if makedirs and nondot:
|
|
895
902
|
_os.makedirs(dst_dir, exist_ok=True)
|
|
896
|
-
make_dst(tmp_path,
|
|
903
|
+
make_dst(tmp_path, isinstance(sync, bool) and sync)
|
|
897
904
|
|
|
898
|
-
if
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
905
|
+
if isinstance(sync, DeferredSync) and sync.defer:
|
|
906
|
+
sync.tmp_file.add(tmp_path)
|
|
907
|
+
sync.fsync_file.add(tmp_path)
|
|
908
|
+
sync.rename_file.append((tmp_path, dst_path, allow_overwrites, dst_dir, dst_dir))
|
|
902
909
|
return
|
|
903
910
|
|
|
904
911
|
try:
|
|
912
|
+
if sync:
|
|
913
|
+
fsync_file(tmp_path)
|
|
905
914
|
rename(tmp_path, dst_path, allow_overwrites, dst_dir=dst_dir)
|
|
906
915
|
except Exception:
|
|
907
916
|
unlink_maybe(tmp_path)
|
|
908
917
|
raise
|
|
909
918
|
|
|
910
|
-
if
|
|
919
|
+
if isinstance(sync, DeferredSync):
|
|
920
|
+
sync.fsync_dir.add(dst_dir)
|
|
921
|
+
return
|
|
922
|
+
|
|
923
|
+
if sync and _POSIX:
|
|
911
924
|
fsync_file(dst_dir, _os.O_DIRECTORY)
|
|
912
925
|
# NB: src_dir == dst_dir
|
|
913
926
|
|
|
@@ -919,7 +932,7 @@ def atomic_copy2(
|
|
|
919
932
|
*,
|
|
920
933
|
follow_symlinks: bool = True,
|
|
921
934
|
makedirs: bool = True,
|
|
922
|
-
|
|
935
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
923
936
|
) -> None:
|
|
924
937
|
"""Atomically copy `src_path` to `dst_path`."""
|
|
925
938
|
|
|
@@ -942,7 +955,7 @@ def atomic_copy2(
|
|
|
942
955
|
|
|
943
956
|
# always use the atomic version here, like rsync does,
|
|
944
957
|
# since copying can be interrupted in the middle
|
|
945
|
-
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
958
|
+
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
946
959
|
|
|
947
960
|
|
|
948
961
|
def atomic_link(
|
|
@@ -952,7 +965,7 @@ def atomic_link(
|
|
|
952
965
|
*,
|
|
953
966
|
follow_symlinks: bool = True,
|
|
954
967
|
makedirs: bool = True,
|
|
955
|
-
|
|
968
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
956
969
|
) -> None:
|
|
957
970
|
"""Atomically hardlink `src_path` to `dst_path`."""
|
|
958
971
|
|
|
@@ -963,10 +976,10 @@ def atomic_link(
|
|
|
963
976
|
_os.link(src_path, dst_path, follow_symlinks=follow_symlinks)
|
|
964
977
|
|
|
965
978
|
# _os.link is atomic, so non-atomic make_file is ok
|
|
966
|
-
if allow_overwrites:
|
|
967
|
-
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
979
|
+
if allow_overwrites or (isinstance(sync, DeferredSync) and sync.defer):
|
|
980
|
+
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
968
981
|
else:
|
|
969
|
-
make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
982
|
+
make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
970
983
|
|
|
971
984
|
|
|
972
985
|
def atomic_symlink(
|
|
@@ -976,7 +989,7 @@ def atomic_symlink(
|
|
|
976
989
|
*,
|
|
977
990
|
follow_symlinks: bool = True,
|
|
978
991
|
makedirs: bool = True,
|
|
979
|
-
|
|
992
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
980
993
|
) -> None:
|
|
981
994
|
"""Atomically symlink `src_path` to `dst_path`."""
|
|
982
995
|
|
|
@@ -987,10 +1000,10 @@ def atomic_symlink(
|
|
|
987
1000
|
_os.symlink(src_path, dst_path)
|
|
988
1001
|
|
|
989
1002
|
# _os.symlink is atomic, so non-atomic make_file is ok
|
|
990
|
-
if allow_overwrites:
|
|
991
|
-
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
1003
|
+
if allow_overwrites or (isinstance(sync, DeferredSync) and sync.defer):
|
|
1004
|
+
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
992
1005
|
else:
|
|
993
|
-
make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
1006
|
+
make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
994
1007
|
|
|
995
1008
|
|
|
996
1009
|
def atomic_link_or_copy2(
|
|
@@ -1000,16 +1013,16 @@ def atomic_link_or_copy2(
|
|
|
1000
1013
|
*,
|
|
1001
1014
|
follow_symlinks: bool = True,
|
|
1002
1015
|
makedirs: bool = True,
|
|
1003
|
-
|
|
1016
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1004
1017
|
) -> None:
|
|
1005
1018
|
"""Atomically hardlink or copy `src_path` to `dst_path`."""
|
|
1006
1019
|
|
|
1007
1020
|
try:
|
|
1008
|
-
atomic_link(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs,
|
|
1021
|
+
atomic_link(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs, sync=sync) # fmt: skip
|
|
1009
1022
|
except OSError as exc:
|
|
1010
1023
|
if exc.errno != _errno.EXDEV:
|
|
1011
1024
|
raise
|
|
1012
|
-
atomic_copy2(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs,
|
|
1025
|
+
atomic_copy2(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs, sync=sync) # fmt: skip
|
|
1013
1026
|
|
|
1014
1027
|
|
|
1015
1028
|
def atomic_move(
|
|
@@ -1019,7 +1032,7 @@ def atomic_move(
|
|
|
1019
1032
|
*,
|
|
1020
1033
|
follow_symlinks: bool = False,
|
|
1021
1034
|
makedirs: bool = True,
|
|
1022
|
-
|
|
1035
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1023
1036
|
) -> None:
|
|
1024
1037
|
"""Atomically move `src_path` to `dst_path`.
|
|
1025
1038
|
|
|
@@ -1029,17 +1042,17 @@ def atomic_move(
|
|
|
1029
1042
|
|
|
1030
1043
|
src_dir, _ = dirname_dot(src_path)
|
|
1031
1044
|
|
|
1032
|
-
atomic_link_or_copy2(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs,
|
|
1045
|
+
atomic_link_or_copy2(src_path, dst_path, allow_overwrites, follow_symlinks=follow_symlinks, makedirs=makedirs, sync=sync) # fmt: skip
|
|
1033
1046
|
|
|
1034
|
-
if
|
|
1035
|
-
after =
|
|
1047
|
+
if isinstance(sync, DeferredSync):
|
|
1048
|
+
after = sync.after
|
|
1036
1049
|
after.unlink_file.add(src_path)
|
|
1037
1050
|
after.fsync_dir.add(src_dir)
|
|
1038
1051
|
return
|
|
1039
1052
|
|
|
1040
1053
|
_os.unlink(src_path)
|
|
1041
1054
|
|
|
1042
|
-
if _POSIX:
|
|
1055
|
+
if sync and _POSIX:
|
|
1043
1056
|
fsync_file(src_dir, _os.O_DIRECTORY)
|
|
1044
1057
|
|
|
1045
1058
|
|
|
@@ -1049,7 +1062,7 @@ def atomic_write(
|
|
|
1049
1062
|
allow_overwrites: bool = False,
|
|
1050
1063
|
*,
|
|
1051
1064
|
makedirs: bool = True,
|
|
1052
|
-
|
|
1065
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1053
1066
|
) -> None:
|
|
1054
1067
|
"""Atomically write given `data` to `dst_path`."""
|
|
1055
1068
|
|
|
@@ -1064,4 +1077,39 @@ def atomic_write(
|
|
|
1064
1077
|
unlink_maybe(tmp_path)
|
|
1065
1078
|
raise
|
|
1066
1079
|
|
|
1067
|
-
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs,
|
|
1080
|
+
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def atomic_unlink(
|
|
1084
|
+
path: _t.AnyStr,
|
|
1085
|
+
*,
|
|
1086
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1087
|
+
) -> None:
|
|
1088
|
+
"""Atomically unlink `path`."""
|
|
1089
|
+
|
|
1090
|
+
dirname, _ = dirname_dot(path)
|
|
1091
|
+
|
|
1092
|
+
if isinstance(sync, DeferredSync) and sync.defer:
|
|
1093
|
+
sync.unlink_file.add(path)
|
|
1094
|
+
sync.fsync_dir.add(dirname)
|
|
1095
|
+
return
|
|
1096
|
+
|
|
1097
|
+
_os.unlink(path)
|
|
1098
|
+
|
|
1099
|
+
if isinstance(sync, DeferredSync):
|
|
1100
|
+
sync.fsync_dir.add(dirname)
|
|
1101
|
+
return
|
|
1102
|
+
|
|
1103
|
+
if sync and _POSIX:
|
|
1104
|
+
fsync_file(dirname, _os.O_DIRECTORY)
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def setup_fs(prog: str | None = None, ext: str = ".part", add_pid: bool = True) -> None:
|
|
1108
|
+
"""Setup temporary files prefix."""
|
|
1109
|
+
if prog is None:
|
|
1110
|
+
prog = _op.basename(_sys.argv[0])
|
|
1111
|
+
|
|
1112
|
+
global _dot_part, _dot_partb
|
|
1113
|
+
pid = f"{_os.getpid()}." if add_pid else ""
|
|
1114
|
+
_dot_part = f".{pid}{prog}{ext}"
|
|
1115
|
+
_dot_partb = _dot_part.encode("ascii")
|