kisstdlib 0.0.11__tar.gz → 0.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/PKG-INFO +1 -1
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/fs.py +160 -160
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/adapter.py +29 -2
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/itertools_ext.py +1 -1
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/parsing.py +120 -2
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/sqlite3_ext.py +1 -1
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/PKG-INFO +1 -1
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/SOURCES.txt +1 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/pyproject.toml +1 -1
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/test/test_basics.py +3 -2
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/test/test_io.py +15 -0
- kisstdlib-0.0.12/test/test_parsing.py +49 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/LICENSE.txt +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/README.md +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/__init__.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/argparse_ext.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/base.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/compression.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/failure.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/getpass_ext.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/__init__.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/base.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/encoder.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/stdio.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/io/wrapper.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/logging_ext.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/py.typed +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/pyrepr.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/signal_ext.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/sorted.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/string_ext.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/tariter.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib/time.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/dependency_links.txt +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/entry_points.txt +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/requires.txt +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib.egg-info/top_level.txt +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/kisstdlib_bin/describe_forest.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/setup.cfg +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/setup.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/test/test_fs.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/test/test_sorted.py +0 -0
- {kisstdlib-0.0.11 → kisstdlib-0.0.12}/test/test_time.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: kisstdlib
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12
|
|
4
4
|
Summary: System programming with conceptually and algebraically simple primitives.
|
|
5
5
|
Author-email: Jan Malakhovski <oxij@oxij.org>
|
|
6
6
|
Project-URL: Homepage, https://oxij.org/software/kisstdlib/
|
|
@@ -67,7 +67,7 @@ def realdir(path: _t.AnyStr, strict: bool = False) -> _t.AnyStr:
|
|
|
67
67
|
Essentially, this returns the canonical path of the inode the `path` points to, even when that
|
|
68
68
|
inode is a symlink.
|
|
69
69
|
"""
|
|
70
|
-
dn, bn = _op.split(path)
|
|
70
|
+
dn, bn = _op.split(_op.abspath(path))
|
|
71
71
|
return _op.join(_op.realpath(dn, strict=strict), bn)
|
|
72
72
|
|
|
73
73
|
|
|
@@ -468,111 +468,6 @@ def describe_forest(
|
|
|
468
468
|
yield [epath, "???"] + emode + emtime + esize
|
|
469
469
|
|
|
470
470
|
|
|
471
|
-
def unlink_maybe(path: str | bytes) -> None:
|
|
472
|
-
"""Try to `os.unlink` and ignore `Exception`s."""
|
|
473
|
-
try:
|
|
474
|
-
_os.unlink(path)
|
|
475
|
-
except Exception:
|
|
476
|
-
pass
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
def fsync_maybe(fd: int) -> None:
|
|
480
|
-
"""Try to `os.fsync` and ignore `errno.EINVAL` errors."""
|
|
481
|
-
try:
|
|
482
|
-
_os.fsync(fd)
|
|
483
|
-
except OSError as exc:
|
|
484
|
-
if exc.errno == _errno.EINVAL:
|
|
485
|
-
# EINVAL means fd is not attached to a file, so we
|
|
486
|
-
# ignore this error
|
|
487
|
-
return
|
|
488
|
-
raise
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
def fsync_file(path: str | bytes, flags: int = 0) -> None:
|
|
492
|
-
"""Run `os.fsync` on a given `path`."""
|
|
493
|
-
oflags = _os.O_RDONLY | _os.O_NOFOLLOW | _os.O_CLOEXEC if _POSIX else _os.O_RDWR
|
|
494
|
-
try:
|
|
495
|
-
fd = _os.open(path, oflags | flags)
|
|
496
|
-
except OSError as exc:
|
|
497
|
-
if exc.errno == _errno.ELOOP:
|
|
498
|
-
# ignore symlinks; doing it this way insead of `stat`ing the path to
|
|
499
|
-
# ensure atomicity
|
|
500
|
-
return
|
|
501
|
-
raise
|
|
502
|
-
try:
|
|
503
|
-
_os.fsync(fd)
|
|
504
|
-
except OSError as exc:
|
|
505
|
-
exc.filename = path
|
|
506
|
-
raise exc
|
|
507
|
-
finally:
|
|
508
|
-
_os.close(fd)
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
def unlink_many(
|
|
512
|
-
paths: _t.Iterable[_t.AnyStr], exceptions: list[Exception] | None = None
|
|
513
|
-
) -> list[_t.AnyStr]:
|
|
514
|
-
"""`os.unlink` many paths, optionally collecting exceptions.
|
|
515
|
-
Returns the paths for which `os.unlink` failed.
|
|
516
|
-
"""
|
|
517
|
-
left = []
|
|
518
|
-
for path in paths:
|
|
519
|
-
try:
|
|
520
|
-
_os.unlink(path)
|
|
521
|
-
except Exception as exc:
|
|
522
|
-
if exceptions is None:
|
|
523
|
-
raise
|
|
524
|
-
left.append(path)
|
|
525
|
-
exceptions.append(exc)
|
|
526
|
-
return left
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
def fsync_many_files(
|
|
530
|
-
paths: _t.Iterable[_t.AnyStr], flags: int = 0, exceptions: list[Exception] | None = None
|
|
531
|
-
) -> list[_t.AnyStr]:
|
|
532
|
-
"""`os.fsync` many paths, optionally collecting exceptions.
|
|
533
|
-
Returns the paths for which `os.fsync` failed.
|
|
534
|
-
"""
|
|
535
|
-
left = []
|
|
536
|
-
for path in paths:
|
|
537
|
-
try:
|
|
538
|
-
fsync_file(path, flags)
|
|
539
|
-
except Exception as exc:
|
|
540
|
-
if exceptions is None:
|
|
541
|
-
raise
|
|
542
|
-
left.append(path)
|
|
543
|
-
exceptions.append(exc)
|
|
544
|
-
return left
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
def rename(
|
|
548
|
-
src_path: _t.AnyStr,
|
|
549
|
-
dst_path: _t.AnyStr,
|
|
550
|
-
allow_overwrites: bool,
|
|
551
|
-
*,
|
|
552
|
-
makedirs: bool = True,
|
|
553
|
-
dst_dir: _t.AnyStr | None = None,
|
|
554
|
-
) -> None:
|
|
555
|
-
if dst_dir is None:
|
|
556
|
-
dst_dir, nondot = dirname_dot(dst_path)
|
|
557
|
-
makedirs = makedirs and nondot
|
|
558
|
-
|
|
559
|
-
if makedirs:
|
|
560
|
-
_os.makedirs(dst_dir, exist_ok=True)
|
|
561
|
-
|
|
562
|
-
if allow_overwrites:
|
|
563
|
-
_os.replace(src_path, dst_path)
|
|
564
|
-
elif _POSIX:
|
|
565
|
-
with Directory(dst_dir) as d:
|
|
566
|
-
d.flock()
|
|
567
|
-
# this is now atomic
|
|
568
|
-
if _os.path.lexists(dst_path):
|
|
569
|
-
raise FileExistsError(_errno.EEXIST, _os.strerror(_errno.EEXIST), dst_path)
|
|
570
|
-
_os.rename(src_path, dst_path)
|
|
571
|
-
else:
|
|
572
|
-
# this is both atomic and fails with `FileExistsError` on Windows
|
|
573
|
-
_os.rename(src_path, dst_path)
|
|
574
|
-
|
|
575
|
-
|
|
576
471
|
DeferredRename = tuple[_t.AnyStr, _t.AnyStr, bool, _t.AnyStr, _t.AnyStr]
|
|
577
472
|
|
|
578
473
|
|
|
@@ -611,6 +506,12 @@ class DeferredSync(_t.Generic[_t.AnyStr]):
|
|
|
611
506
|
self.rename_file = _c.deque()
|
|
612
507
|
self._after = None
|
|
613
508
|
|
|
509
|
+
@property
|
|
510
|
+
def after(self) -> "DeferredSync[_t.AnyStr]":
|
|
511
|
+
if self._after is None:
|
|
512
|
+
self._after = DeferredSync(self.defer)
|
|
513
|
+
return self._after
|
|
514
|
+
|
|
614
515
|
def __repr__(self) -> str:
|
|
615
516
|
return (
|
|
616
517
|
f"""<{self.__class__.__name__}
|
|
@@ -625,12 +526,6 @@ after="""
|
|
|
625
526
|
+ ">"
|
|
626
527
|
)
|
|
627
528
|
|
|
628
|
-
@property
|
|
629
|
-
def after(self) -> "DeferredSync[_t.AnyStr]":
|
|
630
|
-
if self._after is None:
|
|
631
|
-
self._after = DeferredSync(self.defer)
|
|
632
|
-
return self._after
|
|
633
|
-
|
|
634
529
|
def copy(self) -> "DeferredSync[_t.AnyStr]":
|
|
635
530
|
"""Return a shallow copy of this object (elements describing operations are not
|
|
636
531
|
copied, only the structure is).
|
|
@@ -803,6 +698,135 @@ after="""
|
|
|
803
698
|
self.clear()
|
|
804
699
|
|
|
805
700
|
|
|
701
|
+
def fsync_maybe(fd: int) -> None:
|
|
702
|
+
"""Try to `os.fsync` and ignore `errno.EINVAL` errors."""
|
|
703
|
+
try:
|
|
704
|
+
_os.fsync(fd)
|
|
705
|
+
except OSError as exc:
|
|
706
|
+
if exc.errno == _errno.EINVAL:
|
|
707
|
+
# EINVAL means fd is not attached to a file, so we
|
|
708
|
+
# ignore this error
|
|
709
|
+
return
|
|
710
|
+
raise
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def fsync_file(path: str | bytes, flags: int = 0) -> None:
|
|
714
|
+
"""Run `os.fsync` on a given `path`."""
|
|
715
|
+
oflags = _os.O_RDONLY | _os.O_NOFOLLOW | _os.O_CLOEXEC if _POSIX else _os.O_RDWR
|
|
716
|
+
try:
|
|
717
|
+
fd = _os.open(path, oflags | flags)
|
|
718
|
+
except OSError as exc:
|
|
719
|
+
if exc.errno == _errno.ELOOP:
|
|
720
|
+
# ignore symlinks; doing it this way insead of `stat`ing the path to
|
|
721
|
+
# ensure atomicity
|
|
722
|
+
return
|
|
723
|
+
raise
|
|
724
|
+
try:
|
|
725
|
+
_os.fsync(fd)
|
|
726
|
+
except OSError as exc:
|
|
727
|
+
exc.filename = path
|
|
728
|
+
raise exc
|
|
729
|
+
finally:
|
|
730
|
+
_os.close(fd)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def fsync_many_files(
|
|
734
|
+
paths: _t.Iterable[_t.AnyStr], flags: int = 0, exceptions: list[Exception] | None = None
|
|
735
|
+
) -> list[_t.AnyStr]:
|
|
736
|
+
"""`os.fsync` many paths, optionally collecting exceptions.
|
|
737
|
+
Returns the paths for which `os.fsync` failed.
|
|
738
|
+
"""
|
|
739
|
+
left = []
|
|
740
|
+
for path in paths:
|
|
741
|
+
try:
|
|
742
|
+
fsync_file(path, flags)
|
|
743
|
+
except Exception as exc:
|
|
744
|
+
if exceptions is None:
|
|
745
|
+
raise
|
|
746
|
+
left.append(path)
|
|
747
|
+
exceptions.append(exc)
|
|
748
|
+
return left
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def unlink_maybe(path: str | bytes) -> None:
|
|
752
|
+
"""Try to `os.unlink` and ignore `Exception`s."""
|
|
753
|
+
try:
|
|
754
|
+
_os.unlink(path)
|
|
755
|
+
except Exception:
|
|
756
|
+
pass
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def unlink_many(
|
|
760
|
+
paths: _t.Iterable[_t.AnyStr], exceptions: list[Exception] | None = None
|
|
761
|
+
) -> list[_t.AnyStr]:
|
|
762
|
+
"""`os.unlink` many paths, optionally collecting exceptions.
|
|
763
|
+
Returns the paths for which `os.unlink` failed.
|
|
764
|
+
"""
|
|
765
|
+
left = []
|
|
766
|
+
for path in paths:
|
|
767
|
+
try:
|
|
768
|
+
_os.unlink(path)
|
|
769
|
+
except Exception as exc:
|
|
770
|
+
if exceptions is None:
|
|
771
|
+
raise
|
|
772
|
+
left.append(path)
|
|
773
|
+
exceptions.append(exc)
|
|
774
|
+
return left
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def atomic_unlink(
|
|
778
|
+
path: _t.AnyStr,
|
|
779
|
+
*,
|
|
780
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
781
|
+
) -> None:
|
|
782
|
+
"""Atomically unlink `path`."""
|
|
783
|
+
|
|
784
|
+
dirname, _ = dirname_dot(path)
|
|
785
|
+
|
|
786
|
+
if isinstance(sync, DeferredSync) and sync.defer:
|
|
787
|
+
sync.unlink_file.add(path)
|
|
788
|
+
sync.fsync_dir.add(dirname)
|
|
789
|
+
return
|
|
790
|
+
|
|
791
|
+
_os.unlink(path)
|
|
792
|
+
|
|
793
|
+
if isinstance(sync, DeferredSync):
|
|
794
|
+
sync.fsync_dir.add(dirname)
|
|
795
|
+
return
|
|
796
|
+
|
|
797
|
+
if sync and _POSIX:
|
|
798
|
+
fsync_file(dirname, _os.O_DIRECTORY)
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def rename(
|
|
802
|
+
src_path: _t.AnyStr,
|
|
803
|
+
dst_path: _t.AnyStr,
|
|
804
|
+
allow_overwrites: bool,
|
|
805
|
+
*,
|
|
806
|
+
makedirs: bool = True,
|
|
807
|
+
dst_dir: _t.AnyStr | None = None,
|
|
808
|
+
) -> None:
|
|
809
|
+
if dst_dir is None:
|
|
810
|
+
dst_dir, nondot = dirname_dot(dst_path)
|
|
811
|
+
makedirs = makedirs and nondot
|
|
812
|
+
|
|
813
|
+
if makedirs:
|
|
814
|
+
_os.makedirs(dst_dir, exist_ok=True)
|
|
815
|
+
|
|
816
|
+
if allow_overwrites:
|
|
817
|
+
_os.replace(src_path, dst_path)
|
|
818
|
+
elif _POSIX:
|
|
819
|
+
with Directory(dst_dir) as d:
|
|
820
|
+
d.flock()
|
|
821
|
+
# this is now atomic
|
|
822
|
+
if _os.path.lexists(dst_path):
|
|
823
|
+
raise FileExistsError(_errno.EEXIST, _os.strerror(_errno.EEXIST), dst_path)
|
|
824
|
+
_os.rename(src_path, dst_path)
|
|
825
|
+
else:
|
|
826
|
+
# this is both atomic and fails with `FileExistsError` on Windows
|
|
827
|
+
_os.rename(src_path, dst_path)
|
|
828
|
+
|
|
829
|
+
|
|
806
830
|
def atomic_rename(
|
|
807
831
|
src_path: _t.AnyStr,
|
|
808
832
|
dst_path: _t.AnyStr,
|
|
@@ -925,6 +949,30 @@ def atomic_make_file(
|
|
|
925
949
|
# NB: src_dir == dst_dir
|
|
926
950
|
|
|
927
951
|
|
|
952
|
+
def atomic_write(
|
|
953
|
+
data: bytes,
|
|
954
|
+
dst_path: _t.AnyStr,
|
|
955
|
+
allow_overwrites: bool = False,
|
|
956
|
+
*,
|
|
957
|
+
makedirs: bool = True,
|
|
958
|
+
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
959
|
+
) -> None:
|
|
960
|
+
"""Atomically write given `data` to `dst_path`."""
|
|
961
|
+
|
|
962
|
+
def make_dst(tmp_path: _t.AnyStr, fsync_immediately: bool) -> None:
|
|
963
|
+
try:
|
|
964
|
+
with open(tmp_path, "xb") as fdst:
|
|
965
|
+
fdst.write(data)
|
|
966
|
+
fdst.flush()
|
|
967
|
+
if fsync_immediately:
|
|
968
|
+
_os.fsync(fdst.fileno())
|
|
969
|
+
except Exception:
|
|
970
|
+
unlink_maybe(tmp_path)
|
|
971
|
+
raise
|
|
972
|
+
|
|
973
|
+
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
974
|
+
|
|
975
|
+
|
|
928
976
|
def atomic_copy2(
|
|
929
977
|
src_path: _t.AnyStr,
|
|
930
978
|
dst_path: _t.AnyStr,
|
|
@@ -1056,54 +1104,6 @@ def atomic_move(
|
|
|
1056
1104
|
fsync_file(src_dir, _os.O_DIRECTORY)
|
|
1057
1105
|
|
|
1058
1106
|
|
|
1059
|
-
def atomic_write(
|
|
1060
|
-
data: bytes,
|
|
1061
|
-
dst_path: _t.AnyStr,
|
|
1062
|
-
allow_overwrites: bool = False,
|
|
1063
|
-
*,
|
|
1064
|
-
makedirs: bool = True,
|
|
1065
|
-
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1066
|
-
) -> None:
|
|
1067
|
-
"""Atomically write given `data` to `dst_path`."""
|
|
1068
|
-
|
|
1069
|
-
def make_dst(tmp_path: _t.AnyStr, fsync_immediately: bool) -> None:
|
|
1070
|
-
try:
|
|
1071
|
-
with open(tmp_path, "xb") as fdst:
|
|
1072
|
-
fdst.write(data)
|
|
1073
|
-
fdst.flush()
|
|
1074
|
-
if fsync_immediately:
|
|
1075
|
-
_os.fsync(fdst.fileno())
|
|
1076
|
-
except Exception:
|
|
1077
|
-
unlink_maybe(tmp_path)
|
|
1078
|
-
raise
|
|
1079
|
-
|
|
1080
|
-
atomic_make_file(make_dst, dst_path, allow_overwrites, makedirs=makedirs, sync=sync)
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
def atomic_unlink(
|
|
1084
|
-
path: _t.AnyStr,
|
|
1085
|
-
*,
|
|
1086
|
-
sync: DeferredSync[_t.AnyStr] | bool = True,
|
|
1087
|
-
) -> None:
|
|
1088
|
-
"""Atomically unlink `path`."""
|
|
1089
|
-
|
|
1090
|
-
dirname, _ = dirname_dot(path)
|
|
1091
|
-
|
|
1092
|
-
if isinstance(sync, DeferredSync) and sync.defer:
|
|
1093
|
-
sync.unlink_file.add(path)
|
|
1094
|
-
sync.fsync_dir.add(dirname)
|
|
1095
|
-
return
|
|
1096
|
-
|
|
1097
|
-
_os.unlink(path)
|
|
1098
|
-
|
|
1099
|
-
if isinstance(sync, DeferredSync):
|
|
1100
|
-
sync.fsync_dir.add(dirname)
|
|
1101
|
-
return
|
|
1102
|
-
|
|
1103
|
-
if sync and _POSIX:
|
|
1104
|
-
fsync_file(dirname, _os.O_DIRECTORY)
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
1107
|
def setup_fs(prog: str | None = None, ext: str = ".part", add_pid: bool = True) -> None:
|
|
1108
1108
|
"""Setup temporary files prefix."""
|
|
1109
1109
|
if prog is None:
|
|
@@ -23,8 +23,15 @@
|
|
|
23
23
|
"""Adapt almost any folding function into `IOBase` interface."""
|
|
24
24
|
|
|
25
25
|
import typing as _t
|
|
26
|
+
|
|
26
27
|
from zlib import compressobj as _compressobj, decompressobj as _decompressobj
|
|
27
28
|
|
|
29
|
+
from ..parsing import (
|
|
30
|
+
BytesTransformer as _BytesTransformer,
|
|
31
|
+
LF2CRLF as _LF2CRLF,
|
|
32
|
+
CRLF2LF as _CRLF2LF,
|
|
33
|
+
)
|
|
34
|
+
|
|
28
35
|
|
|
29
36
|
class IOAdapter:
|
|
30
37
|
def __init__(self, fobj: _t.Any) -> None:
|
|
@@ -73,7 +80,7 @@ class ReadAdapter(IOAdapter):
|
|
|
73
80
|
self._buffer += self._handle_data(data)
|
|
74
81
|
|
|
75
82
|
if len(self._buffer) == 0:
|
|
76
|
-
return
|
|
83
|
+
return self._buffer
|
|
77
84
|
|
|
78
85
|
if size == -1 or len(self._buffer) == size:
|
|
79
86
|
res = self._buffer
|
|
@@ -89,7 +96,7 @@ class ReadAdapter(IOAdapter):
|
|
|
89
96
|
|
|
90
97
|
|
|
91
98
|
class UpdateFinalizeReader(ReadAdapter):
|
|
92
|
-
def __init__(self,
|
|
99
|
+
def __init__(self, fobj: _t.Any, preprocessor: _t.Any, block_size: int = 4096) -> None:
|
|
93
100
|
super().__init__(fobj, block_size)
|
|
94
101
|
self._preprocessor = preprocessor
|
|
95
102
|
|
|
@@ -114,6 +121,26 @@ class UpdateFinalizeWriter(IOAdapter):
|
|
|
114
121
|
self._fobj.close()
|
|
115
122
|
|
|
116
123
|
|
|
124
|
+
class DOS2UNIXReader(UpdateFinalizeReader):
|
|
125
|
+
def __init__(self, fobj: _t.Any, block_size: int = 4096) -> None:
|
|
126
|
+
super().__init__(fobj, _BytesTransformer(_CRLF2LF), block_size)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class UNIX2DOSReader(UpdateFinalizeReader):
|
|
130
|
+
def __init__(self, fobj: _t.Any, block_size: int = 4096) -> None:
|
|
131
|
+
super().__init__(fobj, _BytesTransformer(_LF2CRLF), block_size)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class DOS2UNIXWriter(UpdateFinalizeWriter):
|
|
135
|
+
def __init__(self, fobj: _t.Any) -> None:
|
|
136
|
+
super().__init__(fobj, _BytesTransformer(_CRLF2LF))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class UNIX2DOSWriter(UpdateFinalizeWriter):
|
|
140
|
+
def __init__(self, fobj: _t.Any) -> None:
|
|
141
|
+
super().__init__(fobj, _BytesTransformer(_LF2CRLF))
|
|
142
|
+
|
|
143
|
+
|
|
117
144
|
class ZlibDecompressor(ReadAdapter):
|
|
118
145
|
def __init__(self, fobj: _t.Any, block_size: int, **kwargs: _t.Any) -> None:
|
|
119
146
|
super().__init__(fobj, block_size)
|
|
@@ -20,13 +20,131 @@
|
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
21
|
# SOFTWARE.
|
|
22
22
|
|
|
23
|
-
"""
|
|
23
|
+
"""Parsing of `str`ings and `bytes`."""
|
|
24
24
|
|
|
25
25
|
import re as _re
|
|
26
26
|
import typing as _t
|
|
27
27
|
|
|
28
|
+
from itertools import islice as _islice
|
|
29
|
+
|
|
28
30
|
from .failure import ParsingFailure
|
|
29
31
|
|
|
32
|
+
|
|
33
|
+
# A state transition function, see `BytesTransformer` below.
|
|
34
|
+
ByteTransitions: _t.TypeAlias = _t.Callable[[int, int], tuple[int, bytes] | None]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def LF2CRLF(_s: int, c: int) -> tuple[int, bytes] | None:
|
|
38
|
+
"Map all LF to CR LF."
|
|
39
|
+
if c == 10:
|
|
40
|
+
# LF
|
|
41
|
+
return 0, b"\r\n"
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def CRLF2LF(s: int, c: int) -> tuple[int, bytes] | None:
|
|
46
|
+
"Map all CR LF sequences to LF."
|
|
47
|
+
if s == 0:
|
|
48
|
+
if c == 13:
|
|
49
|
+
# CR
|
|
50
|
+
return 1, b""
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
if c == 10:
|
|
54
|
+
# CR LF
|
|
55
|
+
return 0, b"\n"
|
|
56
|
+
if c == 13:
|
|
57
|
+
# CR CR
|
|
58
|
+
return 1, b"\r"
|
|
59
|
+
if c == -1:
|
|
60
|
+
# CR EOF
|
|
61
|
+
return 0, b"\r"
|
|
62
|
+
|
|
63
|
+
# CR other
|
|
64
|
+
return 0, bytes([13, c])
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class BytesTransformer:
|
|
68
|
+
"""Transform some `bytes` using a given `ByteTransitions` somewhat similarly to
|
|
69
|
+
`bytearray.transform`, but with more expressive power.
|
|
70
|
+
|
|
71
|
+
This class implements the same `update` + `finalize` API as most cryptographic libraries.
|
|
72
|
+
|
|
73
|
+
`ByteTransitions` is a function from old state number (the first `int`) and a char (the second
|
|
74
|
+
`int`) to new state number and generated `bytes` output. The `None` result means "keep the same
|
|
75
|
+
state, output the input byte". The initial state is `0`, EOF is represented by a `-1` byte.
|
|
76
|
+
|
|
77
|
+
See, e.g. `UNIX2DOS*` and `DOS2UNIX*` in `kisstdlib.io.adapter` for example usages.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
def __init__(self, transitions: ByteTransitions) -> None:
|
|
81
|
+
self._transitions = transitions
|
|
82
|
+
self._stateno = 0
|
|
83
|
+
|
|
84
|
+
def update(self, data: bytes) -> bytes:
|
|
85
|
+
i = 0
|
|
86
|
+
dlen = len(data)
|
|
87
|
+
|
|
88
|
+
stateno = self._stateno
|
|
89
|
+
tf = self._transitions
|
|
90
|
+
|
|
91
|
+
c: int
|
|
92
|
+
t: tuple[int, bytes] | None
|
|
93
|
+
|
|
94
|
+
# skip noops
|
|
95
|
+
while i < dlen:
|
|
96
|
+
c = data[i]
|
|
97
|
+
t = tf(stateno, c)
|
|
98
|
+
if t is None:
|
|
99
|
+
i += 1
|
|
100
|
+
continue
|
|
101
|
+
# nstateno, nout = t
|
|
102
|
+
# if nstateno == stateno and len(nout) == 0 and nout[0] == c:
|
|
103
|
+
# i += 1
|
|
104
|
+
# continue
|
|
105
|
+
break
|
|
106
|
+
else:
|
|
107
|
+
return data
|
|
108
|
+
|
|
109
|
+
if i == dlen:
|
|
110
|
+
# there were only noops
|
|
111
|
+
return data
|
|
112
|
+
|
|
113
|
+
# copy the noop part
|
|
114
|
+
res = bytearray(_islice(data, i))
|
|
115
|
+
|
|
116
|
+
# update on the current char
|
|
117
|
+
stateno, nout = t
|
|
118
|
+
res.extend(nout)
|
|
119
|
+
i += 1
|
|
120
|
+
|
|
121
|
+
# update on the rest
|
|
122
|
+
while i < dlen:
|
|
123
|
+
c = data[i]
|
|
124
|
+
t = tf(stateno, c)
|
|
125
|
+
if t is None:
|
|
126
|
+
res.append(c)
|
|
127
|
+
else:
|
|
128
|
+
stateno, nout = t
|
|
129
|
+
res.extend(nout)
|
|
130
|
+
i += 1
|
|
131
|
+
|
|
132
|
+
self._stateno = stateno
|
|
133
|
+
return bytes(res)
|
|
134
|
+
|
|
135
|
+
def finalize(self) -> bytes:
|
|
136
|
+
t = self._transitions(self._stateno, -1)
|
|
137
|
+
if t is None:
|
|
138
|
+
self._stateno, nout = 0, b""
|
|
139
|
+
else:
|
|
140
|
+
self._stateno, nout = t
|
|
141
|
+
|
|
142
|
+
# ensure it gets reset properly
|
|
143
|
+
assert self._stateno == 0
|
|
144
|
+
|
|
145
|
+
return nout
|
|
146
|
+
|
|
147
|
+
|
|
30
148
|
word_re = _re.compile(r"(\S+)")
|
|
31
149
|
natural_re = _re.compile(r"([0-9]+)")
|
|
32
150
|
integer_re = _re.compile(r"(-?[0-9]+)")
|
|
@@ -40,7 +158,7 @@ ParserParamSpec = _t.ParamSpec("ParserParamSpec")
|
|
|
40
158
|
|
|
41
159
|
|
|
42
160
|
class Parser:
|
|
43
|
-
"""
|
|
161
|
+
"""Parsec-like parser combinator with regexes."""
|
|
44
162
|
|
|
45
163
|
def __init__(self, data: str) -> None:
|
|
46
164
|
self.buffer = data
|
|
@@ -340,7 +340,7 @@ class AppDB(_t.Generic[SettingsType]):
|
|
|
340
340
|
self.commit()
|
|
341
341
|
|
|
342
342
|
self.logger.debug(
|
|
343
|
-
"`%s`:
|
|
343
|
+
"`%s`: finished init at version %d with settings of `%s`",
|
|
344
344
|
path,
|
|
345
345
|
self._version,
|
|
346
346
|
str(self._settings),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: kisstdlib
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12
|
|
4
4
|
Summary: System programming with conceptually and algebraically simple primitives.
|
|
5
5
|
Author-email: Jan Malakhovski <oxij@oxij.org>
|
|
6
6
|
Project-URL: Homepage, https://oxij.org/software/kisstdlib/
|
|
@@ -9,7 +9,7 @@ namespaces = true
|
|
|
9
9
|
"kisstdlib" = ["py.typed"]
|
|
10
10
|
[project]
|
|
11
11
|
name = "kisstdlib"
|
|
12
|
-
version = "0.0.
|
|
12
|
+
version = "0.0.12"
|
|
13
13
|
authors = [{ name = "Jan Malakhovski", email = "oxij@oxij.org" }]
|
|
14
14
|
description = "System programming with conceptually and algebraically simple primitives."
|
|
15
15
|
readme = "README.md"
|
|
@@ -48,6 +48,7 @@ def test_basics() -> None:
|
|
|
48
48
|
assert first([123], None) == first((123,), None) == 123
|
|
49
49
|
assert last([123], None) == last((123,), None) == 123
|
|
50
50
|
assert nth(0, [], None) == nth(0, (), None) == None
|
|
51
|
+
assert nth(0, [123], None) == 123
|
|
51
52
|
|
|
52
53
|
e: _t.Any
|
|
53
54
|
for e in ((), []):
|
|
@@ -78,8 +79,8 @@ def test_basics() -> None:
|
|
|
78
79
|
for i in range(0, len(l) + 1):
|
|
79
80
|
assert drop(i, l) == l[i:]
|
|
80
81
|
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
for i in range(0, len(l) + 1):
|
|
83
|
+
assert nth(i, l, None) == nth(i, t, None) == (l[i] if i < len(l) else None)
|
|
83
84
|
|
|
84
85
|
l.pop()
|
|
85
86
|
|
|
@@ -26,9 +26,24 @@ import io as _io
|
|
|
26
26
|
import typing as _t
|
|
27
27
|
|
|
28
28
|
from kisstdlib.io.base import *
|
|
29
|
+
from kisstdlib.io.adapter import *
|
|
29
30
|
from kisstdlib.io.wrapper import *
|
|
30
31
|
|
|
31
32
|
|
|
33
|
+
def test_adapters() -> None:
|
|
34
|
+
buf = _io.BytesIO()
|
|
35
|
+
|
|
36
|
+
u2d = UNIX2DOSWriter(buf)
|
|
37
|
+
orig = b"Hello, World!\nFoo\rBar\r"
|
|
38
|
+
u2d.write(orig)
|
|
39
|
+
assert buf.getvalue() == b"Hello, World!\r\nFoo\rBar\r"
|
|
40
|
+
|
|
41
|
+
buf.seek(0)
|
|
42
|
+
|
|
43
|
+
d2u = DOS2UNIXReader(buf)
|
|
44
|
+
assert d2u.read() == orig
|
|
45
|
+
|
|
46
|
+
|
|
32
47
|
def test_ansi_set() -> None:
|
|
33
48
|
def check(*args: _t.Any, **kwargs: _t.Any) -> bytes:
|
|
34
49
|
x = TIOWrappedWriter(_io.BytesIO(), ansi=True)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Copyright (c) 2025 Jan Malakhovski <oxij@oxij.org>
|
|
2
|
+
#
|
|
3
|
+
# This file is a part of `kisstdlib` project.
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
# furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
# copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
# SOFTWARE.
|
|
22
|
+
|
|
23
|
+
"""Testing `kisstdlib.parsing` module."""
|
|
24
|
+
|
|
25
|
+
import typing as _t
|
|
26
|
+
|
|
27
|
+
from kisstdlib.parsing import *
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_BytesTransformer() -> None:
|
|
31
|
+
unix2dos = BytesTransformer(LF2CRLF)
|
|
32
|
+
dos2unix = BytesTransformer(CRLF2LF)
|
|
33
|
+
|
|
34
|
+
def check(a: bytes, b: bytes) -> None:
|
|
35
|
+
assert unix2dos.update(a) == b
|
|
36
|
+
assert dos2unix.update(b) == a
|
|
37
|
+
|
|
38
|
+
check(b"", b"")
|
|
39
|
+
check(b"\n", b"\r\n")
|
|
40
|
+
check(b"foo\n", b"foo\r\n")
|
|
41
|
+
check(b"foo\nbar", b"foo\r\nbar")
|
|
42
|
+
check(b"foo\r\n", b"foo\r\r\n")
|
|
43
|
+
check(b"foo\r\nbar", b"foo\r\r\nbar")
|
|
44
|
+
|
|
45
|
+
assert dos2unix.update(b"foo\r") == b"foo"
|
|
46
|
+
assert dos2unix.update(b"bar") == b"\rbar"
|
|
47
|
+
|
|
48
|
+
assert dos2unix.update(b"foo\r") == b"foo"
|
|
49
|
+
assert dos2unix.update(b"\n") == b"\n"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|