PyFoxFile 0.24.0__py3-none-any.whl → 0.24.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyfoxfile-0.24.0.data → pyfoxfile-0.24.4.data}/scripts/foxfile.py +32 -52
- {pyfoxfile-0.24.0.dist-info → pyfoxfile-0.24.4.dist-info}/METADATA +1 -1
- pyfoxfile-0.24.4.dist-info/RECORD +10 -0
- pyfoxfile.py +496 -688
- pyfoxfile-0.24.0.dist-info/RECORD +0 -10
- {pyfoxfile-0.24.0.data → pyfoxfile-0.24.4.data}/scripts/foxneofile.py +0 -0
- {pyfoxfile-0.24.0.data → pyfoxfile-0.24.4.data}/scripts/neofoxfile.py +0 -0
- {pyfoxfile-0.24.0.dist-info → pyfoxfile-0.24.4.dist-info}/WHEEL +0 -0
- {pyfoxfile-0.24.0.dist-info → pyfoxfile-0.24.4.dist-info}/licenses/LICENSE +0 -0
- {pyfoxfile-0.24.0.dist-info → pyfoxfile-0.24.4.dist-info}/top_level.txt +0 -0
- {pyfoxfile-0.24.0.dist-info → pyfoxfile-0.24.4.dist-info}/zip-safe +0 -0
pyfoxfile.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
|
|
15
15
|
Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
|
|
16
16
|
|
|
17
|
-
$FileInfo: pyfoxfile.py - Last Update:
|
|
17
|
+
$FileInfo: pyfoxfile.py - Last Update: 11/3/2025 Ver. 0.24.4 RC 1 - Author: cooldude2k $
|
|
18
18
|
'''
|
|
19
19
|
|
|
20
20
|
from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
|
|
@@ -33,10 +33,10 @@ import socket
|
|
|
33
33
|
import struct
|
|
34
34
|
import hashlib
|
|
35
35
|
import inspect
|
|
36
|
-
import datetime
|
|
37
36
|
import logging
|
|
38
37
|
import zipfile
|
|
39
38
|
import binascii
|
|
39
|
+
import datetime
|
|
40
40
|
import platform
|
|
41
41
|
from io import StringIO, BytesIO
|
|
42
42
|
from collections import namedtuple
|
|
@@ -413,7 +413,12 @@ def is_only_nonprintable(var):
|
|
|
413
413
|
__file_format_multi_dict__ = {}
|
|
414
414
|
__file_format_default__ = "FoxFile"
|
|
415
415
|
__include_defaults__ = True
|
|
416
|
-
__use_inmemfile__ =
|
|
416
|
+
__use_inmemfile__ = True
|
|
417
|
+
__use_spoolfile__ = False
|
|
418
|
+
__use_spooldir__ = tempfile.gettempdir()
|
|
419
|
+
BYTES_PER_MiB = 1024 * 1024
|
|
420
|
+
DEFAULT_SPOOL_MAX = 8 * BYTES_PER_MiB
|
|
421
|
+
__spoolfile_size__ = DEFAULT_SPOOL_MAX
|
|
417
422
|
__program_name__ = "Py"+__file_format_default__
|
|
418
423
|
__use_env_file__ = True
|
|
419
424
|
__use_ini_file__ = True
|
|
@@ -450,6 +455,8 @@ if __use_ini_file__ and os.path.exists(__config_file__):
|
|
|
450
455
|
__program_name__ = decode_unicode_escape(config.get('config', 'proname'))
|
|
451
456
|
__include_defaults__ = config.getboolean('config', 'includedef')
|
|
452
457
|
__use_inmemfile__ = config.getboolean('config', 'inmemfile')
|
|
458
|
+
__use_spoolfile__ = config.getboolean('config', 'usespoolfile')
|
|
459
|
+
__spoolfile_size__ = config.getint('config', 'spoolfilesize')
|
|
453
460
|
# Loop through all sections
|
|
454
461
|
for section in config.sections():
|
|
455
462
|
if section == "config":
|
|
@@ -543,6 +550,8 @@ elif __use_json_file__ and os.path.exists(__config_file__):
|
|
|
543
550
|
__program_name__ = decode_unicode_escape(_get(cfg_config, 'proname', ''))
|
|
544
551
|
__include_defaults__ = _to_bool(_get(cfg_config, 'includedef', False))
|
|
545
552
|
__use_inmemfile__ = _to_bool(_get(cfg_config, 'inmemfile', False))
|
|
553
|
+
__use_spoolfile__ = _to_bool(_get(cfg_config, 'usespoolfile', False))
|
|
554
|
+
__spoolfile_size__ = _to_int(_get(cfg_config, 'spoolfilesize', DEFAULT_SPOOL_MAX))
|
|
546
555
|
|
|
547
556
|
# --- iterate format sections (everything except "config") ---
|
|
548
557
|
required_keys = [
|
|
@@ -628,13 +637,15 @@ __use_alt_inode__ = __file_format_multi_dict__[__file_format_default__]['use_alt
|
|
|
628
637
|
__file_format_extension__ = __file_format_multi_dict__[__file_format_default__]['format_extension']
|
|
629
638
|
__file_format_dict__ = __file_format_multi_dict__[__file_format_default__]
|
|
630
639
|
__project__ = __program_name__
|
|
640
|
+
__program_alt_name__ = __program_name__
|
|
631
641
|
__project_url__ = "https://github.com/GameMaker2k/PyFoxFile"
|
|
632
|
-
|
|
633
|
-
|
|
642
|
+
__project_release_url__ = __project_url__+"/releases/latest"
|
|
643
|
+
__version_info__ = (0, 24, 4, "RC 1", 1)
|
|
644
|
+
__version_date_info__ = (2025, 11, 3, "RC 1", 1)
|
|
634
645
|
__version_date__ = str(__version_date_info__[0]) + "." + str(
|
|
635
646
|
__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2)
|
|
636
647
|
__revision__ = __version_info__[3]
|
|
637
|
-
__revision_id__ = "$Id:
|
|
648
|
+
__revision_id__ = "$Id: 3889d7a367d73addd635e5448a7febafff7919ad $"
|
|
638
649
|
if(__version_info__[4] is not None):
|
|
639
650
|
__version_date_plusrc__ = __version_date__ + \
|
|
640
651
|
"-" + str(__version_date_info__[4])
|
|
@@ -646,6 +657,78 @@ if(__version_info__[3] is not None):
|
|
|
646
657
|
if(__version_info__[3] is None):
|
|
647
658
|
__version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2])
|
|
648
659
|
|
|
660
|
+
# From: https://stackoverflow.com/a/28568003
|
|
661
|
+
# By Phaxmohdem
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def versiontuple(v):
|
|
665
|
+
filled = []
|
|
666
|
+
for point in v.split("."):
|
|
667
|
+
filled.append(point.zfill(8))
|
|
668
|
+
return tuple(filled)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def version_check(myvercheck, newvercheck):
|
|
672
|
+
vercheck = 0
|
|
673
|
+
try:
|
|
674
|
+
from packaging import version
|
|
675
|
+
vercheck = 1
|
|
676
|
+
except ImportError:
|
|
677
|
+
try:
|
|
678
|
+
from distutils.version import LooseVersion, StrictVersion
|
|
679
|
+
vercheck = 2
|
|
680
|
+
except ImportError:
|
|
681
|
+
try:
|
|
682
|
+
from pkg_resources import parse_version
|
|
683
|
+
vercheck = 3
|
|
684
|
+
except ImportError:
|
|
685
|
+
return 5
|
|
686
|
+
# print(myvercheck, newvercheck)
|
|
687
|
+
if (vercheck == 1):
|
|
688
|
+
if (version.parse(myvercheck) == version.parse(newvercheck)):
|
|
689
|
+
return 0
|
|
690
|
+
elif (version.parse(myvercheck) < version.parse(newvercheck)):
|
|
691
|
+
return 1
|
|
692
|
+
elif (version.parse(myvercheck) > version.parse(newvercheck)):
|
|
693
|
+
return 2
|
|
694
|
+
else:
|
|
695
|
+
return 3
|
|
696
|
+
elif (vercheck == 2):
|
|
697
|
+
if (StrictVersion(myvercheck) == StrictVersion(newvercheck)):
|
|
698
|
+
return 0
|
|
699
|
+
elif (StrictVersion(myvercheck) < StrictVersion(newvercheck)):
|
|
700
|
+
return 1
|
|
701
|
+
elif (StrictVersion(myvercheck) > StrictVersion(newvercheck)):
|
|
702
|
+
return 2
|
|
703
|
+
else:
|
|
704
|
+
return 3
|
|
705
|
+
elif (vercheck == 3):
|
|
706
|
+
if (parse_version(myvercheck) == parse_version(newvercheck)):
|
|
707
|
+
return 0
|
|
708
|
+
elif (parse_version(myvercheck) < parse_version(newvercheck)):
|
|
709
|
+
return 1
|
|
710
|
+
elif (parse_version(myvercheck) > parse_version(newvercheck)):
|
|
711
|
+
return 2
|
|
712
|
+
else:
|
|
713
|
+
return 3
|
|
714
|
+
else:
|
|
715
|
+
if (versiontuple(myvercheck) == versiontuple(newvercheck)):
|
|
716
|
+
return 0
|
|
717
|
+
elif (versiontuple(myvercheck) < versiontuple(newvercheck)):
|
|
718
|
+
return 1
|
|
719
|
+
elif (versiontuple(myvercheck) > versiontuple(newvercheck)):
|
|
720
|
+
return 2
|
|
721
|
+
else:
|
|
722
|
+
return 3
|
|
723
|
+
return 4
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def check_version_number(myversion=__version__, proname=__program_alt_name__, newverurl=__project_release_url__):
|
|
727
|
+
prevercheck = download_from_url(newverurl, geturls_headers, geturls_cj)
|
|
728
|
+
newvercheck = re.findall(proname + " ([0-9\\.]+)<\\/a\\>", prevercheck['Content'].decode("UTF-8"))[0]
|
|
729
|
+
myvercheck = re.findall("([0-9\\.]+)", myversion)[0]
|
|
730
|
+
return version_check(myvercheck, newvercheck)
|
|
731
|
+
|
|
649
732
|
# ===== Module-level type code table & helpers (reuse anywhere) =====
|
|
650
733
|
|
|
651
734
|
FT = {
|
|
@@ -746,6 +829,7 @@ try:
|
|
|
746
829
|
compressionsupport.append("lz4")
|
|
747
830
|
except ImportError:
|
|
748
831
|
pass
|
|
832
|
+
'''
|
|
749
833
|
try:
|
|
750
834
|
import lzo
|
|
751
835
|
compressionsupport.append("lzo")
|
|
@@ -753,6 +837,7 @@ try:
|
|
|
753
837
|
except ImportError:
|
|
754
838
|
lzo = None
|
|
755
839
|
pass
|
|
840
|
+
'''
|
|
756
841
|
try:
|
|
757
842
|
import zstandard
|
|
758
843
|
compressionsupport.append("zst")
|
|
@@ -812,11 +897,13 @@ if('lzo' in compressionsupport):
|
|
|
812
897
|
compressionlistalt.append('lzo')
|
|
813
898
|
outextlist.append('lzo')
|
|
814
899
|
outextlistwd.append('.lzo')
|
|
900
|
+
'''
|
|
815
901
|
if('lzop' in compressionsupport):
|
|
816
902
|
compressionlist.append('lzop')
|
|
817
903
|
compressionlistalt.append('lzop')
|
|
818
904
|
outextlist.append('lzop')
|
|
819
905
|
outextlistwd.append('.lzop')
|
|
906
|
+
'''
|
|
820
907
|
if('lzma' in compressionsupport):
|
|
821
908
|
compressionlist.append('lzma')
|
|
822
909
|
compressionlistalt.append('lzma')
|
|
@@ -1962,7 +2049,7 @@ def _normalize_initial_data(data, isbytes, encoding, errors=None):
|
|
|
1962
2049
|
|
|
1963
2050
|
|
|
1964
2051
|
def MkTempFile(data=None,
|
|
1965
|
-
inmem=
|
|
2052
|
+
inmem=__use_inmemfile__,
|
|
1966
2053
|
isbytes=True,
|
|
1967
2054
|
prefix="",
|
|
1968
2055
|
delete=True,
|
|
@@ -1970,9 +2057,9 @@ def MkTempFile(data=None,
|
|
|
1970
2057
|
newline=None, # text mode only; in-memory objects ignore newline semantics
|
|
1971
2058
|
dir=None,
|
|
1972
2059
|
suffix="",
|
|
1973
|
-
use_spool=
|
|
1974
|
-
spool_max=
|
|
1975
|
-
spool_dir=
|
|
2060
|
+
use_spool=__use_spoolfile__,
|
|
2061
|
+
spool_max=__spoolfile_size__,
|
|
2062
|
+
spool_dir=__use_spooldir__:
|
|
1976
2063
|
"""
|
|
1977
2064
|
Return a file-like handle with consistent behavior on Py2.7 and Py3.x.
|
|
1978
2065
|
|
|
@@ -2415,7 +2502,7 @@ class ZlibFile(object):
|
|
|
2415
2502
|
|
|
2416
2503
|
def __init__(self, file_path=None, fileobj=None, mode='rb', level=6, wbits=15,
|
|
2417
2504
|
encoding=None, errors=None, newline=None,
|
|
2418
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2505
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2419
2506
|
|
|
2420
2507
|
if file_path is None and fileobj is None:
|
|
2421
2508
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -2902,7 +2989,7 @@ class GzipFile(object):
|
|
|
2902
2989
|
|
|
2903
2990
|
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
2904
2991
|
level=6, encoding=None, errors=None, newline=None,
|
|
2905
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2992
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2906
2993
|
|
|
2907
2994
|
if file_path is None and fileobj is None:
|
|
2908
2995
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -3326,519 +3413,6 @@ def gzip_decompress_bytes_all_members(blob):
|
|
|
3326
3413
|
"""
|
|
3327
3414
|
return _gzip_decompress_multimember(bytes(blob))
|
|
3328
3415
|
|
|
3329
|
-
|
|
3330
|
-
# ---------- Simple LZO container (NOT real .lzop) ----------
|
|
3331
|
-
# File layout (concatenated members allowed):
|
|
3332
|
-
# [MAGIC 8B] [FLAGS 1B] [ULEN 8B] [CRC32 4B] [CCHUNK...] | repeat...
|
|
3333
|
-
# where:
|
|
3334
|
-
# MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3335
|
-
# FLAGS = bit0: 1 => member has ULEN+CRC, 0 => no header (legacy)
|
|
3336
|
-
# ULEN = uncompressed length (u64 BE)
|
|
3337
|
-
# CRC32 = CRC32 of uncompressed data (u32 BE)
|
|
3338
|
-
# CCHUNK = one or more compressed chunks:
|
|
3339
|
-
# [u32BE chunk_size][chunk_data] ... then a zero-size u32 terminator
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
class LzopFile(object):
|
|
3343
|
-
MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3344
|
-
FLAG_HAS_UHDR = 0x01
|
|
3345
|
-
RAW_CHUNK = 256 * 1024 # 256 KiB per raw (pre-compress) chunk
|
|
3346
|
-
|
|
3347
|
-
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
3348
|
-
level=9, encoding=None, errors=None, newline=None,
|
|
3349
|
-
write_header=True,
|
|
3350
|
-
tolerant_read=False, scan_bytes=(64 << 10),
|
|
3351
|
-
spool_threshold=(8 << 20)):
|
|
3352
|
-
"""
|
|
3353
|
-
Custom LZO file (NOT the lzop(1) format).
|
|
3354
|
-
- streaming write/read, supports concatenated members
|
|
3355
|
-
- optional per-member header (uncompressed length + CRC32)
|
|
3356
|
-
- spooled reads to limit RAM, strict text mode with newline control
|
|
3357
|
-
- tolerant_read: scan forward (up to scan_bytes) to first MAGIC
|
|
3358
|
-
|
|
3359
|
-
:param write_header: if True, include ULEN+CRC32 per member
|
|
3360
|
-
:param tolerant_read: skip leading junk up to scan_bytes to find MAGIC
|
|
3361
|
-
:param scan_bytes: max bytes to scan when tolerant_read=True
|
|
3362
|
-
:param spool_threshold: SpooledTemporaryFile RAM threshold before spill
|
|
3363
|
-
"""
|
|
3364
|
-
if lzo is None:
|
|
3365
|
-
raise ImportError("python-lzo is required for LzopFile")
|
|
3366
|
-
|
|
3367
|
-
if file_path is None and fileobj is None:
|
|
3368
|
-
raise ValueError("Either file_path or fileobj must be provided")
|
|
3369
|
-
if file_path is not None and fileobj is not None:
|
|
3370
|
-
raise ValueError("Only one of file_path or fileobj should be provided")
|
|
3371
|
-
|
|
3372
|
-
if 'b' not in mode and 't' not in mode:
|
|
3373
|
-
mode += 'b'
|
|
3374
|
-
if 'x' in mode and PY2:
|
|
3375
|
-
raise ValueError("Exclusive creation mode 'x' is not supported on Python 2")
|
|
3376
|
-
|
|
3377
|
-
self.file_path = file_path
|
|
3378
|
-
self.file = fileobj
|
|
3379
|
-
self.mode = mode
|
|
3380
|
-
self.level = int(level) # effective: 1 or 9 (clamped)
|
|
3381
|
-
self.encoding = encoding
|
|
3382
|
-
self.errors = errors
|
|
3383
|
-
self.newline = newline
|
|
3384
|
-
self._text_mode = ('t' in mode)
|
|
3385
|
-
|
|
3386
|
-
self._write_header = bool(write_header)
|
|
3387
|
-
|
|
3388
|
-
# Config (read path)
|
|
3389
|
-
self.tolerant_read = bool(tolerant_read)
|
|
3390
|
-
self.scan_bytes = int(scan_bytes)
|
|
3391
|
-
self.spool_threshold = int(spool_threshold)
|
|
3392
|
-
|
|
3393
|
-
# Write state
|
|
3394
|
-
self._crc = 0
|
|
3395
|
-
self._ulen = 0
|
|
3396
|
-
self._open_member = False
|
|
3397
|
-
self._member_header_pos = None # position *after* ULEN+CRC placeholders
|
|
3398
|
-
|
|
3399
|
-
# Read state
|
|
3400
|
-
self._spool = None
|
|
3401
|
-
self._text_reader = None
|
|
3402
|
-
self._position = 0
|
|
3403
|
-
self.closed = False
|
|
3404
|
-
|
|
3405
|
-
internal_mode = mode.replace('t', 'b')
|
|
3406
|
-
if self.file is None:
|
|
3407
|
-
if 'x' in internal_mode and os.path.exists(file_path):
|
|
3408
|
-
raise IOError("File exists: '{}'".format(file_path))
|
|
3409
|
-
self.file = open(file_path, internal_mode)
|
|
3410
|
-
else:
|
|
3411
|
-
if 'r' in internal_mode and not hasattr(self.file, 'read'):
|
|
3412
|
-
raise ValueError("fileobj must support read() in read mode")
|
|
3413
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')) and not hasattr(self.file, 'write'):
|
|
3414
|
-
raise ValueError("fileobj must support write() in write/append mode")
|
|
3415
|
-
|
|
3416
|
-
self._fp = self.file
|
|
3417
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')):
|
|
3418
|
-
# Start a new member at EOF for append
|
|
3419
|
-
if 'a' in internal_mode:
|
|
3420
|
-
try:
|
|
3421
|
-
self.file.seek(0, os.SEEK_END)
|
|
3422
|
-
except Exception:
|
|
3423
|
-
pass
|
|
3424
|
-
# Defer writing header until first write so empty files don’t get empty members
|
|
3425
|
-
elif 'r' in internal_mode:
|
|
3426
|
-
self._load_all_members_spooled()
|
|
3427
|
-
else:
|
|
3428
|
-
raise ValueError("Unsupported mode: {}".format(mode))
|
|
3429
|
-
|
|
3430
|
-
# ---------- helpers ----------
|
|
3431
|
-
@property
|
|
3432
|
-
def name(self):
|
|
3433
|
-
return self.file_path
|
|
3434
|
-
|
|
3435
|
-
def readable(self):
|
|
3436
|
-
return 'r' in self.mode
|
|
3437
|
-
|
|
3438
|
-
def writable(self):
|
|
3439
|
-
return any(ch in self.mode for ch in ('w', 'a', 'x'))
|
|
3440
|
-
|
|
3441
|
-
def seekable(self):
|
|
3442
|
-
return True if self._spool is not None else bool(getattr(self.file, 'seek', None))
|
|
3443
|
-
|
|
3444
|
-
def _normalize_newlines_for_write(self, s):
|
|
3445
|
-
nl = self.newline if self.newline is not None else "\n"
|
|
3446
|
-
return s.replace("\r\n", "\n").replace("\r", "\n").replace("\n", nl)
|
|
3447
|
-
|
|
3448
|
-
def _reader(self):
|
|
3449
|
-
return self._text_reader if self._text_mode else self._spool
|
|
3450
|
-
|
|
3451
|
-
# ---------- Write path ----------
|
|
3452
|
-
def _ensure_member_header(self):
|
|
3453
|
-
if self._open_member:
|
|
3454
|
-
return
|
|
3455
|
-
flags = self.FLAG_HAS_UHDR if self._write_header else 0
|
|
3456
|
-
self.file.write(self.MAGIC)
|
|
3457
|
-
self.file.write(struct.pack(">B", flags))
|
|
3458
|
-
if self._write_header:
|
|
3459
|
-
# placeholders for ULEN+CRC; we’ll backfill on finalize
|
|
3460
|
-
self.file.write(struct.pack(">Q", 0))
|
|
3461
|
-
self.file.write(struct.pack(">I", 0))
|
|
3462
|
-
# position *after* ULEN+CRC placeholders (or after FLAGS if no header)
|
|
3463
|
-
self._member_header_pos = self.file.tell()
|
|
3464
|
-
self._open_member = True
|
|
3465
|
-
# reset member stats
|
|
3466
|
-
self._crc = 0
|
|
3467
|
-
self._ulen = 0
|
|
3468
|
-
|
|
3469
|
-
def write(self, data):
|
|
3470
|
-
if 'r' in self.mode:
|
|
3471
|
-
raise IOError("File not open for writing")
|
|
3472
|
-
|
|
3473
|
-
if self._text_mode:
|
|
3474
|
-
enc = self.encoding or 'UTF-8'
|
|
3475
|
-
errs = self.errors or 'strict'
|
|
3476
|
-
if not isinstance(data, text_type):
|
|
3477
|
-
raise TypeError("write() expects text (unicode/str) in text mode")
|
|
3478
|
-
data = self._normalize_newlines_for_write(data).encode(enc, errs)
|
|
3479
|
-
else:
|
|
3480
|
-
if not isinstance(data, binary_types):
|
|
3481
|
-
raise TypeError("write() expects bytes-like in binary mode")
|
|
3482
|
-
|
|
3483
|
-
# Normalize Py3 memoryview / Py2 bytearray
|
|
3484
|
-
if (not PY2) and isinstance(data, memoryview):
|
|
3485
|
-
data = data.tobytes()
|
|
3486
|
-
elif PY2 and isinstance(data, bytearray):
|
|
3487
|
-
data = bytes(data)
|
|
3488
|
-
|
|
3489
|
-
if not data:
|
|
3490
|
-
return 0
|
|
3491
|
-
|
|
3492
|
-
# Begin member and write header on first write
|
|
3493
|
-
self._ensure_member_header()
|
|
3494
|
-
|
|
3495
|
-
# Update integrity stats
|
|
3496
|
-
self._crc = _crc32u(data, self._crc)
|
|
3497
|
-
self._ulen += len(data)
|
|
3498
|
-
|
|
3499
|
-
# Stream in RAW_CHUNK-sized pieces. Each piece becomes one compressed chunk record.
|
|
3500
|
-
mv = memoryview(data)
|
|
3501
|
-
# clamp level to {1, 9}
|
|
3502
|
-
lvl = 9 if self.level >= 9 else 1
|
|
3503
|
-
for off in range(0, len(data), self.RAW_CHUNK):
|
|
3504
|
-
raw = mv[off:off + self.RAW_CHUNK].tobytes()
|
|
3505
|
-
c = lzo.compress(raw, lvl)
|
|
3506
|
-
self.file.write(struct.pack(">I", len(c)))
|
|
3507
|
-
self.file.write(c)
|
|
3508
|
-
|
|
3509
|
-
return len(data)
|
|
3510
|
-
|
|
3511
|
-
def _flush_member_only(self):
|
|
3512
|
-
"""Finalize the current member: write terminator and backfill header."""
|
|
3513
|
-
if not self._open_member:
|
|
3514
|
-
return
|
|
3515
|
-
# write zero-length chunk terminator
|
|
3516
|
-
self.file.write(struct.pack(">I", 0))
|
|
3517
|
-
if self._write_header:
|
|
3518
|
-
# ULEN is at (_member_header_pos - 12), CRC at (_member_header_pos - 4)
|
|
3519
|
-
ulen_pos = self._member_header_pos - 12
|
|
3520
|
-
crc_pos = self._member_header_pos - 4
|
|
3521
|
-
cur = self.file.tell()
|
|
3522
|
-
# backfill ULEN
|
|
3523
|
-
self.file.seek(ulen_pos, os.SEEK_SET)
|
|
3524
|
-
self.file.write(struct.pack(">Q", self._ulen))
|
|
3525
|
-
# backfill CRC32
|
|
3526
|
-
self.file.seek(crc_pos, os.SEEK_SET)
|
|
3527
|
-
self.file.write(struct.pack(">I", self._crc))
|
|
3528
|
-
# restore position
|
|
3529
|
-
self.file.seek(cur, os.SEEK_SET)
|
|
3530
|
-
# reset for potential new member
|
|
3531
|
-
self._open_member = False
|
|
3532
|
-
self._crc = 0
|
|
3533
|
-
self._ulen = 0
|
|
3534
|
-
self._member_header_pos = None
|
|
3535
|
-
|
|
3536
|
-
def flush(self):
|
|
3537
|
-
if self.closed:
|
|
3538
|
-
return
|
|
3539
|
-
# finalize any open member
|
|
3540
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3541
|
-
self._flush_member_only()
|
|
3542
|
-
if hasattr(self.file, 'flush'):
|
|
3543
|
-
self.file.flush()
|
|
3544
|
-
|
|
3545
|
-
def close(self):
|
|
3546
|
-
if self.closed:
|
|
3547
|
-
return
|
|
3548
|
-
try:
|
|
3549
|
-
# Ensure a clean member terminator & header backfill if needed
|
|
3550
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3551
|
-
self._flush_member_only()
|
|
3552
|
-
if hasattr(self.file, 'flush'):
|
|
3553
|
-
try:
|
|
3554
|
-
self.file.flush()
|
|
3555
|
-
except Exception:
|
|
3556
|
-
pass
|
|
3557
|
-
finally:
|
|
3558
|
-
if self.file_path and self.file is not None:
|
|
3559
|
-
try:
|
|
3560
|
-
self.file.close()
|
|
3561
|
-
except Exception:
|
|
3562
|
-
pass
|
|
3563
|
-
# tear down read handles
|
|
3564
|
-
try:
|
|
3565
|
-
if self._text_reader is not None:
|
|
3566
|
-
self._text_reader.detach()
|
|
3567
|
-
except Exception:
|
|
3568
|
-
pass
|
|
3569
|
-
try:
|
|
3570
|
-
if self._spool is not None:
|
|
3571
|
-
self._spool.close()
|
|
3572
|
-
except Exception:
|
|
3573
|
-
pass
|
|
3574
|
-
self.closed = True
|
|
3575
|
-
|
|
3576
|
-
# ---------- Read path (spooled, multi-member, tolerant scan) ----------
|
|
3577
|
-
def _load_all_members_spooled(self):
|
|
3578
|
-
# Seek to start if possible
|
|
3579
|
-
try:
|
|
3580
|
-
self.file.seek(0)
|
|
3581
|
-
except Exception:
|
|
3582
|
-
pass
|
|
3583
|
-
|
|
3584
|
-
self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
|
|
3585
|
-
|
|
3586
|
-
def read_exact(n, abs_off_ref):
|
|
3587
|
-
"""Read exactly n bytes, updating abs_off_ref[0]."""
|
|
3588
|
-
b = b""
|
|
3589
|
-
while len(b) < n:
|
|
3590
|
-
part = self.file.read(n - len(b))
|
|
3591
|
-
if not part:
|
|
3592
|
-
break
|
|
3593
|
-
b += part
|
|
3594
|
-
abs_off_ref[0] += len(part)
|
|
3595
|
-
return b
|
|
3596
|
-
|
|
3597
|
-
CHUNK = 1 << 20
|
|
3598
|
-
abs_off = [0] # track absolute file offset
|
|
3599
|
-
scanned = 0
|
|
3600
|
-
|
|
3601
|
-
while True:
|
|
3602
|
-
# Locate MAGIC (support tolerant scan across chunk boundaries)
|
|
3603
|
-
head = read_exact(len(self.MAGIC), abs_off)
|
|
3604
|
-
if not head:
|
|
3605
|
-
break # EOF
|
|
3606
|
-
if head != self.MAGIC:
|
|
3607
|
-
# Tolerant scan: slide-by-one until found or limit exceeded
|
|
3608
|
-
buf = head
|
|
3609
|
-
while True:
|
|
3610
|
-
if self.tolerant_read and scanned < self.scan_bytes:
|
|
3611
|
-
nxt = read_exact(1, abs_off)
|
|
3612
|
-
if not nxt:
|
|
3613
|
-
# EOF without finding magic
|
|
3614
|
-
raise ValueError("Invalid LZO container: magic not found before EOF")
|
|
3615
|
-
buf = buf[1:] + nxt
|
|
3616
|
-
scanned += 1
|
|
3617
|
-
if buf == self.MAGIC:
|
|
3618
|
-
break
|
|
3619
|
-
continue
|
|
3620
|
-
raise ValueError("Invalid LZO container magic near offset {}".format(abs_off[0] - len(buf)))
|
|
3621
|
-
# found MAGIC; proceed
|
|
3622
|
-
|
|
3623
|
-
# FLAGS
|
|
3624
|
-
f_b = read_exact(1, abs_off)
|
|
3625
|
-
if len(f_b) != 1:
|
|
3626
|
-
raise ValueError("Truncated header (flags) at offset {}".format(abs_off[0]))
|
|
3627
|
-
flags = ord(f_b) if PY2 else f_b[0]
|
|
3628
|
-
|
|
3629
|
-
# Optional ULEN/CRC
|
|
3630
|
-
ulen = None
|
|
3631
|
-
expect_crc = None
|
|
3632
|
-
if flags & self.FLAG_HAS_UHDR:
|
|
3633
|
-
ulen_b = read_exact(8, abs_off)
|
|
3634
|
-
crc_b = read_exact(4, abs_off)
|
|
3635
|
-
if len(ulen_b) != 8 or len(crc_b) != 4:
|
|
3636
|
-
raise ValueError("Truncated ULEN/CRC header at offset {}".format(abs_off[0]))
|
|
3637
|
-
ulen = struct.unpack(">Q", ulen_b)[0]
|
|
3638
|
-
expect_crc = struct.unpack(">I", crc_b)[0]
|
|
3639
|
-
|
|
3640
|
-
# Chunk loop
|
|
3641
|
-
m_crc = 0
|
|
3642
|
-
m_len = 0
|
|
3643
|
-
while True:
|
|
3644
|
-
sz_b = read_exact(4, abs_off)
|
|
3645
|
-
if len(sz_b) != 4:
|
|
3646
|
-
raise ValueError("Truncated chunk size at offset {}".format(abs_off[0]))
|
|
3647
|
-
csz = struct.unpack(">I", sz_b)[0]
|
|
3648
|
-
if csz == 0:
|
|
3649
|
-
break # end of member
|
|
3650
|
-
cdata = read_exact(csz, abs_off)
|
|
3651
|
-
if len(cdata) != csz:
|
|
3652
|
-
raise ValueError("Truncated chunk payload at offset {}".format(abs_off[0]))
|
|
3653
|
-
try:
|
|
3654
|
-
raw = lzo.decompress(cdata)
|
|
3655
|
-
except Exception as e:
|
|
3656
|
-
raise ValueError("LZO decompression error at offset {}: {}".format(abs_off[0], e))
|
|
3657
|
-
self._spool.write(raw)
|
|
3658
|
-
m_len += len(raw)
|
|
3659
|
-
m_crc = _crc32u(raw, m_crc)
|
|
3660
|
-
|
|
3661
|
-
# Validate member integrity if header present
|
|
3662
|
-
if ulen is not None and m_len != ulen:
|
|
3663
|
-
raise ValueError("Member length mismatch ({} != {})".format(m_len, ulen))
|
|
3664
|
-
if expect_crc is not None and m_crc != expect_crc:
|
|
3665
|
-
raise ValueError("Member CRC32 mismatch (got 0x{:08x}, want 0x{:08x})"
|
|
3666
|
-
.format(m_crc, expect_crc))
|
|
3667
|
-
|
|
3668
|
-
# Prepare read handles
|
|
3669
|
-
try:
|
|
3670
|
-
self._spool.seek(0)
|
|
3671
|
-
except Exception:
|
|
3672
|
-
pass
|
|
3673
|
-
|
|
3674
|
-
if self._text_mode:
|
|
3675
|
-
enc = self.encoding or 'UTF-8'
|
|
3676
|
-
errs = self.errors or 'strict'
|
|
3677
|
-
# newline=None => universal newline translation; exact string if provided
|
|
3678
|
-
self._text_reader = io.TextIOWrapper(self._spool, encoding=enc, errors=errs, newline=self.newline)
|
|
3679
|
-
try:
|
|
3680
|
-
self._text_reader.seek(0)
|
|
3681
|
-
except Exception:
|
|
3682
|
-
pass
|
|
3683
|
-
|
|
3684
|
-
self._position = 0
|
|
3685
|
-
|
|
3686
|
-
# ---------- Buffered read API (delegates to spool/text wrapper) ----------
|
|
3687
|
-
def read(self, size=-1):
|
|
3688
|
-
if self.closed:
|
|
3689
|
-
raise ValueError("I/O operation on closed file")
|
|
3690
|
-
if 'r' not in self.mode:
|
|
3691
|
-
raise IOError("File not open for reading")
|
|
3692
|
-
r = self._reader()
|
|
3693
|
-
if r is None:
|
|
3694
|
-
raise IOError("Reader not initialized")
|
|
3695
|
-
out = r.read() if (size is None or size < 0) else r.read(int(size))
|
|
3696
|
-
try:
|
|
3697
|
-
self._position = r.tell()
|
|
3698
|
-
except Exception:
|
|
3699
|
-
pass
|
|
3700
|
-
return out
|
|
3701
|
-
|
|
3702
|
-
def readline(self, size=-1):
|
|
3703
|
-
if self.closed:
|
|
3704
|
-
raise ValueError("I/O operation on closed file")
|
|
3705
|
-
if 'r' not in self.mode:
|
|
3706
|
-
raise IOError("File not open for reading")
|
|
3707
|
-
r = self._reader()
|
|
3708
|
-
if r is None:
|
|
3709
|
-
raise IOError("Reader not initialized")
|
|
3710
|
-
out = r.readline() if (size is None or size < 0) else r.readline(int(size))
|
|
3711
|
-
try:
|
|
3712
|
-
self._position = r.tell()
|
|
3713
|
-
except Exception:
|
|
3714
|
-
pass
|
|
3715
|
-
if not self._text_mode and out is None:
|
|
3716
|
-
return b""
|
|
3717
|
-
if self._text_mode and out is None:
|
|
3718
|
-
return text_type("")
|
|
3719
|
-
return out
|
|
3720
|
-
|
|
3721
|
-
def __iter__(self):
|
|
3722
|
-
return self
|
|
3723
|
-
|
|
3724
|
-
def __next__(self):
|
|
3725
|
-
line = self.readline()
|
|
3726
|
-
if (self._text_mode and line == "") or (not self._text_mode and line == b""):
|
|
3727
|
-
raise StopIteration
|
|
3728
|
-
return line
|
|
3729
|
-
|
|
3730
|
-
if PY2:
|
|
3731
|
-
next = __next__
|
|
3732
|
-
|
|
3733
|
-
def seek(self, offset, whence=0):
|
|
3734
|
-
if self.closed:
|
|
3735
|
-
raise ValueError("I/O operation on closed file")
|
|
3736
|
-
if 'r' not in self.mode:
|
|
3737
|
-
raise IOError("File not open for reading")
|
|
3738
|
-
r = self._reader()
|
|
3739
|
-
if r is None:
|
|
3740
|
-
raise IOError("Reader not initialized")
|
|
3741
|
-
newpos = r.seek(int(offset), int(whence))
|
|
3742
|
-
self._position = newpos
|
|
3743
|
-
return newpos
|
|
3744
|
-
|
|
3745
|
-
def tell(self):
|
|
3746
|
-
if self._reader() is not None:
|
|
3747
|
-
try:
|
|
3748
|
-
self._position = self._reader().tell()
|
|
3749
|
-
except Exception:
|
|
3750
|
-
pass
|
|
3751
|
-
return self._position
|
|
3752
|
-
|
|
3753
|
-
# ---------- Misc ----------
|
|
3754
|
-
def fileno(self):
|
|
3755
|
-
if hasattr(self.file, 'fileno'):
|
|
3756
|
-
return self.file.fileno()
|
|
3757
|
-
raise OSError("Underlying file object does not support fileno()")
|
|
3758
|
-
|
|
3759
|
-
def isatty(self):
|
|
3760
|
-
return bool(getattr(self.file, 'isatty', lambda: False)())
|
|
3761
|
-
|
|
3762
|
-
def truncate(self, size=None):
|
|
3763
|
-
# Prevent corruption of compressed streams
|
|
3764
|
-
raise OSError("truncate() is not supported for compressed streams")
|
|
3765
|
-
|
|
3766
|
-
# ---------- Convenience constructors ----------
|
|
3767
|
-
@classmethod
|
|
3768
|
-
def open(cls, path, mode='rb', **kw):
|
|
3769
|
-
"""
|
|
3770
|
-
Mirror built-in open() but for LzopFile.
|
|
3771
|
-
Example:
|
|
3772
|
-
with LzopFile.open("data.lzo", "rt", encoding="utf-8") as f:
|
|
3773
|
-
print(f.readline())
|
|
3774
|
-
"""
|
|
3775
|
-
return cls(file_path=path, mode=mode, **kw)
|
|
3776
|
-
|
|
3777
|
-
@classmethod
|
|
3778
|
-
def from_fileobj(cls, fileobj, mode='rb', **kw):
|
|
3779
|
-
"""
|
|
3780
|
-
Wrap an existing file-like object (caller retains ownership).
|
|
3781
|
-
"""
|
|
3782
|
-
return cls(fileobj=fileobj, mode=mode, **kw)
|
|
3783
|
-
|
|
3784
|
-
@classmethod
|
|
3785
|
-
def from_bytes(cls, data, mode='rb', **kw):
|
|
3786
|
-
"""
|
|
3787
|
-
Read from an in-memory bytes buffer.
|
|
3788
|
-
Example:
|
|
3789
|
-
f = LzopFile.from_bytes(blob, mode='rt', encoding='utf-8', tolerant_read=True)
|
|
3790
|
-
text = f.read()
|
|
3791
|
-
"""
|
|
3792
|
-
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
3793
|
-
raise TypeError("from_bytes() expects a bytes-like object")
|
|
3794
|
-
bio = io.BytesIO(bytes(data) if not isinstance(data, bytes) else data)
|
|
3795
|
-
return cls(fileobj=bio, mode=mode, **kw)
|
|
3796
|
-
|
|
3797
|
-
# compatibility aliases for unwrapping utilities
|
|
3798
|
-
@property
|
|
3799
|
-
def fileobj(self):
|
|
3800
|
-
return self.file
|
|
3801
|
-
|
|
3802
|
-
@property
|
|
3803
|
-
def myfileobj(self):
|
|
3804
|
-
return self.file
|
|
3805
|
-
|
|
3806
|
-
# ---------- Top-level helpers ----------
|
|
3807
|
-
def lzop_compress_bytes(payload, level=9, text=False, **kw):
|
|
3808
|
-
"""
|
|
3809
|
-
Compress 'payload' into a single LZO member (our custom container) and return bytes.
|
|
3810
|
-
- text=True: 'payload' is text; encoding/newline/errors handled via LzopFile('wt')
|
|
3811
|
-
- text=False: 'payload' is bytes-like; written via LzopFile('wb')
|
|
3812
|
-
Kwargs forwarded: write_header (default True), newline/encoding/errors, etc.
|
|
3813
|
-
"""
|
|
3814
|
-
bio = io.BytesIO()
|
|
3815
|
-
mode = 'wt' if text else 'wb'
|
|
3816
|
-
f = LzopFile(fileobj=bio, mode=mode, level=level, **kw)
|
|
3817
|
-
try:
|
|
3818
|
-
f.write(payload)
|
|
3819
|
-
f.flush() # finalize member (writes terminator + backfills header)
|
|
3820
|
-
finally:
|
|
3821
|
-
f.close()
|
|
3822
|
-
return bio.getvalue()
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
def lzop_decompress_bytes(blob, mode='rb', tolerant_read=False, scan_bytes=(64 << 10),
|
|
3826
|
-
spool_threshold=(8 << 20), **kw):
|
|
3827
|
-
"""
|
|
3828
|
-
Decompress bytes produced by this custom container.
|
|
3829
|
-
- mode='rb' -> returns bytes; mode='rt' -> returns text (set encoding/errors/newline in kw)
|
|
3830
|
-
- tolerant_read/scan_bytes/spool_threshold forwarded to LzopFile
|
|
3831
|
-
"""
|
|
3832
|
-
if not isinstance(blob, (bytes, bytearray, memoryview)):
|
|
3833
|
-
raise TypeError("lzop_decompress_bytes() expects a bytes-like object")
|
|
3834
|
-
f = LzopFile.from_bytes(blob, mode=mode, tolerant_read=tolerant_read,
|
|
3835
|
-
scan_bytes=scan_bytes, spool_threshold=spool_threshold, **kw)
|
|
3836
|
-
try:
|
|
3837
|
-
return f.read()
|
|
3838
|
-
finally:
|
|
3839
|
-
f.close()
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
3416
|
def TarFileCheck(infile):
|
|
3843
3417
|
try:
|
|
3844
3418
|
if is_tarfile(infile):
|
|
@@ -4307,19 +3881,62 @@ def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspe
|
|
|
4307
3881
|
return hmac.compare_digest(want, calc)
|
|
4308
3882
|
|
|
4309
3883
|
|
|
4310
|
-
# =========================
|
|
4311
|
-
# Incremental CRC context
|
|
4312
|
-
# =========================
|
|
4313
|
-
CRCSpec = namedtuple("CRCSpec", "width poly init xorout refin refout")
|
|
3884
|
+
# =========================
|
|
3885
|
+
# Incremental CRC context
|
|
3886
|
+
# =========================
|
|
3887
|
+
CRCSpec = namedtuple("CRCSpec", "width poly init xorout refin refout")
|
|
3888
|
+
|
|
3889
|
+
_CRC_SPECS = {
|
|
3890
|
+
"crc16_ansi": CRCSpec(16, 0x8005, 0xFFFF, 0x0000, True, True),
|
|
3891
|
+
"crc16_ccitt": CRCSpec(16, 0x1021, 0xFFFF, 0x0000, False, False),
|
|
3892
|
+
"crc16_x25": CRCSpec(16, 0x1021, 0xFFFF, 0xFFFF, True, True),
|
|
3893
|
+
"crc16_kermit":CRCSpec(16, 0x1021, 0x0000, 0x0000, True, True),
|
|
3894
|
+
"crc64_ecma": CRCSpec(64, 0x42F0E1EBA9EA3693, 0x0000000000000000, 0x0000000000000000, False, False),
|
|
3895
|
+
"crc64_iso": CRCSpec(64, 0x000000000000001B, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, True, True),
|
|
3896
|
+
}
|
|
3897
|
+
|
|
3898
|
+
# --- helpers --------------------------------------------------------------
|
|
3899
|
+
|
|
3900
|
+
try:
|
|
3901
|
+
# Python 2 may not have algorithms_available
|
|
3902
|
+
_ALGORITHMS_AVAILABLE = set(hashlib.algorithms_available)
|
|
3903
|
+
except AttributeError:
|
|
3904
|
+
_ALGORITHMS_AVAILABLE = set(getattr(hashlib, "algorithms", []))
|
|
3905
|
+
|
|
3906
|
+
|
|
3907
|
+
def _coerce_bytes(data):
|
|
3908
|
+
"""Return `data` as a bytes object (Py2 / Py3)."""
|
|
3909
|
+
if isinstance(data, memoryview):
|
|
3910
|
+
# Py3 has .tobytes(), Py2 falls back to bytes()
|
|
3911
|
+
try:
|
|
3912
|
+
return data.tobytes()
|
|
3913
|
+
except AttributeError:
|
|
3914
|
+
return bytes(data)
|
|
3915
|
+
|
|
3916
|
+
if isinstance(data, bytearray):
|
|
3917
|
+
return bytes(data)
|
|
3918
|
+
|
|
3919
|
+
if not isinstance(data, bytes):
|
|
3920
|
+
# E.g. list of ints, unicode, etc.
|
|
3921
|
+
return bytes(bytearray(data))
|
|
3922
|
+
|
|
3923
|
+
return data
|
|
3924
|
+
|
|
3925
|
+
|
|
3926
|
+
def _bytes_to_int(b):
|
|
3927
|
+
"""Big-endian bytes -> int, Py2/3 safe."""
|
|
3928
|
+
if not isinstance(b, (bytes, bytearray)):
|
|
3929
|
+
b = _coerce_bytes(b)
|
|
3930
|
+
|
|
3931
|
+
value = 0
|
|
3932
|
+
for ch in b:
|
|
3933
|
+
if not isinstance(ch, int): # Py2: ch is a 1-char string
|
|
3934
|
+
ch = ord(ch)
|
|
3935
|
+
value = (value << 8) | ch
|
|
3936
|
+
return value
|
|
3937
|
+
|
|
4314
3938
|
|
|
4315
|
-
|
|
4316
|
-
"crc16_ansi": CRCSpec(16, 0x8005, 0xFFFF, 0x0000, True, True),
|
|
4317
|
-
"crc16_ccitt": CRCSpec(16, 0x1021, 0xFFFF, 0x0000, False, False),
|
|
4318
|
-
"crc16_x25": CRCSpec(16, 0x1021, 0xFFFF, 0xFFFF, True, True),
|
|
4319
|
-
"crc16_kermit":CRCSpec(16, 0x1021, 0x0000, 0x0000, True, True),
|
|
4320
|
-
"crc64_ecma": CRCSpec(64, 0x42F0E1EBA9EA3693, 0x0000000000000000, 0x0000000000000000, False, False),
|
|
4321
|
-
"crc64_iso": CRCSpec(64, 0x000000000000001B, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, True, True),
|
|
4322
|
-
}
|
|
3939
|
+
# --- your existing CRCContext (unchanged) ---------------------------------
|
|
4323
3940
|
|
|
4324
3941
|
class CRCContext(object):
|
|
4325
3942
|
__slots__ = ("spec", "table", "mask", "shift", "crc")
|
|
@@ -4334,10 +3951,11 @@ class CRCContext(object):
|
|
|
4334
3951
|
def update(self, data):
|
|
4335
3952
|
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
4336
3953
|
data = bytes(bytearray(data))
|
|
3954
|
+
buf = _mv_tobytes(memoryview(data))
|
|
4337
3955
|
if self.spec.refin:
|
|
4338
3956
|
c = self.crc
|
|
4339
3957
|
tbl = self.table
|
|
4340
|
-
for b in
|
|
3958
|
+
for b in buf:
|
|
4341
3959
|
if not isinstance(b, int): # Py2
|
|
4342
3960
|
b = ord(b)
|
|
4343
3961
|
c = tbl[(c ^ b) & 0xFF] ^ (c >> 8)
|
|
@@ -4347,7 +3965,7 @@ class CRCContext(object):
|
|
|
4347
3965
|
tbl = self.table
|
|
4348
3966
|
sh = self.shift
|
|
4349
3967
|
msk = self.mask
|
|
4350
|
-
for b in
|
|
3968
|
+
for b in buf:
|
|
4351
3969
|
if not isinstance(b, int):
|
|
4352
3970
|
b = ord(b)
|
|
4353
3971
|
c = tbl[((c >> sh) ^ b) & 0xFF] ^ ((c << 8) & msk)
|
|
@@ -4364,6 +3982,82 @@ class CRCContext(object):
|
|
|
4364
3982
|
width_hex = (self.spec.width + 3) // 4
|
|
4365
3983
|
return format(self.digest_int(), "0{}x".format(width_hex)).lower()
|
|
4366
3984
|
|
|
3985
|
+
|
|
3986
|
+
# --- hashlib-backed implementation ---------------------------------------
|
|
3987
|
+
|
|
3988
|
+
class _HashlibCRCWrapper(object):
|
|
3989
|
+
"""
|
|
3990
|
+
Wrap a hashlib object to present the same interface as CRCContext
|
|
3991
|
+
(update, digest_int, hexdigest).
|
|
3992
|
+
|
|
3993
|
+
Assumes the hashlib algorithm already implements the exact CRC
|
|
3994
|
+
specification (refin/refout/xorout/etc.).
|
|
3995
|
+
"""
|
|
3996
|
+
__slots__ = ("_h", "spec", "mask", "width_hex")
|
|
3997
|
+
|
|
3998
|
+
def __init__(self, algo_name, spec):
|
|
3999
|
+
self._h = hashlib.new(algo_name)
|
|
4000
|
+
self.spec = spec
|
|
4001
|
+
self.mask = (1 << spec.width) - 1
|
|
4002
|
+
self.width_hex = (spec.width + 3) // 4
|
|
4003
|
+
|
|
4004
|
+
def update(self, data):
|
|
4005
|
+
self._h.update(_coerce_bytes(data))
|
|
4006
|
+
return self
|
|
4007
|
+
|
|
4008
|
+
def digest_int(self):
|
|
4009
|
+
# Convert final digest bytes to an integer and mask to width
|
|
4010
|
+
value = _bytes_to_int(self._h.digest())
|
|
4011
|
+
return value & self.mask
|
|
4012
|
+
|
|
4013
|
+
def hexdigest(self):
|
|
4014
|
+
h = self._h.hexdigest().lower()
|
|
4015
|
+
# Normalize to the same number of hex digits as CRCContext
|
|
4016
|
+
if len(h) < self.width_hex:
|
|
4017
|
+
h = ("0" * (self.width_hex - len(h))) + h
|
|
4018
|
+
elif len(h) > self.width_hex:
|
|
4019
|
+
h = h[-self.width_hex:]
|
|
4020
|
+
return h
|
|
4021
|
+
|
|
4022
|
+
|
|
4023
|
+
# --- public class: choose hashlib or fallback -----------------------------
|
|
4024
|
+
|
|
4025
|
+
class CRC(object):
|
|
4026
|
+
"""
|
|
4027
|
+
CRC wrapper that uses hashlib if available, otherwise falls back to
|
|
4028
|
+
the pure-Python CRCContext.
|
|
4029
|
+
|
|
4030
|
+
spec.hashlib_name (preferred) or spec.name is used as the hashlib
|
|
4031
|
+
algorithm name, e.g. 'crc32', 'crc32c', etc.
|
|
4032
|
+
"""
|
|
4033
|
+
|
|
4034
|
+
__slots__ = ("spec", "_impl")
|
|
4035
|
+
|
|
4036
|
+
def __init__(self, spec):
|
|
4037
|
+
self.spec = spec
|
|
4038
|
+
|
|
4039
|
+
algo_name = getattr(spec, "hashlib_name", None) or getattr(spec, "name", None)
|
|
4040
|
+
impl = None
|
|
4041
|
+
|
|
4042
|
+
if algo_name and algo_name in _ALGORITHMS_AVAILABLE:
|
|
4043
|
+
# Use hashlib-backed implementation
|
|
4044
|
+
impl = _HashlibCRCWrapper(algo_name, spec)
|
|
4045
|
+
else:
|
|
4046
|
+
# Fallback to your pure-Python implementation
|
|
4047
|
+
impl = CRCContext(spec)
|
|
4048
|
+
|
|
4049
|
+
self._impl = impl
|
|
4050
|
+
|
|
4051
|
+
def update(self, data):
|
|
4052
|
+
self._impl.update(data)
|
|
4053
|
+
return self
|
|
4054
|
+
|
|
4055
|
+
def digest_int(self):
|
|
4056
|
+
return self._impl.digest_int()
|
|
4057
|
+
|
|
4058
|
+
def hexdigest(self):
|
|
4059
|
+
return self._impl.hexdigest()
|
|
4060
|
+
|
|
4367
4061
|
def crc_context_from_name(name_norm):
|
|
4368
4062
|
spec = _CRC_SPECS.get(name_norm)
|
|
4369
4063
|
if spec is None:
|
|
@@ -5231,7 +4925,7 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
|
|
|
5231
4925
|
cfcontents.close()
|
|
5232
4926
|
fcontents.seek(0, 0)
|
|
5233
4927
|
fccs = GetFileChecksum(
|
|
5234
|
-
fcontents
|
|
4928
|
+
fcontents, HeaderOut[-3].lower(), False, formatspecs)
|
|
5235
4929
|
fcontentend = fp.tell()
|
|
5236
4930
|
if(re.findall("^\\+([0-9]+)", fseeknextfile)):
|
|
5237
4931
|
fseeknextasnum = int(fseeknextfile.replace("+", ""))
|
|
@@ -6470,7 +6164,7 @@ def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], jsondata={}, fi
|
|
|
6470
6164
|
return fp
|
|
6471
6165
|
|
|
6472
6166
|
|
|
6473
|
-
def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False,
|
|
6167
|
+
def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
6474
6168
|
if(not hasattr(fp, "write")):
|
|
6475
6169
|
return False
|
|
6476
6170
|
advancedlist = formatspecs['use_advanced_list']
|
|
@@ -6520,6 +6214,16 @@ def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], ext
|
|
|
6520
6214
|
numfiles = int(len(GetDirList))
|
|
6521
6215
|
fnumfiles = format(numfiles, 'x').lower()
|
|
6522
6216
|
AppendFileHeader(fp, numfiles, "UTF-8", [], checksumtype[0], formatspecs)
|
|
6217
|
+
try:
|
|
6218
|
+
fp.flush()
|
|
6219
|
+
if(hasattr(os, "sync")):
|
|
6220
|
+
os.fsync(fp.fileno())
|
|
6221
|
+
except io.UnsupportedOperation:
|
|
6222
|
+
pass
|
|
6223
|
+
except AttributeError:
|
|
6224
|
+
pass
|
|
6225
|
+
except OSError:
|
|
6226
|
+
pass
|
|
6523
6227
|
FullSizeFilesAlt = 0
|
|
6524
6228
|
for curfname in GetDirList:
|
|
6525
6229
|
fencoding = "UTF-8"
|
|
@@ -6771,6 +6475,16 @@ def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], ext
|
|
|
6771
6475
|
fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev, fdev_minor, fdev_major, "+"+str(len(formatspecs['format_delimiter']))]
|
|
6772
6476
|
AppendFileHeaderWithContent(
|
|
6773
6477
|
fp, tmpoutlist, extradata, jsondata, fcontents.read(), [checksumtype[1], checksumtype[2], checksumtype[3]], formatspecs)
|
|
6478
|
+
try:
|
|
6479
|
+
fp.flush()
|
|
6480
|
+
if(hasattr(os, "sync")):
|
|
6481
|
+
os.fsync(fp.fileno())
|
|
6482
|
+
except io.UnsupportedOperation:
|
|
6483
|
+
pass
|
|
6484
|
+
except AttributeError:
|
|
6485
|
+
pass
|
|
6486
|
+
except OSError:
|
|
6487
|
+
pass
|
|
6774
6488
|
return fp
|
|
6775
6489
|
|
|
6776
6490
|
def AppendFilesWithContentFromTarFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
@@ -6779,8 +6493,6 @@ def AppendFilesWithContentFromTarFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
6779
6493
|
if(verbose):
|
|
6780
6494
|
logging.basicConfig(format="%(message)s",
|
|
6781
6495
|
stream=sys.stdout, level=logging.DEBUG)
|
|
6782
|
-
formver = formatspecs['format_ver']
|
|
6783
|
-
fileheaderver = str(int(formver.replace(".", "")))
|
|
6784
6496
|
curinode = 0
|
|
6785
6497
|
curfid = 0
|
|
6786
6498
|
inodelist = []
|
|
@@ -6848,6 +6560,16 @@ def AppendFilesWithContentFromTarFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
6848
6560
|
return False
|
|
6849
6561
|
numfiles = int(len(tarfp.getmembers()))
|
|
6850
6562
|
AppendFileHeader(fp, numfiles, "UTF-8", [], checksumtype[0], formatspecs)
|
|
6563
|
+
try:
|
|
6564
|
+
fp.flush()
|
|
6565
|
+
if(hasattr(os, "sync")):
|
|
6566
|
+
os.fsync(fp.fileno())
|
|
6567
|
+
except io.UnsupportedOperation:
|
|
6568
|
+
pass
|
|
6569
|
+
except AttributeError:
|
|
6570
|
+
pass
|
|
6571
|
+
except OSError:
|
|
6572
|
+
pass
|
|
6851
6573
|
for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
|
|
6852
6574
|
fencoding = "UTF-8"
|
|
6853
6575
|
if(re.findall("^[.|/]", member.name)):
|
|
@@ -6986,6 +6708,16 @@ def AppendFilesWithContentFromTarFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
6986
6708
|
fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev, fdev_minor, fdev_major, "+"+str(len(formatspecs['format_delimiter']))]
|
|
6987
6709
|
AppendFileHeaderWithContent(
|
|
6988
6710
|
fp, tmpoutlist, extradata, jsondata, fcontents.read(), [checksumtype[1], checksumtype[2], checksumtype[3]], formatspecs)
|
|
6711
|
+
try:
|
|
6712
|
+
fp.flush()
|
|
6713
|
+
if(hasattr(os, "sync")):
|
|
6714
|
+
os.fsync(fp.fileno())
|
|
6715
|
+
except io.UnsupportedOperation:
|
|
6716
|
+
pass
|
|
6717
|
+
except AttributeError:
|
|
6718
|
+
pass
|
|
6719
|
+
except OSError:
|
|
6720
|
+
pass
|
|
6989
6721
|
fcontents.close()
|
|
6990
6722
|
return fp
|
|
6991
6723
|
|
|
@@ -6995,8 +6727,6 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
6995
6727
|
if(verbose):
|
|
6996
6728
|
logging.basicConfig(format="%(message)s",
|
|
6997
6729
|
stream=sys.stdout, level=logging.DEBUG)
|
|
6998
|
-
formver = formatspecs['format_ver']
|
|
6999
|
-
fileheaderver = str(int(formver.replace(".", "")))
|
|
7000
6730
|
curinode = 0
|
|
7001
6731
|
curfid = 0
|
|
7002
6732
|
inodelist = []
|
|
@@ -7034,6 +6764,16 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7034
6764
|
VerbosePrintOut("Bad file found!")
|
|
7035
6765
|
numfiles = int(len(zipfp.infolist()))
|
|
7036
6766
|
AppendFileHeader(fp, numfiles, "UTF-8", [], checksumtype[0], formatspecs)
|
|
6767
|
+
try:
|
|
6768
|
+
fp.flush()
|
|
6769
|
+
if(hasattr(os, "sync")):
|
|
6770
|
+
os.fsync(fp.fileno())
|
|
6771
|
+
except io.UnsupportedOperation:
|
|
6772
|
+
pass
|
|
6773
|
+
except AttributeError:
|
|
6774
|
+
pass
|
|
6775
|
+
except OSError:
|
|
6776
|
+
pass
|
|
7037
6777
|
for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
|
|
7038
6778
|
fencoding = "UTF-8"
|
|
7039
6779
|
if(re.findall("^[.|/]", member.filename)):
|
|
@@ -7044,9 +6784,9 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7044
6784
|
if(verbose):
|
|
7045
6785
|
VerbosePrintOut(fname)
|
|
7046
6786
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7047
|
-
fpremode = int(stat.S_IFDIR
|
|
6787
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7048
6788
|
else:
|
|
7049
|
-
fpremode = int(stat.S_IFREG
|
|
6789
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7050
6790
|
flinkcount = 0
|
|
7051
6791
|
ftype = 0
|
|
7052
6792
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
@@ -7075,37 +6815,42 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7075
6815
|
fbtime = format(
|
|
7076
6816
|
int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower()
|
|
7077
6817
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
7078
|
-
fwinattributes = format(int(zipinfo.external_attr), 'x').lower()
|
|
6818
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7079
6819
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7080
|
-
fmode = format(int(stat.S_IFDIR
|
|
7081
|
-
fchmode = stat.S_IMODE(int(stat.S_IFDIR
|
|
7082
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFDIR
|
|
6820
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6821
|
+
fchmode = stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))
|
|
6822
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))
|
|
7083
6823
|
else:
|
|
7084
|
-
fmode = format(int(stat.S_IFREG
|
|
7085
|
-
fchmode = stat.S_IMODE(int(stat.S_IFREG
|
|
7086
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFREG
|
|
6824
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6825
|
+
fchmode = stat.S_IMODE(int(stat.S_IFREG | 0x1b6))
|
|
6826
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFREG | 0x1b6))
|
|
7087
6827
|
elif(zipinfo.create_system == 3):
|
|
7088
|
-
fwinattributes = format(int(
|
|
7089
|
-
|
|
7090
|
-
|
|
7091
|
-
|
|
7092
|
-
|
|
7093
|
-
|
|
7094
|
-
|
|
7095
|
-
|
|
7096
|
-
|
|
7097
|
-
|
|
7098
|
-
|
|
7099
|
-
|
|
7100
|
-
|
|
6828
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
6829
|
+
fmode = format(int((zipinfo.external_attr >> 16) & 0xFFFF), 'x').lower()
|
|
6830
|
+
prefmode = int((zipinfo.external_attr >> 16) & 0xFFFF)
|
|
6831
|
+
if (prefmode == 0):
|
|
6832
|
+
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
6833
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6834
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
6835
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6836
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6837
|
+
else:
|
|
6838
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6839
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
6840
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6841
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6842
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6843
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6844
|
+
else:
|
|
6845
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7101
6846
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7102
|
-
fmode = format(int(stat.S_IFDIR
|
|
7103
|
-
prefmode = int(stat.S_IFDIR
|
|
6847
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6848
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
7104
6849
|
fchmode = stat.S_IMODE(prefmode)
|
|
7105
6850
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7106
6851
|
else:
|
|
7107
|
-
fmode = format(int(stat.S_IFREG
|
|
7108
|
-
prefmode = int(stat.S_IFREG
|
|
6852
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6853
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
7109
6854
|
fchmode = stat.S_IMODE(prefmode)
|
|
7110
6855
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7111
6856
|
fcompression = ""
|
|
@@ -7196,6 +6941,16 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7196
6941
|
fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev, fdev_minor, fdev_major, "+"+str(len(formatspecs['format_delimiter']))]
|
|
7197
6942
|
AppendFileHeaderWithContent(
|
|
7198
6943
|
fp, tmpoutlist, extradata, jsondata, fcontents.read(), [checksumtype[1], checksumtype[2], checksumtype[3]], formatspecs)
|
|
6944
|
+
try:
|
|
6945
|
+
fp.flush()
|
|
6946
|
+
if(hasattr(os, "sync")):
|
|
6947
|
+
os.fsync(fp.fileno())
|
|
6948
|
+
except io.UnsupportedOperation:
|
|
6949
|
+
pass
|
|
6950
|
+
except AttributeError:
|
|
6951
|
+
pass
|
|
6952
|
+
except OSError:
|
|
6953
|
+
pass
|
|
7199
6954
|
fcontents.close()
|
|
7200
6955
|
return fp
|
|
7201
6956
|
|
|
@@ -7210,8 +6965,6 @@ if(rarfile_support):
|
|
|
7210
6965
|
if(verbose):
|
|
7211
6966
|
logging.basicConfig(format="%(message)s",
|
|
7212
6967
|
stream=sys.stdout, level=logging.DEBUG)
|
|
7213
|
-
formver = formatspecs['format_ver']
|
|
7214
|
-
fileheaderver = str(int(formver.replace(".", "")))
|
|
7215
6968
|
curinode = 0
|
|
7216
6969
|
curfid = 0
|
|
7217
6970
|
inodelist = []
|
|
@@ -7228,6 +6981,16 @@ if(rarfile_support):
|
|
|
7228
6981
|
VerbosePrintOut("Bad file found!")
|
|
7229
6982
|
numfiles = int(len(rarfp.infolist()))
|
|
7230
6983
|
AppendFileHeader(fp, numfiles, "UTF-8", [], checksumtype[0], formatspecs)
|
|
6984
|
+
try:
|
|
6985
|
+
fp.flush()
|
|
6986
|
+
if(hasattr(os, "sync")):
|
|
6987
|
+
os.fsync(fp.fileno())
|
|
6988
|
+
except io.UnsupportedOperation:
|
|
6989
|
+
pass
|
|
6990
|
+
except AttributeError:
|
|
6991
|
+
pass
|
|
6992
|
+
except OSError:
|
|
6993
|
+
pass
|
|
7231
6994
|
try:
|
|
7232
6995
|
fp.flush()
|
|
7233
6996
|
if(hasattr(os, "sync")):
|
|
@@ -7269,11 +7032,11 @@ if(rarfile_support):
|
|
|
7269
7032
|
if(is_unix and member.external_attr != 0):
|
|
7270
7033
|
fpremode = int(member.external_attr)
|
|
7271
7034
|
elif(member.is_file()):
|
|
7272
|
-
fpremode = int(stat.S_IFREG
|
|
7035
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7273
7036
|
elif(member.is_symlink()):
|
|
7274
|
-
fpremode = int(stat.S_IFLNK
|
|
7037
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
7275
7038
|
elif(member.is_dir()):
|
|
7276
|
-
fpremode = int(stat.S_IFDIR
|
|
7039
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7277
7040
|
if(is_windows and member.external_attr != 0):
|
|
7278
7041
|
fwinattributes = format(int(member.external_attr), 'x').lower()
|
|
7279
7042
|
else:
|
|
@@ -7326,23 +7089,23 @@ if(rarfile_support):
|
|
|
7326
7089
|
ftypemod = format(
|
|
7327
7090
|
int(stat.S_IFMT(member.external_attr)), 'x').lower()
|
|
7328
7091
|
elif(member.is_file()):
|
|
7329
|
-
fmode = format(int(stat.S_IFREG
|
|
7092
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7330
7093
|
fchmode = format(
|
|
7331
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7094
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7332
7095
|
ftypemod = format(
|
|
7333
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7096
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7334
7097
|
elif(member.is_symlink()):
|
|
7335
|
-
fmode = format(int(stat.S_IFLNK
|
|
7098
|
+
fmode = format(int(stat.S_IFLNK | 0x1b6), 'x').lower()
|
|
7336
7099
|
fchmode = format(
|
|
7337
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7100
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7338
7101
|
ftypemod = format(
|
|
7339
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7102
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7340
7103
|
elif(member.is_dir()):
|
|
7341
|
-
fmode = format(int(stat.S_IFDIR
|
|
7104
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7342
7105
|
fchmode = format(
|
|
7343
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7106
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7344
7107
|
ftypemod = format(
|
|
7345
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7108
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7346
7109
|
try:
|
|
7347
7110
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7348
7111
|
except AttributeError:
|
|
@@ -7432,15 +7195,25 @@ if(rarfile_support):
|
|
|
7432
7195
|
fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev, fdev_minor, fdev_major, "+"+str(len(formatspecs['format_delimiter']))]
|
|
7433
7196
|
AppendFileHeaderWithContent(
|
|
7434
7197
|
fp, tmpoutlist, extradata, jsondata, fcontents.read(), [checksumtype[1], checksumtype[2], checksumtype[3]], formatspecs)
|
|
7198
|
+
try:
|
|
7199
|
+
fp.flush()
|
|
7200
|
+
if(hasattr(os, "sync")):
|
|
7201
|
+
os.fsync(fp.fileno())
|
|
7202
|
+
except io.UnsupportedOperation:
|
|
7203
|
+
pass
|
|
7204
|
+
except AttributeError:
|
|
7205
|
+
pass
|
|
7206
|
+
except OSError:
|
|
7207
|
+
pass
|
|
7435
7208
|
fcontents.close()
|
|
7436
7209
|
return fp
|
|
7437
7210
|
|
|
7438
7211
|
if(not py7zr_support):
|
|
7439
|
-
def
|
|
7212
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7440
7213
|
return False
|
|
7441
7214
|
|
|
7442
7215
|
if(py7zr_support):
|
|
7443
|
-
def
|
|
7216
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7444
7217
|
if(not hasattr(fp, "write")):
|
|
7445
7218
|
return False
|
|
7446
7219
|
if(verbose):
|
|
@@ -7464,6 +7237,16 @@ if(py7zr_support):
|
|
|
7464
7237
|
VerbosePrintOut("Bad file found!")
|
|
7465
7238
|
numfiles = int(len(szpfp.list()))
|
|
7466
7239
|
AppendFileHeader(fp, numfiles, "UTF-8", [], checksumtype[0], formatspecs)
|
|
7240
|
+
try:
|
|
7241
|
+
fp.flush()
|
|
7242
|
+
if(hasattr(os, "sync")):
|
|
7243
|
+
os.fsync(fp.fileno())
|
|
7244
|
+
except io.UnsupportedOperation:
|
|
7245
|
+
pass
|
|
7246
|
+
except AttributeError:
|
|
7247
|
+
pass
|
|
7248
|
+
except OSError:
|
|
7249
|
+
pass
|
|
7467
7250
|
for member in sorted(szpfp.list(), key=lambda x: x.filename):
|
|
7468
7251
|
fencoding = "UTF-8"
|
|
7469
7252
|
if(re.findall("^[.|/]", member.filename)):
|
|
@@ -7473,9 +7256,9 @@ if(py7zr_support):
|
|
|
7473
7256
|
if(verbose):
|
|
7474
7257
|
VerbosePrintOut(fname)
|
|
7475
7258
|
if(not member.is_directory):
|
|
7476
|
-
fpremode = int(stat.S_IFREG
|
|
7259
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7477
7260
|
elif(member.is_directory):
|
|
7478
|
-
fpremode = int(stat.S_IFDIR
|
|
7261
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7479
7262
|
fwinattributes = format(int(0), 'x').lower()
|
|
7480
7263
|
fcompression = ""
|
|
7481
7264
|
fcsize = format(int(0), 'x').lower()
|
|
@@ -7499,17 +7282,17 @@ if(py7zr_support):
|
|
|
7499
7282
|
fctime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7500
7283
|
fbtime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7501
7284
|
if(member.is_directory):
|
|
7502
|
-
fmode = format(int(stat.S_IFDIR
|
|
7285
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7503
7286
|
fchmode = format(
|
|
7504
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7287
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7505
7288
|
ftypemod = format(
|
|
7506
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7289
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7507
7290
|
else:
|
|
7508
|
-
fmode = format(int(stat.S_IFREG
|
|
7291
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7509
7292
|
fchmode = format(
|
|
7510
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7293
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7511
7294
|
ftypemod = format(
|
|
7512
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7295
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7513
7296
|
try:
|
|
7514
7297
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7515
7298
|
except AttributeError:
|
|
@@ -7602,10 +7385,20 @@ if(py7zr_support):
|
|
|
7602
7385
|
fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev, fdev_minor, fdev_major, "+"+str(len(formatspecs['format_delimiter']))]
|
|
7603
7386
|
AppendFileHeaderWithContent(
|
|
7604
7387
|
fp, tmpoutlist, extradata, jsondata, fcontents.read(), [checksumtype[1], checksumtype[2], checksumtype[3]], formatspecs)
|
|
7388
|
+
try:
|
|
7389
|
+
fp.flush()
|
|
7390
|
+
if(hasattr(os, "sync")):
|
|
7391
|
+
os.fsync(fp.fileno())
|
|
7392
|
+
except io.UnsupportedOperation:
|
|
7393
|
+
pass
|
|
7394
|
+
except AttributeError:
|
|
7395
|
+
pass
|
|
7396
|
+
except OSError:
|
|
7397
|
+
pass
|
|
7605
7398
|
fcontents.close()
|
|
7606
7399
|
return fp
|
|
7607
7400
|
|
|
7608
|
-
def AppendListsWithContent(inlist, fp, dirlistfromtxt=False,
|
|
7401
|
+
def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7609
7402
|
if(not hasattr(fp, "write")):
|
|
7610
7403
|
return False
|
|
7611
7404
|
if(verbose):
|
|
@@ -7667,12 +7460,12 @@ def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extr
|
|
|
7667
7460
|
return fp
|
|
7668
7461
|
|
|
7669
7462
|
|
|
7670
|
-
def AppendInFileWithContent(infile, fp, dirlistfromtxt=False,
|
|
7463
|
+
def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, extradata=[], jsondata={}, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7671
7464
|
inlist = ReadInFileWithContentToList(infile, "auto", 0, 0, False, False, True, False, formatspecs)
|
|
7672
|
-
return AppendListsWithContent(inlist, fp, dirlistfromtxt,
|
|
7465
|
+
return AppendListsWithContent(inlist, fp, dirlistfromtxt, extradata, jsondata, followlink, checksumtype, formatspecs, verbose)
|
|
7673
7466
|
|
|
7674
7467
|
|
|
7675
|
-
def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt,
|
|
7468
|
+
def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7676
7469
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
7677
7470
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
7678
7471
|
get_in_ext = os.path.splitext(outfile)
|
|
@@ -7716,7 +7509,7 @@ def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, fmtt
|
|
|
7716
7509
|
fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
|
|
7717
7510
|
except PermissionError:
|
|
7718
7511
|
return False
|
|
7719
|
-
AppendFilesWithContent(infiles, fp, dirlistfromtxt,
|
|
7512
|
+
AppendFilesWithContent(infiles, fp, dirlistfromtxt, extradata, jsondata, compression,
|
|
7720
7513
|
compresswholefile, compressionlevel, compressionuselist, followlink, checksumtype, formatspecs, verbose)
|
|
7721
7514
|
if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
|
|
7722
7515
|
fp = CompressOpenFileAlt(
|
|
@@ -7753,12 +7546,12 @@ def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, fmtt
|
|
|
7753
7546
|
fp.close()
|
|
7754
7547
|
return True
|
|
7755
7548
|
|
|
7756
|
-
def AppendFilesWithContentToStackedOutFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt,
|
|
7549
|
+
def AppendFilesWithContentToStackedOutFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7757
7550
|
if not isinstance(infiles, list):
|
|
7758
7551
|
infiles = [infiles]
|
|
7759
7552
|
returnout = False
|
|
7760
7553
|
for infileslist in infiles:
|
|
7761
|
-
returnout = AppendFilesWithContentToOutFile(infileslist, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist,
|
|
7554
|
+
returnout = AppendFilesWithContentToOutFile(infileslist, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, followlink, checksumtype, formatspecs, verbose, True)
|
|
7762
7555
|
if(not returnout):
|
|
7763
7556
|
break
|
|
7764
7557
|
else:
|
|
@@ -7768,7 +7561,7 @@ def AppendFilesWithContentToStackedOutFile(infiles, outfile, dirlistfromtxt=Fals
|
|
|
7768
7561
|
return True
|
|
7769
7562
|
return returnout
|
|
7770
7563
|
|
|
7771
|
-
def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None,
|
|
7564
|
+
def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, extradata=[], jsondata={}, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
7772
7565
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
7773
7566
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
7774
7567
|
get_in_ext = os.path.splitext(outfile)
|
|
@@ -7809,7 +7602,7 @@ def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, fmtty
|
|
|
7809
7602
|
fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
|
|
7810
7603
|
except PermissionError:
|
|
7811
7604
|
return False
|
|
7812
|
-
AppendListsWithContent(inlist, fp, dirlistfromtxt,
|
|
7605
|
+
AppendListsWithContent(inlist, fp, dirlistfromtxt, extradata, jsondata, compression,
|
|
7813
7606
|
compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose)
|
|
7814
7607
|
if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
|
|
7815
7608
|
fp = CompressOpenFileAlt(
|
|
@@ -7927,6 +7720,21 @@ def AppendFilesWithContentFromTarFileToOutFile(infiles, outfile, fmttype="auto",
|
|
|
7927
7720
|
fp.close()
|
|
7928
7721
|
return True
|
|
7929
7722
|
|
|
7723
|
+
def AppendFilesWithContentFromTarFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7724
|
+
if not isinstance(infiles, list):
|
|
7725
|
+
infiles = [infiles]
|
|
7726
|
+
returnout = False
|
|
7727
|
+
for infileslist in infiles:
|
|
7728
|
+
returnout = AppendFilesWithContentFromTarFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
7729
|
+
if(not returnout):
|
|
7730
|
+
break
|
|
7731
|
+
else:
|
|
7732
|
+
outfile = returnout
|
|
7733
|
+
if(not returnfp and returnout):
|
|
7734
|
+
returnout.close()
|
|
7735
|
+
return True
|
|
7736
|
+
return returnout
|
|
7737
|
+
|
|
7930
7738
|
def AppendFilesWithContentFromZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7931
7739
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
7932
7740
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
@@ -8007,6 +7815,21 @@ def AppendFilesWithContentFromZipFileToOutFile(infiles, outfile, fmttype="auto",
|
|
|
8007
7815
|
fp.close()
|
|
8008
7816
|
return True
|
|
8009
7817
|
|
|
7818
|
+
def AppendFilesWithContentFromZipFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7819
|
+
if not isinstance(infiles, list):
|
|
7820
|
+
infiles = [infiles]
|
|
7821
|
+
returnout = False
|
|
7822
|
+
for infileslist in infiles:
|
|
7823
|
+
returnout = AppendFilesWithContentFromZipFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
7824
|
+
if(not returnout):
|
|
7825
|
+
break
|
|
7826
|
+
else:
|
|
7827
|
+
outfile = returnout
|
|
7828
|
+
if(not returnfp and returnout):
|
|
7829
|
+
returnout.close()
|
|
7830
|
+
return True
|
|
7831
|
+
return returnout
|
|
7832
|
+
|
|
8010
7833
|
if(not rarfile_support):
|
|
8011
7834
|
def AppendFilesWithContentFromRarFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8012
7835
|
return False
|
|
@@ -8092,12 +7915,27 @@ if(rarfile_support):
|
|
|
8092
7915
|
fp.close()
|
|
8093
7916
|
return True
|
|
8094
7917
|
|
|
7918
|
+
def AppendFilesWithContentFromRarFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
7919
|
+
if not isinstance(infiles, list):
|
|
7920
|
+
infiles = [infiles]
|
|
7921
|
+
returnout = False
|
|
7922
|
+
for infileslist in infiles:
|
|
7923
|
+
returnout = AppendFilesWithContentFromRarFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
7924
|
+
if(not returnout):
|
|
7925
|
+
break
|
|
7926
|
+
else:
|
|
7927
|
+
outfile = returnout
|
|
7928
|
+
if(not returnfp and returnout):
|
|
7929
|
+
returnout.close()
|
|
7930
|
+
return True
|
|
7931
|
+
return returnout
|
|
7932
|
+
|
|
8095
7933
|
if(not py7zr_support):
|
|
8096
|
-
def
|
|
7934
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8097
7935
|
return False
|
|
8098
7936
|
|
|
8099
7937
|
if(py7zr_support):
|
|
8100
|
-
def
|
|
7938
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8101
7939
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
8102
7940
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
8103
7941
|
get_in_ext = os.path.splitext(outfile)
|
|
@@ -8139,7 +7977,7 @@ if(py7zr_support):
|
|
|
8139
7977
|
fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
|
|
8140
7978
|
except PermissionError:
|
|
8141
7979
|
return False
|
|
8142
|
-
|
|
7980
|
+
AppendFilesWithContentFromSevenZipFile(infiles, fp, extradata, jsondata, compression,
|
|
8143
7981
|
compresswholefile, compressionlevel, compressionuselist, checksumtype, formatspecs, verbose)
|
|
8144
7982
|
if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
|
|
8145
7983
|
fp = CompressOpenFileAlt(
|
|
@@ -8177,9 +8015,24 @@ if(py7zr_support):
|
|
|
8177
8015
|
fp.close()
|
|
8178
8016
|
return True
|
|
8179
8017
|
|
|
8180
|
-
def
|
|
8018
|
+
def AppendFilesWithContentFromSevenZipFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8019
|
+
if not isinstance(infiles, list):
|
|
8020
|
+
infiles = [infiles]
|
|
8021
|
+
returnout = False
|
|
8022
|
+
for infileslist in infiles:
|
|
8023
|
+
returnout = AppendFilesWithContentFromSevenZipFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
8024
|
+
if(not returnout):
|
|
8025
|
+
break
|
|
8026
|
+
else:
|
|
8027
|
+
outfile = returnout
|
|
8028
|
+
if(not returnfp and returnout):
|
|
8029
|
+
returnout.close()
|
|
8030
|
+
return True
|
|
8031
|
+
return returnout
|
|
8032
|
+
|
|
8033
|
+
def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, extradata=[], jsondata={}, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
8181
8034
|
inlist = ReadInFileWithContentToList(infile, "auto", 0, 0, False, False, True, False, formatspecs)
|
|
8182
|
-
return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel,
|
|
8035
|
+
return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, extradata, jsondata, followlink, checksumtype, formatspecs, verbose, returnfp)
|
|
8183
8036
|
|
|
8184
8037
|
|
|
8185
8038
|
def PrintPermissionString(fchmode, ftype):
|
|
@@ -8871,8 +8724,6 @@ def CheckCompressionSubType(infile, formatspecs=__file_format_multi_dict__, file
|
|
|
8871
8724
|
fp = pyzstd.zstdfile.ZstdFile(infile, mode="rb")
|
|
8872
8725
|
else:
|
|
8873
8726
|
return Flase
|
|
8874
|
-
elif((compresscheck == "lzo" or compresscheck == "lzop") and compresscheck in compressionsupport):
|
|
8875
|
-
fp = LzopFile(infile, mode="rb")
|
|
8876
8727
|
elif((compresscheck == "lzma" or compresscheck == "xz") and compresscheck in compressionsupport):
|
|
8877
8728
|
fp = lzma.open(infile, "rb")
|
|
8878
8729
|
elif(compresscheck == "zlib" and compresscheck in compressionsupport):
|
|
@@ -8995,8 +8846,6 @@ def UncompressFileAlt(fp, formatspecs=__file_format_multi_dict__, filestart=0,
|
|
|
8995
8846
|
return False
|
|
8996
8847
|
elif kind == "lz4" and "lz4" in compressionsupport:
|
|
8997
8848
|
wrapped = lz4.frame.LZ4FrameFile(src, mode="rb")
|
|
8998
|
-
elif kind in ("lzo","lzop") and (("lzo" in compressionsupport) or ("lzop" in compressionsupport)):
|
|
8999
|
-
wrapped = LzopFile(fileobj=src, mode="rb")
|
|
9000
8849
|
elif kind == "zlib" and "zlib" in compressionsupport:
|
|
9001
8850
|
wrapped = ZlibFile(fileobj=src, mode="rb")
|
|
9002
8851
|
else:
|
|
@@ -9068,8 +8917,6 @@ def UncompressFile(infile, formatspecs=__file_format_multi_dict__, mode="rb",
|
|
|
9068
8917
|
return False
|
|
9069
8918
|
elif (compresscheck == "lz4" and "lz4" in compressionsupport):
|
|
9070
8919
|
fp = lz4.frame.open(infile, mode)
|
|
9071
|
-
elif ((compresscheck == "lzo" or compresscheck == "lzop") and "lzop" in compressionsupport):
|
|
9072
|
-
fp = LzopFile(infile, mode=mode)
|
|
9073
8920
|
elif ((compresscheck == "lzma" or compresscheck == "xz") and "xz" in compressionsupport):
|
|
9074
8921
|
fp = lzma.open(infile, mode)
|
|
9075
8922
|
elif (compresscheck == "zlib" and "zlib" in compressionsupport):
|
|
@@ -9637,7 +9484,7 @@ def fast_copy(infp, outfp, bufsize=1 << 20):
|
|
|
9637
9484
|
outfp.write(data)
|
|
9638
9485
|
|
|
9639
9486
|
|
|
9640
|
-
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=
|
|
9487
|
+
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
|
|
9641
9488
|
"""
|
|
9642
9489
|
Copy a disk file into an mmap-backed destination (FileLikeAdapter).
|
|
9643
9490
|
Falls back to buffered copy if the source cannot be mmapped.
|
|
@@ -9863,9 +9710,6 @@ def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None,
|
|
|
9863
9710
|
elif (fextname == ".lz4" and "lz4" in compressionsupport):
|
|
9864
9711
|
outfp = FileLikeAdapter(lz4.frame.open(outfile, mode, compression_level=compressionlevel), mode="wb")
|
|
9865
9712
|
|
|
9866
|
-
elif (fextname == ".lzo" and "lzop" in compressionsupport):
|
|
9867
|
-
outfp = FileLikeAdapter(LzopFile(outfile, mode=mode, level=compressionlevel), mode="wb")
|
|
9868
|
-
|
|
9869
9713
|
elif (fextname == ".lzma" and "lzma" in compressionsupport):
|
|
9870
9714
|
try:
|
|
9871
9715
|
outfp = FileLikeAdapter(
|
|
@@ -9955,10 +9799,10 @@ def CheckSumSupportAlt(checkfor, guaranteed=True):
|
|
|
9955
9799
|
|
|
9956
9800
|
|
|
9957
9801
|
def PackFoxFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
9958
|
-
return AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist,
|
|
9802
|
+
return AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, followlink, checksumtype, formatspecs, verbose, returnfp)
|
|
9959
9803
|
|
|
9960
9804
|
def PackStackedFoxFile(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, followlink=False, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
9961
|
-
return AppendFilesWithContentToStackedOutFile(infiles, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist,
|
|
9805
|
+
return AppendFilesWithContentToStackedOutFile(infiles, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, followlink, checksumtype, formatspecs, verbose, returnfp)
|
|
9962
9806
|
|
|
9963
9807
|
def PackFoxFileFromDirList(infiles, outfile, dirlistfromtxt=False, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, followlink=False, checksumtype=["crc32", "crc32", "crc32"], extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
9964
9808
|
return PackFoxFile(infiles, outfile, dirlistfromtxt, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, followlink, checksumtype, extradata, formatspecs, verbose, returnfp)
|
|
@@ -9987,7 +9831,7 @@ if(not py7zr_support):
|
|
|
9987
9831
|
|
|
9988
9832
|
if(py7zr_support):
|
|
9989
9833
|
def PackFoxFileFromSevenZipFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
9990
|
-
return
|
|
9834
|
+
return AppendFilesWithContentFromSevenZipFileToOutFile(infile, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, returnfp)
|
|
9991
9835
|
|
|
9992
9836
|
|
|
9993
9837
|
def PackFoxFileFromInFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
@@ -10095,14 +9939,6 @@ def FoxFileValidate(infile, fmttype="auto", filestart=0,
|
|
|
10095
9939
|
checkcompressfile = CheckCompressionSubType(fp, formatspecs, filestart, True)
|
|
10096
9940
|
if(IsNestedDict(formatspecs) and checkcompressfile in formatspecs):
|
|
10097
9941
|
formatspecs = formatspecs[checkcompressfile]
|
|
10098
|
-
if(checkcompressfile == "tarfile" and TarFileCheck(infile)):
|
|
10099
|
-
return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10100
|
-
elif(checkcompressfile == "zipfile" and zipfile.is_zipfile(infile)):
|
|
10101
|
-
return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10102
|
-
elif(rarfile_support and checkcompressfile == "rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
|
|
10103
|
-
return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10104
|
-
elif(py7zr_support and checkcompressfile == "7zipfile" and py7zr.is_7zfile(infile)):
|
|
10105
|
-
return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10106
9942
|
elif(IsSingleDict(formatspecs) and checkcompressfile != formatspecs['format_magic']):
|
|
10107
9943
|
return False
|
|
10108
9944
|
elif(IsNestedDict(formatspecs) and checkcompressfile not in formatspecs):
|
|
@@ -10464,9 +10300,10 @@ def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__fi
|
|
|
10464
10300
|
while True:
|
|
10465
10301
|
if outstartfile >= outfsize: # stop when function signals False
|
|
10466
10302
|
break
|
|
10467
|
-
is_valid_file =
|
|
10303
|
+
is_valid_file = ArchiveFileValidate(infile, fmttype, outstartfile, formatspecs, seektoend, verbose, True)
|
|
10468
10304
|
if is_valid_file is False: # stop when function signals False
|
|
10469
10305
|
outretval.append(is_valid_file)
|
|
10306
|
+
break
|
|
10470
10307
|
else:
|
|
10471
10308
|
outretval.append(True)
|
|
10472
10309
|
infile = is_valid_file
|
|
@@ -11472,35 +11309,6 @@ def MultipleFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, s
|
|
|
11472
11309
|
return outretval
|
|
11473
11310
|
|
|
11474
11311
|
|
|
11475
|
-
def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, returnfp=False):
|
|
11476
|
-
outretval = []
|
|
11477
|
-
outstartfile = filestart
|
|
11478
|
-
outfsize = float('inf')
|
|
11479
|
-
while True:
|
|
11480
|
-
if outstartfile >= outfsize: # stop when function signals False
|
|
11481
|
-
break
|
|
11482
|
-
is_valid_file = FoxFileValidate(infile, fmttype, filestart, formatspecs, seektoend, verbose, True)
|
|
11483
|
-
if is_valid_file is False: # stop when function signals False
|
|
11484
|
-
outretval.append(is_valid_file)
|
|
11485
|
-
else:
|
|
11486
|
-
outretval.append(True)
|
|
11487
|
-
infile = is_valid_file
|
|
11488
|
-
outstartfile = infile.tell()
|
|
11489
|
-
try:
|
|
11490
|
-
infile.seek(0, 2)
|
|
11491
|
-
except OSError:
|
|
11492
|
-
SeekToEndOfFile(infile)
|
|
11493
|
-
except ValueError:
|
|
11494
|
-
SeekToEndOfFile(infile)
|
|
11495
|
-
outfsize = infile.tell()
|
|
11496
|
-
infile.seek(outstartfile, 0)
|
|
11497
|
-
if(returnfp):
|
|
11498
|
-
return infile
|
|
11499
|
-
else:
|
|
11500
|
-
infile.close()
|
|
11501
|
-
return outretval
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
11312
|
def StackedFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, newstyle=False, returnfp=False):
|
|
11505
11313
|
outretval = []
|
|
11506
11314
|
outstartfile = filestart
|
|
@@ -11711,11 +11519,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11711
11519
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
11712
11520
|
fwinattributes = int(zipinfo.external_attr)
|
|
11713
11521
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11714
|
-
fmode = int(stat.S_IFDIR
|
|
11715
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11716
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11522
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11523
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11524
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11717
11525
|
else:
|
|
11718
|
-
fmode = int(stat.S_IFREG
|
|
11526
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11719
11527
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11720
11528
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11721
11529
|
elif(zipinfo.create_system == 3):
|
|
@@ -11731,11 +11539,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11731
11539
|
else:
|
|
11732
11540
|
fwinattributes = int(0)
|
|
11733
11541
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11734
|
-
fmode = int(stat.S_IFDIR
|
|
11735
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11736
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11542
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11543
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11544
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11737
11545
|
else:
|
|
11738
|
-
fmode = int(stat.S_IFREG
|
|
11546
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11739
11547
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11740
11548
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11741
11549
|
returnval.update({lcfi: member.filename})
|
|
@@ -11846,11 +11654,11 @@ if(rarfile_support):
|
|
|
11846
11654
|
if(is_unix and member.external_attr != 0):
|
|
11847
11655
|
fpremode = int(member.external_attr)
|
|
11848
11656
|
elif(member.is_file()):
|
|
11849
|
-
fpremode = int(stat.S_IFREG
|
|
11657
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
11850
11658
|
elif(member.is_symlink()):
|
|
11851
|
-
fpremode = int(stat.S_IFLNK
|
|
11659
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
11852
11660
|
elif(member.is_dir()):
|
|
11853
|
-
fpremode = int(stat.S_IFDIR
|
|
11661
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
11854
11662
|
if(is_windows and member.external_attr != 0):
|
|
11855
11663
|
fwinattributes = int(member.external_attr)
|
|
11856
11664
|
else:
|
|
@@ -11860,17 +11668,17 @@ if(rarfile_support):
|
|
|
11860
11668
|
fchmode = int(stat.S_IMODE(member.external_attr))
|
|
11861
11669
|
ftypemod = int(stat.S_IFMT(member.external_attr))
|
|
11862
11670
|
elif(member.is_file()):
|
|
11863
|
-
fmode = int(stat.S_IFREG
|
|
11864
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFREG
|
|
11865
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG
|
|
11671
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11672
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6)))
|
|
11673
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6)))
|
|
11866
11674
|
elif(member.is_symlink()):
|
|
11867
|
-
fmode = int(stat.S_IFLNK
|
|
11868
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
11869
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11675
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11676
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11677
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
11870
11678
|
elif(member.is_dir()):
|
|
11871
|
-
fmode = int(stat.S_IFDIR
|
|
11872
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11873
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11679
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11680
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11681
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11874
11682
|
returnval.update({lcfi: member.filename})
|
|
11875
11683
|
if(not verbose):
|
|
11876
11684
|
VerbosePrintOut(member.filename)
|
|
@@ -11966,18 +11774,18 @@ if(py7zr_support):
|
|
|
11966
11774
|
else:
|
|
11967
11775
|
fname = "./"+member.filename
|
|
11968
11776
|
if(not member.is_directory):
|
|
11969
|
-
fpremode = int(stat.S_IFREG
|
|
11777
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
11970
11778
|
elif(member.is_directory):
|
|
11971
|
-
fpremode = int(stat.S_IFDIR
|
|
11779
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
11972
11780
|
fwinattributes = int(0)
|
|
11973
11781
|
if(member.is_directory):
|
|
11974
|
-
fmode = int(stat.S_IFDIR
|
|
11975
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11976
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11782
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11783
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11784
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11977
11785
|
else:
|
|
11978
|
-
fmode = int(stat.S_IFLNK
|
|
11979
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
11980
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11786
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11787
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11788
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
11981
11789
|
returnval.update({lcfi: member.filename})
|
|
11982
11790
|
if(not verbose):
|
|
11983
11791
|
VerbosePrintOut(member.filename)
|