PyArchiveFile 0.24.2__py3-none-any.whl → 0.24.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyarchivefile-0.24.2.data → pyarchivefile-0.24.6.data}/scripts/archivefile.py +28 -48
- {pyarchivefile-0.24.2.dist-info → pyarchivefile-0.24.6.dist-info}/METADATA +1 -1
- pyarchivefile-0.24.6.dist-info/RECORD +10 -0
- pyarchivefile.py +309 -656
- pyarchivefile-0.24.2.dist-info/RECORD +0 -10
- {pyarchivefile-0.24.2.data → pyarchivefile-0.24.6.data}/scripts/archiveneofile.py +0 -0
- {pyarchivefile-0.24.2.data → pyarchivefile-0.24.6.data}/scripts/neoarchivefile.py +0 -0
- {pyarchivefile-0.24.2.dist-info → pyarchivefile-0.24.6.dist-info}/WHEEL +0 -0
- {pyarchivefile-0.24.2.dist-info → pyarchivefile-0.24.6.dist-info}/licenses/LICENSE +0 -0
- {pyarchivefile-0.24.2.dist-info → pyarchivefile-0.24.6.dist-info}/top_level.txt +0 -0
- {pyarchivefile-0.24.2.dist-info → pyarchivefile-0.24.6.dist-info}/zip-safe +0 -0
pyarchivefile.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
|
|
15
15
|
Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
|
|
16
16
|
|
|
17
|
-
$FileInfo: pyarchivefile.py - Last Update:
|
|
17
|
+
$FileInfo: pyarchivefile.py - Last Update: 11/3/2025 Ver. 0.24.6 RC 1 - Author: cooldude2k $
|
|
18
18
|
'''
|
|
19
19
|
|
|
20
20
|
from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
|
|
@@ -33,10 +33,10 @@ import socket
|
|
|
33
33
|
import struct
|
|
34
34
|
import hashlib
|
|
35
35
|
import inspect
|
|
36
|
-
import datetime
|
|
37
36
|
import logging
|
|
38
37
|
import zipfile
|
|
39
38
|
import binascii
|
|
39
|
+
import datetime
|
|
40
40
|
import platform
|
|
41
41
|
from io import StringIO, BytesIO
|
|
42
42
|
from collections import namedtuple
|
|
@@ -413,7 +413,12 @@ def is_only_nonprintable(var):
|
|
|
413
413
|
__file_format_multi_dict__ = {}
|
|
414
414
|
__file_format_default__ = "ArchiveFile"
|
|
415
415
|
__include_defaults__ = True
|
|
416
|
-
__use_inmemfile__ =
|
|
416
|
+
__use_inmemfile__ = True
|
|
417
|
+
__use_spoolfile__ = False
|
|
418
|
+
__use_spooldir__ = tempfile.gettempdir()
|
|
419
|
+
BYTES_PER_MiB = 1024 * 1024
|
|
420
|
+
DEFAULT_SPOOL_MAX = 8 * BYTES_PER_MiB
|
|
421
|
+
__spoolfile_size__ = DEFAULT_SPOOL_MAX
|
|
417
422
|
__program_name__ = "Py"+__file_format_default__
|
|
418
423
|
__use_env_file__ = True
|
|
419
424
|
__use_ini_file__ = True
|
|
@@ -450,6 +455,8 @@ if __use_ini_file__ and os.path.exists(__config_file__):
|
|
|
450
455
|
__program_name__ = decode_unicode_escape(config.get('config', 'proname'))
|
|
451
456
|
__include_defaults__ = config.getboolean('config', 'includedef')
|
|
452
457
|
__use_inmemfile__ = config.getboolean('config', 'inmemfile')
|
|
458
|
+
__use_spoolfile__ = config.getboolean('config', 'usespoolfile')
|
|
459
|
+
__spoolfile_size__ = config.getint('config', 'spoolfilesize')
|
|
453
460
|
# Loop through all sections
|
|
454
461
|
for section in config.sections():
|
|
455
462
|
if section == "config":
|
|
@@ -543,6 +550,8 @@ elif __use_json_file__ and os.path.exists(__config_file__):
|
|
|
543
550
|
__program_name__ = decode_unicode_escape(_get(cfg_config, 'proname', ''))
|
|
544
551
|
__include_defaults__ = _to_bool(_get(cfg_config, 'includedef', False))
|
|
545
552
|
__use_inmemfile__ = _to_bool(_get(cfg_config, 'inmemfile', False))
|
|
553
|
+
__use_spoolfile__ = _to_bool(_get(cfg_config, 'usespoolfile', False))
|
|
554
|
+
__spoolfile_size__ = _to_int(_get(cfg_config, 'spoolfilesize', DEFAULT_SPOOL_MAX))
|
|
546
555
|
|
|
547
556
|
# --- iterate format sections (everything except "config") ---
|
|
548
557
|
required_keys = [
|
|
@@ -622,13 +631,15 @@ __use_alt_inode__ = __file_format_multi_dict__[__file_format_default__]['use_alt
|
|
|
622
631
|
__file_format_extension__ = __file_format_multi_dict__[__file_format_default__]['format_extension']
|
|
623
632
|
__file_format_dict__ = __file_format_multi_dict__[__file_format_default__]
|
|
624
633
|
__project__ = __program_name__
|
|
634
|
+
__program_alt_name__ = __program_name__
|
|
625
635
|
__project_url__ = "https://github.com/GameMaker2k/PyArchiveFile"
|
|
626
|
-
|
|
627
|
-
|
|
636
|
+
__project_release_url__ = __project_url__+"/releases/latest"
|
|
637
|
+
__version_info__ = (0, 24, 6, "RC 1", 1)
|
|
638
|
+
__version_date_info__ = (2025, 11, 3, "RC 1", 1)
|
|
628
639
|
__version_date__ = str(__version_date_info__[0]) + "." + str(
|
|
629
640
|
__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2)
|
|
630
641
|
__revision__ = __version_info__[3]
|
|
631
|
-
__revision_id__ = "$Id:
|
|
642
|
+
__revision_id__ = "$Id: 45d7adad4e0436f23599e18a9fb332d3724fa26d $"
|
|
632
643
|
if(__version_info__[4] is not None):
|
|
633
644
|
__version_date_plusrc__ = __version_date__ + \
|
|
634
645
|
"-" + str(__version_date_info__[4])
|
|
@@ -640,6 +651,78 @@ if(__version_info__[3] is not None):
|
|
|
640
651
|
if(__version_info__[3] is None):
|
|
641
652
|
__version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2])
|
|
642
653
|
|
|
654
|
+
# From: https://stackoverflow.com/a/28568003
|
|
655
|
+
# By Phaxmohdem
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def versiontuple(v):
|
|
659
|
+
filled = []
|
|
660
|
+
for point in v.split("."):
|
|
661
|
+
filled.append(point.zfill(8))
|
|
662
|
+
return tuple(filled)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def version_check(myvercheck, newvercheck):
|
|
666
|
+
vercheck = 0
|
|
667
|
+
try:
|
|
668
|
+
from packaging import version
|
|
669
|
+
vercheck = 1
|
|
670
|
+
except ImportError:
|
|
671
|
+
try:
|
|
672
|
+
from distutils.version import LooseVersion, StrictVersion
|
|
673
|
+
vercheck = 2
|
|
674
|
+
except ImportError:
|
|
675
|
+
try:
|
|
676
|
+
from pkg_resources import parse_version
|
|
677
|
+
vercheck = 3
|
|
678
|
+
except ImportError:
|
|
679
|
+
return 5
|
|
680
|
+
# print(myvercheck, newvercheck)
|
|
681
|
+
if (vercheck == 1):
|
|
682
|
+
if (version.parse(myvercheck) == version.parse(newvercheck)):
|
|
683
|
+
return 0
|
|
684
|
+
elif (version.parse(myvercheck) < version.parse(newvercheck)):
|
|
685
|
+
return 1
|
|
686
|
+
elif (version.parse(myvercheck) > version.parse(newvercheck)):
|
|
687
|
+
return 2
|
|
688
|
+
else:
|
|
689
|
+
return 3
|
|
690
|
+
elif (vercheck == 2):
|
|
691
|
+
if (StrictVersion(myvercheck) == StrictVersion(newvercheck)):
|
|
692
|
+
return 0
|
|
693
|
+
elif (StrictVersion(myvercheck) < StrictVersion(newvercheck)):
|
|
694
|
+
return 1
|
|
695
|
+
elif (StrictVersion(myvercheck) > StrictVersion(newvercheck)):
|
|
696
|
+
return 2
|
|
697
|
+
else:
|
|
698
|
+
return 3
|
|
699
|
+
elif (vercheck == 3):
|
|
700
|
+
if (parse_version(myvercheck) == parse_version(newvercheck)):
|
|
701
|
+
return 0
|
|
702
|
+
elif (parse_version(myvercheck) < parse_version(newvercheck)):
|
|
703
|
+
return 1
|
|
704
|
+
elif (parse_version(myvercheck) > parse_version(newvercheck)):
|
|
705
|
+
return 2
|
|
706
|
+
else:
|
|
707
|
+
return 3
|
|
708
|
+
else:
|
|
709
|
+
if (versiontuple(myvercheck) == versiontuple(newvercheck)):
|
|
710
|
+
return 0
|
|
711
|
+
elif (versiontuple(myvercheck) < versiontuple(newvercheck)):
|
|
712
|
+
return 1
|
|
713
|
+
elif (versiontuple(myvercheck) > versiontuple(newvercheck)):
|
|
714
|
+
return 2
|
|
715
|
+
else:
|
|
716
|
+
return 3
|
|
717
|
+
return 4
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def check_version_number(myversion=__version__, proname=__program_alt_name__, newverurl=__project_release_url__):
|
|
721
|
+
prevercheck = download_from_url(newverurl, geturls_headers, geturls_cj)
|
|
722
|
+
newvercheck = re.findall(proname + " ([0-9\\.]+)<\\/a\\>", prevercheck['Content'].decode("UTF-8"))[0]
|
|
723
|
+
myvercheck = re.findall("([0-9\\.]+)", myversion)[0]
|
|
724
|
+
return version_check(myvercheck, newvercheck)
|
|
725
|
+
|
|
643
726
|
# ===== Module-level type code table & helpers (reuse anywhere) =====
|
|
644
727
|
|
|
645
728
|
FT = {
|
|
@@ -740,6 +823,7 @@ try:
|
|
|
740
823
|
compressionsupport.append("lz4")
|
|
741
824
|
except ImportError:
|
|
742
825
|
pass
|
|
826
|
+
'''
|
|
743
827
|
try:
|
|
744
828
|
import lzo
|
|
745
829
|
compressionsupport.append("lzo")
|
|
@@ -747,6 +831,7 @@ try:
|
|
|
747
831
|
except ImportError:
|
|
748
832
|
lzo = None
|
|
749
833
|
pass
|
|
834
|
+
'''
|
|
750
835
|
try:
|
|
751
836
|
import zstandard
|
|
752
837
|
compressionsupport.append("zst")
|
|
@@ -806,11 +891,13 @@ if('lzo' in compressionsupport):
|
|
|
806
891
|
compressionlistalt.append('lzo')
|
|
807
892
|
outextlist.append('lzo')
|
|
808
893
|
outextlistwd.append('.lzo')
|
|
894
|
+
'''
|
|
809
895
|
if('lzop' in compressionsupport):
|
|
810
896
|
compressionlist.append('lzop')
|
|
811
897
|
compressionlistalt.append('lzop')
|
|
812
898
|
outextlist.append('lzop')
|
|
813
899
|
outextlistwd.append('.lzop')
|
|
900
|
+
'''
|
|
814
901
|
if('lzma' in compressionsupport):
|
|
815
902
|
compressionlist.append('lzma')
|
|
816
903
|
compressionlistalt.append('lzma')
|
|
@@ -1956,7 +2043,7 @@ def _normalize_initial_data(data, isbytes, encoding, errors=None):
|
|
|
1956
2043
|
|
|
1957
2044
|
|
|
1958
2045
|
def MkTempFile(data=None,
|
|
1959
|
-
inmem=
|
|
2046
|
+
inmem=__use_inmemfile__,
|
|
1960
2047
|
isbytes=True,
|
|
1961
2048
|
prefix="",
|
|
1962
2049
|
delete=True,
|
|
@@ -1964,9 +2051,9 @@ def MkTempFile(data=None,
|
|
|
1964
2051
|
newline=None, # text mode only; in-memory objects ignore newline semantics
|
|
1965
2052
|
dir=None,
|
|
1966
2053
|
suffix="",
|
|
1967
|
-
use_spool=
|
|
1968
|
-
spool_max=
|
|
1969
|
-
spool_dir=
|
|
2054
|
+
use_spool=__use_spoolfile__,
|
|
2055
|
+
spool_max=__spoolfile_size__,
|
|
2056
|
+
spool_dir=__use_spooldir__):
|
|
1970
2057
|
"""
|
|
1971
2058
|
Return a file-like handle with consistent behavior on Py2.7 and Py3.x.
|
|
1972
2059
|
|
|
@@ -2409,7 +2496,7 @@ class ZlibFile(object):
|
|
|
2409
2496
|
|
|
2410
2497
|
def __init__(self, file_path=None, fileobj=None, mode='rb', level=6, wbits=15,
|
|
2411
2498
|
encoding=None, errors=None, newline=None,
|
|
2412
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2499
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2413
2500
|
|
|
2414
2501
|
if file_path is None and fileobj is None:
|
|
2415
2502
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -2896,7 +2983,7 @@ class GzipFile(object):
|
|
|
2896
2983
|
|
|
2897
2984
|
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
2898
2985
|
level=6, encoding=None, errors=None, newline=None,
|
|
2899
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2986
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2900
2987
|
|
|
2901
2988
|
if file_path is None and fileobj is None:
|
|
2902
2989
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -3320,519 +3407,6 @@ def gzip_decompress_bytes_all_members(blob):
|
|
|
3320
3407
|
"""
|
|
3321
3408
|
return _gzip_decompress_multimember(bytes(blob))
|
|
3322
3409
|
|
|
3323
|
-
|
|
3324
|
-
# ---------- Simple LZO container (NOT real .lzop) ----------
|
|
3325
|
-
# File layout (concatenated members allowed):
|
|
3326
|
-
# [MAGIC 8B] [FLAGS 1B] [ULEN 8B] [CRC32 4B] [CCHUNK...] | repeat...
|
|
3327
|
-
# where:
|
|
3328
|
-
# MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3329
|
-
# FLAGS = bit0: 1 => member has ULEN+CRC, 0 => no header (legacy)
|
|
3330
|
-
# ULEN = uncompressed length (u64 BE)
|
|
3331
|
-
# CRC32 = CRC32 of uncompressed data (u32 BE)
|
|
3332
|
-
# CCHUNK = one or more compressed chunks:
|
|
3333
|
-
# [u32BE chunk_size][chunk_data] ... then a zero-size u32 terminator
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
class LzopFile(object):
|
|
3337
|
-
MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3338
|
-
FLAG_HAS_UHDR = 0x01
|
|
3339
|
-
RAW_CHUNK = 256 * 1024 # 256 KiB per raw (pre-compress) chunk
|
|
3340
|
-
|
|
3341
|
-
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
3342
|
-
level=9, encoding=None, errors=None, newline=None,
|
|
3343
|
-
write_header=True,
|
|
3344
|
-
tolerant_read=False, scan_bytes=(64 << 10),
|
|
3345
|
-
spool_threshold=(8 << 20)):
|
|
3346
|
-
"""
|
|
3347
|
-
Custom LZO file (NOT the lzop(1) format).
|
|
3348
|
-
- streaming write/read, supports concatenated members
|
|
3349
|
-
- optional per-member header (uncompressed length + CRC32)
|
|
3350
|
-
- spooled reads to limit RAM, strict text mode with newline control
|
|
3351
|
-
- tolerant_read: scan forward (up to scan_bytes) to first MAGIC
|
|
3352
|
-
|
|
3353
|
-
:param write_header: if True, include ULEN+CRC32 per member
|
|
3354
|
-
:param tolerant_read: skip leading junk up to scan_bytes to find MAGIC
|
|
3355
|
-
:param scan_bytes: max bytes to scan when tolerant_read=True
|
|
3356
|
-
:param spool_threshold: SpooledTemporaryFile RAM threshold before spill
|
|
3357
|
-
"""
|
|
3358
|
-
if lzo is None:
|
|
3359
|
-
raise ImportError("python-lzo is required for LzopFile")
|
|
3360
|
-
|
|
3361
|
-
if file_path is None and fileobj is None:
|
|
3362
|
-
raise ValueError("Either file_path or fileobj must be provided")
|
|
3363
|
-
if file_path is not None and fileobj is not None:
|
|
3364
|
-
raise ValueError("Only one of file_path or fileobj should be provided")
|
|
3365
|
-
|
|
3366
|
-
if 'b' not in mode and 't' not in mode:
|
|
3367
|
-
mode += 'b'
|
|
3368
|
-
if 'x' in mode and PY2:
|
|
3369
|
-
raise ValueError("Exclusive creation mode 'x' is not supported on Python 2")
|
|
3370
|
-
|
|
3371
|
-
self.file_path = file_path
|
|
3372
|
-
self.file = fileobj
|
|
3373
|
-
self.mode = mode
|
|
3374
|
-
self.level = int(level) # effective: 1 or 9 (clamped)
|
|
3375
|
-
self.encoding = encoding
|
|
3376
|
-
self.errors = errors
|
|
3377
|
-
self.newline = newline
|
|
3378
|
-
self._text_mode = ('t' in mode)
|
|
3379
|
-
|
|
3380
|
-
self._write_header = bool(write_header)
|
|
3381
|
-
|
|
3382
|
-
# Config (read path)
|
|
3383
|
-
self.tolerant_read = bool(tolerant_read)
|
|
3384
|
-
self.scan_bytes = int(scan_bytes)
|
|
3385
|
-
self.spool_threshold = int(spool_threshold)
|
|
3386
|
-
|
|
3387
|
-
# Write state
|
|
3388
|
-
self._crc = 0
|
|
3389
|
-
self._ulen = 0
|
|
3390
|
-
self._open_member = False
|
|
3391
|
-
self._member_header_pos = None # position *after* ULEN+CRC placeholders
|
|
3392
|
-
|
|
3393
|
-
# Read state
|
|
3394
|
-
self._spool = None
|
|
3395
|
-
self._text_reader = None
|
|
3396
|
-
self._position = 0
|
|
3397
|
-
self.closed = False
|
|
3398
|
-
|
|
3399
|
-
internal_mode = mode.replace('t', 'b')
|
|
3400
|
-
if self.file is None:
|
|
3401
|
-
if 'x' in internal_mode and os.path.exists(file_path):
|
|
3402
|
-
raise IOError("File exists: '{}'".format(file_path))
|
|
3403
|
-
self.file = open(file_path, internal_mode)
|
|
3404
|
-
else:
|
|
3405
|
-
if 'r' in internal_mode and not hasattr(self.file, 'read'):
|
|
3406
|
-
raise ValueError("fileobj must support read() in read mode")
|
|
3407
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')) and not hasattr(self.file, 'write'):
|
|
3408
|
-
raise ValueError("fileobj must support write() in write/append mode")
|
|
3409
|
-
|
|
3410
|
-
self._fp = self.file
|
|
3411
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')):
|
|
3412
|
-
# Start a new member at EOF for append
|
|
3413
|
-
if 'a' in internal_mode:
|
|
3414
|
-
try:
|
|
3415
|
-
self.file.seek(0, os.SEEK_END)
|
|
3416
|
-
except Exception:
|
|
3417
|
-
pass
|
|
3418
|
-
# Defer writing header until first write so empty files don’t get empty members
|
|
3419
|
-
elif 'r' in internal_mode:
|
|
3420
|
-
self._load_all_members_spooled()
|
|
3421
|
-
else:
|
|
3422
|
-
raise ValueError("Unsupported mode: {}".format(mode))
|
|
3423
|
-
|
|
3424
|
-
# ---------- helpers ----------
|
|
3425
|
-
@property
|
|
3426
|
-
def name(self):
|
|
3427
|
-
return self.file_path
|
|
3428
|
-
|
|
3429
|
-
def readable(self):
|
|
3430
|
-
return 'r' in self.mode
|
|
3431
|
-
|
|
3432
|
-
def writable(self):
|
|
3433
|
-
return any(ch in self.mode for ch in ('w', 'a', 'x'))
|
|
3434
|
-
|
|
3435
|
-
def seekable(self):
|
|
3436
|
-
return True if self._spool is not None else bool(getattr(self.file, 'seek', None))
|
|
3437
|
-
|
|
3438
|
-
def _normalize_newlines_for_write(self, s):
|
|
3439
|
-
nl = self.newline if self.newline is not None else "\n"
|
|
3440
|
-
return s.replace("\r\n", "\n").replace("\r", "\n").replace("\n", nl)
|
|
3441
|
-
|
|
3442
|
-
def _reader(self):
|
|
3443
|
-
return self._text_reader if self._text_mode else self._spool
|
|
3444
|
-
|
|
3445
|
-
# ---------- Write path ----------
|
|
3446
|
-
def _ensure_member_header(self):
|
|
3447
|
-
if self._open_member:
|
|
3448
|
-
return
|
|
3449
|
-
flags = self.FLAG_HAS_UHDR if self._write_header else 0
|
|
3450
|
-
self.file.write(self.MAGIC)
|
|
3451
|
-
self.file.write(struct.pack(">B", flags))
|
|
3452
|
-
if self._write_header:
|
|
3453
|
-
# placeholders for ULEN+CRC; we’ll backfill on finalize
|
|
3454
|
-
self.file.write(struct.pack(">Q", 0))
|
|
3455
|
-
self.file.write(struct.pack(">I", 0))
|
|
3456
|
-
# position *after* ULEN+CRC placeholders (or after FLAGS if no header)
|
|
3457
|
-
self._member_header_pos = self.file.tell()
|
|
3458
|
-
self._open_member = True
|
|
3459
|
-
# reset member stats
|
|
3460
|
-
self._crc = 0
|
|
3461
|
-
self._ulen = 0
|
|
3462
|
-
|
|
3463
|
-
def write(self, data):
|
|
3464
|
-
if 'r' in self.mode:
|
|
3465
|
-
raise IOError("File not open for writing")
|
|
3466
|
-
|
|
3467
|
-
if self._text_mode:
|
|
3468
|
-
enc = self.encoding or 'UTF-8'
|
|
3469
|
-
errs = self.errors or 'strict'
|
|
3470
|
-
if not isinstance(data, text_type):
|
|
3471
|
-
raise TypeError("write() expects text (unicode/str) in text mode")
|
|
3472
|
-
data = self._normalize_newlines_for_write(data).encode(enc, errs)
|
|
3473
|
-
else:
|
|
3474
|
-
if not isinstance(data, binary_types):
|
|
3475
|
-
raise TypeError("write() expects bytes-like in binary mode")
|
|
3476
|
-
|
|
3477
|
-
# Normalize Py3 memoryview / Py2 bytearray
|
|
3478
|
-
if (not PY2) and isinstance(data, memoryview):
|
|
3479
|
-
data = data.tobytes()
|
|
3480
|
-
elif PY2 and isinstance(data, bytearray):
|
|
3481
|
-
data = bytes(data)
|
|
3482
|
-
|
|
3483
|
-
if not data:
|
|
3484
|
-
return 0
|
|
3485
|
-
|
|
3486
|
-
# Begin member and write header on first write
|
|
3487
|
-
self._ensure_member_header()
|
|
3488
|
-
|
|
3489
|
-
# Update integrity stats
|
|
3490
|
-
self._crc = _crc32u(data, self._crc)
|
|
3491
|
-
self._ulen += len(data)
|
|
3492
|
-
|
|
3493
|
-
# Stream in RAW_CHUNK-sized pieces. Each piece becomes one compressed chunk record.
|
|
3494
|
-
mv = memoryview(data)
|
|
3495
|
-
# clamp level to {1, 9}
|
|
3496
|
-
lvl = 9 if self.level >= 9 else 1
|
|
3497
|
-
for off in range(0, len(data), self.RAW_CHUNK):
|
|
3498
|
-
raw = mv[off:off + self.RAW_CHUNK].tobytes()
|
|
3499
|
-
c = lzo.compress(raw, lvl)
|
|
3500
|
-
self.file.write(struct.pack(">I", len(c)))
|
|
3501
|
-
self.file.write(c)
|
|
3502
|
-
|
|
3503
|
-
return len(data)
|
|
3504
|
-
|
|
3505
|
-
def _flush_member_only(self):
|
|
3506
|
-
"""Finalize the current member: write terminator and backfill header."""
|
|
3507
|
-
if not self._open_member:
|
|
3508
|
-
return
|
|
3509
|
-
# write zero-length chunk terminator
|
|
3510
|
-
self.file.write(struct.pack(">I", 0))
|
|
3511
|
-
if self._write_header:
|
|
3512
|
-
# ULEN is at (_member_header_pos - 12), CRC at (_member_header_pos - 4)
|
|
3513
|
-
ulen_pos = self._member_header_pos - 12
|
|
3514
|
-
crc_pos = self._member_header_pos - 4
|
|
3515
|
-
cur = self.file.tell()
|
|
3516
|
-
# backfill ULEN
|
|
3517
|
-
self.file.seek(ulen_pos, os.SEEK_SET)
|
|
3518
|
-
self.file.write(struct.pack(">Q", self._ulen))
|
|
3519
|
-
# backfill CRC32
|
|
3520
|
-
self.file.seek(crc_pos, os.SEEK_SET)
|
|
3521
|
-
self.file.write(struct.pack(">I", self._crc))
|
|
3522
|
-
# restore position
|
|
3523
|
-
self.file.seek(cur, os.SEEK_SET)
|
|
3524
|
-
# reset for potential new member
|
|
3525
|
-
self._open_member = False
|
|
3526
|
-
self._crc = 0
|
|
3527
|
-
self._ulen = 0
|
|
3528
|
-
self._member_header_pos = None
|
|
3529
|
-
|
|
3530
|
-
def flush(self):
|
|
3531
|
-
if self.closed:
|
|
3532
|
-
return
|
|
3533
|
-
# finalize any open member
|
|
3534
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3535
|
-
self._flush_member_only()
|
|
3536
|
-
if hasattr(self.file, 'flush'):
|
|
3537
|
-
self.file.flush()
|
|
3538
|
-
|
|
3539
|
-
def close(self):
|
|
3540
|
-
if self.closed:
|
|
3541
|
-
return
|
|
3542
|
-
try:
|
|
3543
|
-
# Ensure a clean member terminator & header backfill if needed
|
|
3544
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3545
|
-
self._flush_member_only()
|
|
3546
|
-
if hasattr(self.file, 'flush'):
|
|
3547
|
-
try:
|
|
3548
|
-
self.file.flush()
|
|
3549
|
-
except Exception:
|
|
3550
|
-
pass
|
|
3551
|
-
finally:
|
|
3552
|
-
if self.file_path and self.file is not None:
|
|
3553
|
-
try:
|
|
3554
|
-
self.file.close()
|
|
3555
|
-
except Exception:
|
|
3556
|
-
pass
|
|
3557
|
-
# tear down read handles
|
|
3558
|
-
try:
|
|
3559
|
-
if self._text_reader is not None:
|
|
3560
|
-
self._text_reader.detach()
|
|
3561
|
-
except Exception:
|
|
3562
|
-
pass
|
|
3563
|
-
try:
|
|
3564
|
-
if self._spool is not None:
|
|
3565
|
-
self._spool.close()
|
|
3566
|
-
except Exception:
|
|
3567
|
-
pass
|
|
3568
|
-
self.closed = True
|
|
3569
|
-
|
|
3570
|
-
# ---------- Read path (spooled, multi-member, tolerant scan) ----------
|
|
3571
|
-
def _load_all_members_spooled(self):
|
|
3572
|
-
# Seek to start if possible
|
|
3573
|
-
try:
|
|
3574
|
-
self.file.seek(0)
|
|
3575
|
-
except Exception:
|
|
3576
|
-
pass
|
|
3577
|
-
|
|
3578
|
-
self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
|
|
3579
|
-
|
|
3580
|
-
def read_exact(n, abs_off_ref):
|
|
3581
|
-
"""Read exactly n bytes, updating abs_off_ref[0]."""
|
|
3582
|
-
b = b""
|
|
3583
|
-
while len(b) < n:
|
|
3584
|
-
part = self.file.read(n - len(b))
|
|
3585
|
-
if not part:
|
|
3586
|
-
break
|
|
3587
|
-
b += part
|
|
3588
|
-
abs_off_ref[0] += len(part)
|
|
3589
|
-
return b
|
|
3590
|
-
|
|
3591
|
-
CHUNK = 1 << 20
|
|
3592
|
-
abs_off = [0] # track absolute file offset
|
|
3593
|
-
scanned = 0
|
|
3594
|
-
|
|
3595
|
-
while True:
|
|
3596
|
-
# Locate MAGIC (support tolerant scan across chunk boundaries)
|
|
3597
|
-
head = read_exact(len(self.MAGIC), abs_off)
|
|
3598
|
-
if not head:
|
|
3599
|
-
break # EOF
|
|
3600
|
-
if head != self.MAGIC:
|
|
3601
|
-
# Tolerant scan: slide-by-one until found or limit exceeded
|
|
3602
|
-
buf = head
|
|
3603
|
-
while True:
|
|
3604
|
-
if self.tolerant_read and scanned < self.scan_bytes:
|
|
3605
|
-
nxt = read_exact(1, abs_off)
|
|
3606
|
-
if not nxt:
|
|
3607
|
-
# EOF without finding magic
|
|
3608
|
-
raise ValueError("Invalid LZO container: magic not found before EOF")
|
|
3609
|
-
buf = buf[1:] + nxt
|
|
3610
|
-
scanned += 1
|
|
3611
|
-
if buf == self.MAGIC:
|
|
3612
|
-
break
|
|
3613
|
-
continue
|
|
3614
|
-
raise ValueError("Invalid LZO container magic near offset {}".format(abs_off[0] - len(buf)))
|
|
3615
|
-
# found MAGIC; proceed
|
|
3616
|
-
|
|
3617
|
-
# FLAGS
|
|
3618
|
-
f_b = read_exact(1, abs_off)
|
|
3619
|
-
if len(f_b) != 1:
|
|
3620
|
-
raise ValueError("Truncated header (flags) at offset {}".format(abs_off[0]))
|
|
3621
|
-
flags = ord(f_b) if PY2 else f_b[0]
|
|
3622
|
-
|
|
3623
|
-
# Optional ULEN/CRC
|
|
3624
|
-
ulen = None
|
|
3625
|
-
expect_crc = None
|
|
3626
|
-
if flags & self.FLAG_HAS_UHDR:
|
|
3627
|
-
ulen_b = read_exact(8, abs_off)
|
|
3628
|
-
crc_b = read_exact(4, abs_off)
|
|
3629
|
-
if len(ulen_b) != 8 or len(crc_b) != 4:
|
|
3630
|
-
raise ValueError("Truncated ULEN/CRC header at offset {}".format(abs_off[0]))
|
|
3631
|
-
ulen = struct.unpack(">Q", ulen_b)[0]
|
|
3632
|
-
expect_crc = struct.unpack(">I", crc_b)[0]
|
|
3633
|
-
|
|
3634
|
-
# Chunk loop
|
|
3635
|
-
m_crc = 0
|
|
3636
|
-
m_len = 0
|
|
3637
|
-
while True:
|
|
3638
|
-
sz_b = read_exact(4, abs_off)
|
|
3639
|
-
if len(sz_b) != 4:
|
|
3640
|
-
raise ValueError("Truncated chunk size at offset {}".format(abs_off[0]))
|
|
3641
|
-
csz = struct.unpack(">I", sz_b)[0]
|
|
3642
|
-
if csz == 0:
|
|
3643
|
-
break # end of member
|
|
3644
|
-
cdata = read_exact(csz, abs_off)
|
|
3645
|
-
if len(cdata) != csz:
|
|
3646
|
-
raise ValueError("Truncated chunk payload at offset {}".format(abs_off[0]))
|
|
3647
|
-
try:
|
|
3648
|
-
raw = lzo.decompress(cdata)
|
|
3649
|
-
except Exception as e:
|
|
3650
|
-
raise ValueError("LZO decompression error at offset {}: {}".format(abs_off[0], e))
|
|
3651
|
-
self._spool.write(raw)
|
|
3652
|
-
m_len += len(raw)
|
|
3653
|
-
m_crc = _crc32u(raw, m_crc)
|
|
3654
|
-
|
|
3655
|
-
# Validate member integrity if header present
|
|
3656
|
-
if ulen is not None and m_len != ulen:
|
|
3657
|
-
raise ValueError("Member length mismatch ({} != {})".format(m_len, ulen))
|
|
3658
|
-
if expect_crc is not None and m_crc != expect_crc:
|
|
3659
|
-
raise ValueError("Member CRC32 mismatch (got 0x{:08x}, want 0x{:08x})"
|
|
3660
|
-
.format(m_crc, expect_crc))
|
|
3661
|
-
|
|
3662
|
-
# Prepare read handles
|
|
3663
|
-
try:
|
|
3664
|
-
self._spool.seek(0)
|
|
3665
|
-
except Exception:
|
|
3666
|
-
pass
|
|
3667
|
-
|
|
3668
|
-
if self._text_mode:
|
|
3669
|
-
enc = self.encoding or 'UTF-8'
|
|
3670
|
-
errs = self.errors or 'strict'
|
|
3671
|
-
# newline=None => universal newline translation; exact string if provided
|
|
3672
|
-
self._text_reader = io.TextIOWrapper(self._spool, encoding=enc, errors=errs, newline=self.newline)
|
|
3673
|
-
try:
|
|
3674
|
-
self._text_reader.seek(0)
|
|
3675
|
-
except Exception:
|
|
3676
|
-
pass
|
|
3677
|
-
|
|
3678
|
-
self._position = 0
|
|
3679
|
-
|
|
3680
|
-
# ---------- Buffered read API (delegates to spool/text wrapper) ----------
|
|
3681
|
-
def read(self, size=-1):
|
|
3682
|
-
if self.closed:
|
|
3683
|
-
raise ValueError("I/O operation on closed file")
|
|
3684
|
-
if 'r' not in self.mode:
|
|
3685
|
-
raise IOError("File not open for reading")
|
|
3686
|
-
r = self._reader()
|
|
3687
|
-
if r is None:
|
|
3688
|
-
raise IOError("Reader not initialized")
|
|
3689
|
-
out = r.read() if (size is None or size < 0) else r.read(int(size))
|
|
3690
|
-
try:
|
|
3691
|
-
self._position = r.tell()
|
|
3692
|
-
except Exception:
|
|
3693
|
-
pass
|
|
3694
|
-
return out
|
|
3695
|
-
|
|
3696
|
-
def readline(self, size=-1):
|
|
3697
|
-
if self.closed:
|
|
3698
|
-
raise ValueError("I/O operation on closed file")
|
|
3699
|
-
if 'r' not in self.mode:
|
|
3700
|
-
raise IOError("File not open for reading")
|
|
3701
|
-
r = self._reader()
|
|
3702
|
-
if r is None:
|
|
3703
|
-
raise IOError("Reader not initialized")
|
|
3704
|
-
out = r.readline() if (size is None or size < 0) else r.readline(int(size))
|
|
3705
|
-
try:
|
|
3706
|
-
self._position = r.tell()
|
|
3707
|
-
except Exception:
|
|
3708
|
-
pass
|
|
3709
|
-
if not self._text_mode and out is None:
|
|
3710
|
-
return b""
|
|
3711
|
-
if self._text_mode and out is None:
|
|
3712
|
-
return text_type("")
|
|
3713
|
-
return out
|
|
3714
|
-
|
|
3715
|
-
def __iter__(self):
|
|
3716
|
-
return self
|
|
3717
|
-
|
|
3718
|
-
def __next__(self):
|
|
3719
|
-
line = self.readline()
|
|
3720
|
-
if (self._text_mode and line == "") or (not self._text_mode and line == b""):
|
|
3721
|
-
raise StopIteration
|
|
3722
|
-
return line
|
|
3723
|
-
|
|
3724
|
-
if PY2:
|
|
3725
|
-
next = __next__
|
|
3726
|
-
|
|
3727
|
-
def seek(self, offset, whence=0):
|
|
3728
|
-
if self.closed:
|
|
3729
|
-
raise ValueError("I/O operation on closed file")
|
|
3730
|
-
if 'r' not in self.mode:
|
|
3731
|
-
raise IOError("File not open for reading")
|
|
3732
|
-
r = self._reader()
|
|
3733
|
-
if r is None:
|
|
3734
|
-
raise IOError("Reader not initialized")
|
|
3735
|
-
newpos = r.seek(int(offset), int(whence))
|
|
3736
|
-
self._position = newpos
|
|
3737
|
-
return newpos
|
|
3738
|
-
|
|
3739
|
-
def tell(self):
|
|
3740
|
-
if self._reader() is not None:
|
|
3741
|
-
try:
|
|
3742
|
-
self._position = self._reader().tell()
|
|
3743
|
-
except Exception:
|
|
3744
|
-
pass
|
|
3745
|
-
return self._position
|
|
3746
|
-
|
|
3747
|
-
# ---------- Misc ----------
|
|
3748
|
-
def fileno(self):
|
|
3749
|
-
if hasattr(self.file, 'fileno'):
|
|
3750
|
-
return self.file.fileno()
|
|
3751
|
-
raise OSError("Underlying file object does not support fileno()")
|
|
3752
|
-
|
|
3753
|
-
def isatty(self):
|
|
3754
|
-
return bool(getattr(self.file, 'isatty', lambda: False)())
|
|
3755
|
-
|
|
3756
|
-
def truncate(self, size=None):
|
|
3757
|
-
# Prevent corruption of compressed streams
|
|
3758
|
-
raise OSError("truncate() is not supported for compressed streams")
|
|
3759
|
-
|
|
3760
|
-
# ---------- Convenience constructors ----------
|
|
3761
|
-
@classmethod
|
|
3762
|
-
def open(cls, path, mode='rb', **kw):
|
|
3763
|
-
"""
|
|
3764
|
-
Mirror built-in open() but for LzopFile.
|
|
3765
|
-
Example:
|
|
3766
|
-
with LzopFile.open("data.lzo", "rt", encoding="utf-8") as f:
|
|
3767
|
-
print(f.readline())
|
|
3768
|
-
"""
|
|
3769
|
-
return cls(file_path=path, mode=mode, **kw)
|
|
3770
|
-
|
|
3771
|
-
@classmethod
|
|
3772
|
-
def from_fileobj(cls, fileobj, mode='rb', **kw):
|
|
3773
|
-
"""
|
|
3774
|
-
Wrap an existing file-like object (caller retains ownership).
|
|
3775
|
-
"""
|
|
3776
|
-
return cls(fileobj=fileobj, mode=mode, **kw)
|
|
3777
|
-
|
|
3778
|
-
@classmethod
|
|
3779
|
-
def from_bytes(cls, data, mode='rb', **kw):
|
|
3780
|
-
"""
|
|
3781
|
-
Read from an in-memory bytes buffer.
|
|
3782
|
-
Example:
|
|
3783
|
-
f = LzopFile.from_bytes(blob, mode='rt', encoding='utf-8', tolerant_read=True)
|
|
3784
|
-
text = f.read()
|
|
3785
|
-
"""
|
|
3786
|
-
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
3787
|
-
raise TypeError("from_bytes() expects a bytes-like object")
|
|
3788
|
-
bio = io.BytesIO(bytes(data) if not isinstance(data, bytes) else data)
|
|
3789
|
-
return cls(fileobj=bio, mode=mode, **kw)
|
|
3790
|
-
|
|
3791
|
-
# compatibility aliases for unwrapping utilities
|
|
3792
|
-
@property
|
|
3793
|
-
def fileobj(self):
|
|
3794
|
-
return self.file
|
|
3795
|
-
|
|
3796
|
-
@property
|
|
3797
|
-
def myfileobj(self):
|
|
3798
|
-
return self.file
|
|
3799
|
-
|
|
3800
|
-
# ---------- Top-level helpers ----------
|
|
3801
|
-
def lzop_compress_bytes(payload, level=9, text=False, **kw):
|
|
3802
|
-
"""
|
|
3803
|
-
Compress 'payload' into a single LZO member (our custom container) and return bytes.
|
|
3804
|
-
- text=True: 'payload' is text; encoding/newline/errors handled via LzopFile('wt')
|
|
3805
|
-
- text=False: 'payload' is bytes-like; written via LzopFile('wb')
|
|
3806
|
-
Kwargs forwarded: write_header (default True), newline/encoding/errors, etc.
|
|
3807
|
-
"""
|
|
3808
|
-
bio = io.BytesIO()
|
|
3809
|
-
mode = 'wt' if text else 'wb'
|
|
3810
|
-
f = LzopFile(fileobj=bio, mode=mode, level=level, **kw)
|
|
3811
|
-
try:
|
|
3812
|
-
f.write(payload)
|
|
3813
|
-
f.flush() # finalize member (writes terminator + backfills header)
|
|
3814
|
-
finally:
|
|
3815
|
-
f.close()
|
|
3816
|
-
return bio.getvalue()
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
def lzop_decompress_bytes(blob, mode='rb', tolerant_read=False, scan_bytes=(64 << 10),
|
|
3820
|
-
spool_threshold=(8 << 20), **kw):
|
|
3821
|
-
"""
|
|
3822
|
-
Decompress bytes produced by this custom container.
|
|
3823
|
-
- mode='rb' -> returns bytes; mode='rt' -> returns text (set encoding/errors/newline in kw)
|
|
3824
|
-
- tolerant_read/scan_bytes/spool_threshold forwarded to LzopFile
|
|
3825
|
-
"""
|
|
3826
|
-
if not isinstance(blob, (bytes, bytearray, memoryview)):
|
|
3827
|
-
raise TypeError("lzop_decompress_bytes() expects a bytes-like object")
|
|
3828
|
-
f = LzopFile.from_bytes(blob, mode=mode, tolerant_read=tolerant_read,
|
|
3829
|
-
scan_bytes=scan_bytes, spool_threshold=spool_threshold, **kw)
|
|
3830
|
-
try:
|
|
3831
|
-
return f.read()
|
|
3832
|
-
finally:
|
|
3833
|
-
f.close()
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
3410
|
def TarFileCheck(infile):
|
|
3837
3411
|
try:
|
|
3838
3412
|
if is_tarfile(infile):
|
|
@@ -4088,6 +3662,49 @@ _CRC_SPECS = {
|
|
|
4088
3662
|
"crc64_iso": CRCSpec(64, 0x000000000000001B, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, True, True),
|
|
4089
3663
|
}
|
|
4090
3664
|
|
|
3665
|
+
# --- helpers --------------------------------------------------------------
|
|
3666
|
+
|
|
3667
|
+
try:
|
|
3668
|
+
# Python 2 may not have algorithms_available
|
|
3669
|
+
_ALGORITHMS_AVAILABLE = set(hashlib.algorithms_available)
|
|
3670
|
+
except AttributeError:
|
|
3671
|
+
_ALGORITHMS_AVAILABLE = set(getattr(hashlib, "algorithms", []))
|
|
3672
|
+
|
|
3673
|
+
|
|
3674
|
+
def _coerce_bytes(data):
|
|
3675
|
+
"""Return `data` as a bytes object (Py2 / Py3)."""
|
|
3676
|
+
if isinstance(data, memoryview):
|
|
3677
|
+
# Py3 has .tobytes(), Py2 falls back to bytes()
|
|
3678
|
+
try:
|
|
3679
|
+
return data.tobytes()
|
|
3680
|
+
except AttributeError:
|
|
3681
|
+
return bytes(data)
|
|
3682
|
+
|
|
3683
|
+
if isinstance(data, bytearray):
|
|
3684
|
+
return bytes(data)
|
|
3685
|
+
|
|
3686
|
+
if not isinstance(data, bytes):
|
|
3687
|
+
# E.g. list of ints, unicode, etc.
|
|
3688
|
+
return bytes(bytearray(data))
|
|
3689
|
+
|
|
3690
|
+
return data
|
|
3691
|
+
|
|
3692
|
+
|
|
3693
|
+
def _bytes_to_int(b):
|
|
3694
|
+
"""Big-endian bytes -> int, Py2/3 safe."""
|
|
3695
|
+
if not isinstance(b, (bytes, bytearray)):
|
|
3696
|
+
b = _coerce_bytes(b)
|
|
3697
|
+
|
|
3698
|
+
value = 0
|
|
3699
|
+
for ch in b:
|
|
3700
|
+
if not isinstance(ch, int): # Py2: ch is a 1-char string
|
|
3701
|
+
ch = ord(ch)
|
|
3702
|
+
value = (value << 8) | ch
|
|
3703
|
+
return value
|
|
3704
|
+
|
|
3705
|
+
|
|
3706
|
+
# --- your existing CRCContext (unchanged) ---------------------------------
|
|
3707
|
+
|
|
4091
3708
|
class CRCContext(object):
|
|
4092
3709
|
__slots__ = ("spec", "table", "mask", "shift", "crc")
|
|
4093
3710
|
|
|
@@ -4132,6 +3749,82 @@ class CRCContext(object):
|
|
|
4132
3749
|
width_hex = (self.spec.width + 3) // 4
|
|
4133
3750
|
return format(self.digest_int(), "0{}x".format(width_hex)).lower()
|
|
4134
3751
|
|
|
3752
|
+
|
|
3753
|
+
# --- hashlib-backed implementation ---------------------------------------
|
|
3754
|
+
|
|
3755
|
+
class _HashlibCRCWrapper(object):
|
|
3756
|
+
"""
|
|
3757
|
+
Wrap a hashlib object to present the same interface as CRCContext
|
|
3758
|
+
(update, digest_int, hexdigest).
|
|
3759
|
+
|
|
3760
|
+
Assumes the hashlib algorithm already implements the exact CRC
|
|
3761
|
+
specification (refin/refout/xorout/etc.).
|
|
3762
|
+
"""
|
|
3763
|
+
__slots__ = ("_h", "spec", "mask", "width_hex")
|
|
3764
|
+
|
|
3765
|
+
def __init__(self, algo_name, spec):
|
|
3766
|
+
self._h = hashlib.new(algo_name)
|
|
3767
|
+
self.spec = spec
|
|
3768
|
+
self.mask = (1 << spec.width) - 1
|
|
3769
|
+
self.width_hex = (spec.width + 3) // 4
|
|
3770
|
+
|
|
3771
|
+
def update(self, data):
|
|
3772
|
+
self._h.update(_coerce_bytes(data))
|
|
3773
|
+
return self
|
|
3774
|
+
|
|
3775
|
+
def digest_int(self):
|
|
3776
|
+
# Convert final digest bytes to an integer and mask to width
|
|
3777
|
+
value = _bytes_to_int(self._h.digest())
|
|
3778
|
+
return value & self.mask
|
|
3779
|
+
|
|
3780
|
+
def hexdigest(self):
|
|
3781
|
+
h = self._h.hexdigest().lower()
|
|
3782
|
+
# Normalize to the same number of hex digits as CRCContext
|
|
3783
|
+
if len(h) < self.width_hex:
|
|
3784
|
+
h = ("0" * (self.width_hex - len(h))) + h
|
|
3785
|
+
elif len(h) > self.width_hex:
|
|
3786
|
+
h = h[-self.width_hex:]
|
|
3787
|
+
return h
|
|
3788
|
+
|
|
3789
|
+
|
|
3790
|
+
# --- public class: choose hashlib or fallback -----------------------------
|
|
3791
|
+
|
|
3792
|
+
class CRC(object):
|
|
3793
|
+
"""
|
|
3794
|
+
CRC wrapper that uses hashlib if available, otherwise falls back to
|
|
3795
|
+
the pure-Python CRCContext.
|
|
3796
|
+
|
|
3797
|
+
spec.hashlib_name (preferred) or spec.name is used as the hashlib
|
|
3798
|
+
algorithm name, e.g. 'crc32', 'crc32c', etc.
|
|
3799
|
+
"""
|
|
3800
|
+
|
|
3801
|
+
__slots__ = ("spec", "_impl")
|
|
3802
|
+
|
|
3803
|
+
def __init__(self, spec):
|
|
3804
|
+
self.spec = spec
|
|
3805
|
+
|
|
3806
|
+
algo_name = getattr(spec, "hashlib_name", None) or getattr(spec, "name", None)
|
|
3807
|
+
impl = None
|
|
3808
|
+
|
|
3809
|
+
if algo_name and algo_name in _ALGORITHMS_AVAILABLE:
|
|
3810
|
+
# Use hashlib-backed implementation
|
|
3811
|
+
impl = _HashlibCRCWrapper(algo_name, spec)
|
|
3812
|
+
else:
|
|
3813
|
+
# Fallback to your pure-Python implementation
|
|
3814
|
+
impl = CRCContext(spec)
|
|
3815
|
+
|
|
3816
|
+
self._impl = impl
|
|
3817
|
+
|
|
3818
|
+
def update(self, data):
|
|
3819
|
+
self._impl.update(data)
|
|
3820
|
+
return self
|
|
3821
|
+
|
|
3822
|
+
def digest_int(self):
|
|
3823
|
+
return self._impl.digest_int()
|
|
3824
|
+
|
|
3825
|
+
def hexdigest(self):
|
|
3826
|
+
return self._impl.hexdigest()
|
|
3827
|
+
|
|
4135
3828
|
def crc_context_from_name(name_norm):
|
|
4136
3829
|
spec = _CRC_SPECS.get(name_norm)
|
|
4137
3830
|
if spec is None:
|
|
@@ -5225,7 +4918,7 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
|
|
|
5225
4918
|
cfcontents.close()
|
|
5226
4919
|
fcontents.seek(0, 0)
|
|
5227
4920
|
fccs = GetFileChecksum(
|
|
5228
|
-
fcontents
|
|
4921
|
+
fcontents, HeaderOut[-3].lower(), False, formatspecs)
|
|
5229
4922
|
fcontentend = fp.tell()
|
|
5230
4923
|
if(re.findall("^\\+([0-9]+)", fseeknextfile)):
|
|
5231
4924
|
fseeknextasnum = int(fseeknextfile.replace("+", ""))
|
|
@@ -7083,9 +6776,9 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7083
6776
|
if(verbose):
|
|
7084
6777
|
VerbosePrintOut(fname)
|
|
7085
6778
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7086
|
-
fpremode = int(stat.S_IFDIR
|
|
6779
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7087
6780
|
else:
|
|
7088
|
-
fpremode = int(stat.S_IFREG
|
|
6781
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7089
6782
|
flinkcount = 0
|
|
7090
6783
|
ftype = 0
|
|
7091
6784
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
@@ -7114,37 +6807,42 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7114
6807
|
fbtime = format(
|
|
7115
6808
|
int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower()
|
|
7116
6809
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
7117
|
-
fwinattributes = format(int(zipinfo.external_attr), 'x').lower()
|
|
6810
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7118
6811
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7119
|
-
fmode = format(int(stat.S_IFDIR
|
|
7120
|
-
fchmode = stat.S_IMODE(int(stat.S_IFDIR
|
|
7121
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFDIR
|
|
6812
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6813
|
+
fchmode = stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))
|
|
6814
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))
|
|
7122
6815
|
else:
|
|
7123
|
-
fmode = format(int(stat.S_IFREG
|
|
7124
|
-
fchmode = stat.S_IMODE(int(stat.S_IFREG
|
|
7125
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFREG
|
|
6816
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6817
|
+
fchmode = stat.S_IMODE(int(stat.S_IFREG | 0x1b6))
|
|
6818
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFREG | 0x1b6))
|
|
7126
6819
|
elif(zipinfo.create_system == 3):
|
|
7127
|
-
fwinattributes = format(int(
|
|
7128
|
-
|
|
7129
|
-
|
|
7130
|
-
|
|
7131
|
-
|
|
7132
|
-
|
|
7133
|
-
|
|
7134
|
-
|
|
7135
|
-
|
|
7136
|
-
|
|
7137
|
-
|
|
7138
|
-
|
|
7139
|
-
|
|
6820
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
6821
|
+
fmode = format(int((zipinfo.external_attr >> 16) & 0xFFFF), 'x').lower()
|
|
6822
|
+
prefmode = int((zipinfo.external_attr >> 16) & 0xFFFF)
|
|
6823
|
+
if (prefmode == 0):
|
|
6824
|
+
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
6825
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6826
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
6827
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6828
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6829
|
+
else:
|
|
6830
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6831
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
6832
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6833
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6834
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6835
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6836
|
+
else:
|
|
6837
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7140
6838
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7141
|
-
fmode = format(int(stat.S_IFDIR
|
|
7142
|
-
prefmode = int(stat.S_IFDIR
|
|
6839
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6840
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
7143
6841
|
fchmode = stat.S_IMODE(prefmode)
|
|
7144
6842
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7145
6843
|
else:
|
|
7146
|
-
fmode = format(int(stat.S_IFREG
|
|
7147
|
-
prefmode = int(stat.S_IFREG
|
|
6844
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6845
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
7148
6846
|
fchmode = stat.S_IMODE(prefmode)
|
|
7149
6847
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7150
6848
|
fcompression = ""
|
|
@@ -7326,11 +7024,11 @@ if(rarfile_support):
|
|
|
7326
7024
|
if(is_unix and member.external_attr != 0):
|
|
7327
7025
|
fpremode = int(member.external_attr)
|
|
7328
7026
|
elif(member.is_file()):
|
|
7329
|
-
fpremode = int(stat.S_IFREG
|
|
7027
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7330
7028
|
elif(member.is_symlink()):
|
|
7331
|
-
fpremode = int(stat.S_IFLNK
|
|
7029
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
7332
7030
|
elif(member.is_dir()):
|
|
7333
|
-
fpremode = int(stat.S_IFDIR
|
|
7031
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7334
7032
|
if(is_windows and member.external_attr != 0):
|
|
7335
7033
|
fwinattributes = format(int(member.external_attr), 'x').lower()
|
|
7336
7034
|
else:
|
|
@@ -7383,23 +7081,23 @@ if(rarfile_support):
|
|
|
7383
7081
|
ftypemod = format(
|
|
7384
7082
|
int(stat.S_IFMT(member.external_attr)), 'x').lower()
|
|
7385
7083
|
elif(member.is_file()):
|
|
7386
|
-
fmode = format(int(stat.S_IFREG
|
|
7084
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7387
7085
|
fchmode = format(
|
|
7388
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7086
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7389
7087
|
ftypemod = format(
|
|
7390
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7088
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7391
7089
|
elif(member.is_symlink()):
|
|
7392
|
-
fmode = format(int(stat.S_IFLNK
|
|
7090
|
+
fmode = format(int(stat.S_IFLNK | 0x1b6), 'x').lower()
|
|
7393
7091
|
fchmode = format(
|
|
7394
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7092
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7395
7093
|
ftypemod = format(
|
|
7396
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7094
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7397
7095
|
elif(member.is_dir()):
|
|
7398
|
-
fmode = format(int(stat.S_IFDIR
|
|
7096
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7399
7097
|
fchmode = format(
|
|
7400
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7098
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7401
7099
|
ftypemod = format(
|
|
7402
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7100
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7403
7101
|
try:
|
|
7404
7102
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7405
7103
|
except AttributeError:
|
|
@@ -7503,11 +7201,11 @@ if(rarfile_support):
|
|
|
7503
7201
|
return fp
|
|
7504
7202
|
|
|
7505
7203
|
if(not py7zr_support):
|
|
7506
|
-
def
|
|
7204
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7507
7205
|
return False
|
|
7508
7206
|
|
|
7509
7207
|
if(py7zr_support):
|
|
7510
|
-
def
|
|
7208
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7511
7209
|
if(not hasattr(fp, "write")):
|
|
7512
7210
|
return False
|
|
7513
7211
|
if(verbose):
|
|
@@ -7550,9 +7248,9 @@ if(py7zr_support):
|
|
|
7550
7248
|
if(verbose):
|
|
7551
7249
|
VerbosePrintOut(fname)
|
|
7552
7250
|
if(not member.is_directory):
|
|
7553
|
-
fpremode = int(stat.S_IFREG
|
|
7251
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7554
7252
|
elif(member.is_directory):
|
|
7555
|
-
fpremode = int(stat.S_IFDIR
|
|
7253
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7556
7254
|
fwinattributes = format(int(0), 'x').lower()
|
|
7557
7255
|
fcompression = ""
|
|
7558
7256
|
fcsize = format(int(0), 'x').lower()
|
|
@@ -7576,17 +7274,17 @@ if(py7zr_support):
|
|
|
7576
7274
|
fctime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7577
7275
|
fbtime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7578
7276
|
if(member.is_directory):
|
|
7579
|
-
fmode = format(int(stat.S_IFDIR
|
|
7277
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7580
7278
|
fchmode = format(
|
|
7581
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7279
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7582
7280
|
ftypemod = format(
|
|
7583
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7281
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7584
7282
|
else:
|
|
7585
|
-
fmode = format(int(stat.S_IFREG
|
|
7283
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7586
7284
|
fchmode = format(
|
|
7587
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7285
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7588
7286
|
ftypemod = format(
|
|
7589
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7287
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7590
7288
|
try:
|
|
7591
7289
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7592
7290
|
except AttributeError:
|
|
@@ -8225,11 +7923,11 @@ def AppendFilesWithContentFromRarFileToStackedOutFile(infiles, outfile, fmttype=
|
|
|
8225
7923
|
return returnout
|
|
8226
7924
|
|
|
8227
7925
|
if(not py7zr_support):
|
|
8228
|
-
def
|
|
7926
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8229
7927
|
return False
|
|
8230
7928
|
|
|
8231
7929
|
if(py7zr_support):
|
|
8232
|
-
def
|
|
7930
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8233
7931
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
8234
7932
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
8235
7933
|
get_in_ext = os.path.splitext(outfile)
|
|
@@ -8271,7 +7969,7 @@ if(py7zr_support):
|
|
|
8271
7969
|
fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
|
|
8272
7970
|
except PermissionError:
|
|
8273
7971
|
return False
|
|
8274
|
-
|
|
7972
|
+
AppendFilesWithContentFromSevenZipFile(infiles, fp, extradata, jsondata, compression,
|
|
8275
7973
|
compresswholefile, compressionlevel, compressionuselist, checksumtype, formatspecs, verbose)
|
|
8276
7974
|
if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
|
|
8277
7975
|
fp = CompressOpenFileAlt(
|
|
@@ -8309,12 +8007,12 @@ if(py7zr_support):
|
|
|
8309
8007
|
fp.close()
|
|
8310
8008
|
return True
|
|
8311
8009
|
|
|
8312
|
-
def
|
|
8010
|
+
def AppendFilesWithContentFromSevenZipFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8313
8011
|
if not isinstance(infiles, list):
|
|
8314
8012
|
infiles = [infiles]
|
|
8315
8013
|
returnout = False
|
|
8316
8014
|
for infileslist in infiles:
|
|
8317
|
-
returnout =
|
|
8015
|
+
returnout = AppendFilesWithContentFromSevenZipFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
8318
8016
|
if(not returnout):
|
|
8319
8017
|
break
|
|
8320
8018
|
else:
|
|
@@ -9018,8 +8716,6 @@ def CheckCompressionSubType(infile, formatspecs=__file_format_multi_dict__, file
|
|
|
9018
8716
|
fp = pyzstd.zstdfile.ZstdFile(infile, mode="rb")
|
|
9019
8717
|
else:
|
|
9020
8718
|
return Flase
|
|
9021
|
-
elif((compresscheck == "lzo" or compresscheck == "lzop") and compresscheck in compressionsupport):
|
|
9022
|
-
fp = LzopFile(infile, mode="rb")
|
|
9023
8719
|
elif((compresscheck == "lzma" or compresscheck == "xz") and compresscheck in compressionsupport):
|
|
9024
8720
|
fp = lzma.open(infile, "rb")
|
|
9025
8721
|
elif(compresscheck == "zlib" and compresscheck in compressionsupport):
|
|
@@ -9142,8 +8838,6 @@ def UncompressFileAlt(fp, formatspecs=__file_format_multi_dict__, filestart=0,
|
|
|
9142
8838
|
return False
|
|
9143
8839
|
elif kind == "lz4" and "lz4" in compressionsupport:
|
|
9144
8840
|
wrapped = lz4.frame.LZ4FrameFile(src, mode="rb")
|
|
9145
|
-
elif kind in ("lzo","lzop") and (("lzo" in compressionsupport) or ("lzop" in compressionsupport)):
|
|
9146
|
-
wrapped = LzopFile(fileobj=src, mode="rb")
|
|
9147
8841
|
elif kind == "zlib" and "zlib" in compressionsupport:
|
|
9148
8842
|
wrapped = ZlibFile(fileobj=src, mode="rb")
|
|
9149
8843
|
else:
|
|
@@ -9215,8 +8909,6 @@ def UncompressFile(infile, formatspecs=__file_format_multi_dict__, mode="rb",
|
|
|
9215
8909
|
return False
|
|
9216
8910
|
elif (compresscheck == "lz4" and "lz4" in compressionsupport):
|
|
9217
8911
|
fp = lz4.frame.open(infile, mode)
|
|
9218
|
-
elif ((compresscheck == "lzo" or compresscheck == "lzop") and "lzop" in compressionsupport):
|
|
9219
|
-
fp = LzopFile(infile, mode=mode)
|
|
9220
8912
|
elif ((compresscheck == "lzma" or compresscheck == "xz") and "xz" in compressionsupport):
|
|
9221
8913
|
fp = lzma.open(infile, mode)
|
|
9222
8914
|
elif (compresscheck == "zlib" and "zlib" in compressionsupport):
|
|
@@ -9784,7 +9476,7 @@ def fast_copy(infp, outfp, bufsize=1 << 20):
|
|
|
9784
9476
|
outfp.write(data)
|
|
9785
9477
|
|
|
9786
9478
|
|
|
9787
|
-
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=
|
|
9479
|
+
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
|
|
9788
9480
|
"""
|
|
9789
9481
|
Copy a disk file into an mmap-backed destination (FileLikeAdapter).
|
|
9790
9482
|
Falls back to buffered copy if the source cannot be mmapped.
|
|
@@ -10010,9 +9702,6 @@ def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None,
|
|
|
10010
9702
|
elif (fextname == ".lz4" and "lz4" in compressionsupport):
|
|
10011
9703
|
outfp = FileLikeAdapter(lz4.frame.open(outfile, mode, compression_level=compressionlevel), mode="wb")
|
|
10012
9704
|
|
|
10013
|
-
elif (fextname == ".lzo" and "lzop" in compressionsupport):
|
|
10014
|
-
outfp = FileLikeAdapter(LzopFile(outfile, mode=mode, level=compressionlevel), mode="wb")
|
|
10015
|
-
|
|
10016
9705
|
elif (fextname == ".lzma" and "lzma" in compressionsupport):
|
|
10017
9706
|
try:
|
|
10018
9707
|
outfp = FileLikeAdapter(
|
|
@@ -10134,7 +9823,7 @@ if(not py7zr_support):
|
|
|
10134
9823
|
|
|
10135
9824
|
if(py7zr_support):
|
|
10136
9825
|
def PackArchiveFileFromSevenZipFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
10137
|
-
return
|
|
9826
|
+
return AppendFilesWithContentFromSevenZipFileToOutFile(infile, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, returnfp)
|
|
10138
9827
|
|
|
10139
9828
|
|
|
10140
9829
|
def PackArchiveFileFromInFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
@@ -10242,14 +9931,6 @@ def ArchiveFileValidate(infile, fmttype="auto", filestart=0,
|
|
|
10242
9931
|
checkcompressfile = CheckCompressionSubType(fp, formatspecs, filestart, True)
|
|
10243
9932
|
if(IsNestedDict(formatspecs) and checkcompressfile in formatspecs):
|
|
10244
9933
|
formatspecs = formatspecs[checkcompressfile]
|
|
10245
|
-
if(checkcompressfile == "tarfile" and TarFileCheck(infile)):
|
|
10246
|
-
return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10247
|
-
elif(checkcompressfile == "zipfile" and zipfile.is_zipfile(infile)):
|
|
10248
|
-
return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10249
|
-
elif(rarfile_support and checkcompressfile == "rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
|
|
10250
|
-
return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10251
|
-
elif(py7zr_support and checkcompressfile == "7zipfile" and py7zr.is_7zfile(infile)):
|
|
10252
|
-
return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10253
9934
|
elif(IsSingleDict(formatspecs) and checkcompressfile != formatspecs['format_magic']):
|
|
10254
9935
|
return False
|
|
10255
9936
|
elif(IsNestedDict(formatspecs) and checkcompressfile not in formatspecs):
|
|
@@ -10611,9 +10292,10 @@ def StackedArchiveFileValidate(infile, fmttype="auto", filestart=0, formatspecs=
|
|
|
10611
10292
|
while True:
|
|
10612
10293
|
if outstartfile >= outfsize: # stop when function signals False
|
|
10613
10294
|
break
|
|
10614
|
-
is_valid_file = ArchiveFileValidate(infile, fmttype,
|
|
10295
|
+
is_valid_file = ArchiveFileValidate(infile, fmttype, outstartfile, formatspecs, seektoend, verbose, True)
|
|
10615
10296
|
if is_valid_file is False: # stop when function signals False
|
|
10616
10297
|
outretval.append(is_valid_file)
|
|
10298
|
+
break
|
|
10617
10299
|
else:
|
|
10618
10300
|
outretval.append(True)
|
|
10619
10301
|
infile = is_valid_file
|
|
@@ -11619,35 +11301,6 @@ def MultipleArchiveFileListFiles(infile, fmttype="auto", filestart=0, seekstart=
|
|
|
11619
11301
|
return outretval
|
|
11620
11302
|
|
|
11621
11303
|
|
|
11622
|
-
def StackedArchiveFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, returnfp=False):
|
|
11623
|
-
outretval = []
|
|
11624
|
-
outstartfile = filestart
|
|
11625
|
-
outfsize = float('inf')
|
|
11626
|
-
while True:
|
|
11627
|
-
if outstartfile >= outfsize: # stop when function signals False
|
|
11628
|
-
break
|
|
11629
|
-
is_valid_file = ArchiveFileValidate(infile, fmttype, filestart, formatspecs, seektoend, verbose, True)
|
|
11630
|
-
if is_valid_file is False: # stop when function signals False
|
|
11631
|
-
outretval.append(is_valid_file)
|
|
11632
|
-
else:
|
|
11633
|
-
outretval.append(True)
|
|
11634
|
-
infile = is_valid_file
|
|
11635
|
-
outstartfile = infile.tell()
|
|
11636
|
-
try:
|
|
11637
|
-
infile.seek(0, 2)
|
|
11638
|
-
except OSError:
|
|
11639
|
-
SeekToEndOfFile(infile)
|
|
11640
|
-
except ValueError:
|
|
11641
|
-
SeekToEndOfFile(infile)
|
|
11642
|
-
outfsize = infile.tell()
|
|
11643
|
-
infile.seek(outstartfile, 0)
|
|
11644
|
-
if(returnfp):
|
|
11645
|
-
return infile
|
|
11646
|
-
else:
|
|
11647
|
-
infile.close()
|
|
11648
|
-
return outretval
|
|
11649
|
-
|
|
11650
|
-
|
|
11651
11304
|
def StackedArchiveFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, newstyle=False, returnfp=False):
|
|
11652
11305
|
outretval = []
|
|
11653
11306
|
outstartfile = filestart
|
|
@@ -11858,11 +11511,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11858
11511
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
11859
11512
|
fwinattributes = int(zipinfo.external_attr)
|
|
11860
11513
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11861
|
-
fmode = int(stat.S_IFDIR
|
|
11862
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11863
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11514
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11515
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11516
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11864
11517
|
else:
|
|
11865
|
-
fmode = int(stat.S_IFREG
|
|
11518
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11866
11519
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11867
11520
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11868
11521
|
elif(zipinfo.create_system == 3):
|
|
@@ -11878,11 +11531,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11878
11531
|
else:
|
|
11879
11532
|
fwinattributes = int(0)
|
|
11880
11533
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11881
|
-
fmode = int(stat.S_IFDIR
|
|
11882
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11883
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11534
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11535
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11536
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11884
11537
|
else:
|
|
11885
|
-
fmode = int(stat.S_IFREG
|
|
11538
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11886
11539
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11887
11540
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11888
11541
|
returnval.update({lcfi: member.filename})
|
|
@@ -11993,11 +11646,11 @@ if(rarfile_support):
|
|
|
11993
11646
|
if(is_unix and member.external_attr != 0):
|
|
11994
11647
|
fpremode = int(member.external_attr)
|
|
11995
11648
|
elif(member.is_file()):
|
|
11996
|
-
fpremode = int(stat.S_IFREG
|
|
11649
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
11997
11650
|
elif(member.is_symlink()):
|
|
11998
|
-
fpremode = int(stat.S_IFLNK
|
|
11651
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
11999
11652
|
elif(member.is_dir()):
|
|
12000
|
-
fpremode = int(stat.S_IFDIR
|
|
11653
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
12001
11654
|
if(is_windows and member.external_attr != 0):
|
|
12002
11655
|
fwinattributes = int(member.external_attr)
|
|
12003
11656
|
else:
|
|
@@ -12007,17 +11660,17 @@ if(rarfile_support):
|
|
|
12007
11660
|
fchmode = int(stat.S_IMODE(member.external_attr))
|
|
12008
11661
|
ftypemod = int(stat.S_IFMT(member.external_attr))
|
|
12009
11662
|
elif(member.is_file()):
|
|
12010
|
-
fmode = int(stat.S_IFREG
|
|
12011
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFREG
|
|
12012
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG
|
|
11663
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11664
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6)))
|
|
11665
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6)))
|
|
12013
11666
|
elif(member.is_symlink()):
|
|
12014
|
-
fmode = int(stat.S_IFLNK
|
|
12015
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
12016
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11667
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11668
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11669
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
12017
11670
|
elif(member.is_dir()):
|
|
12018
|
-
fmode = int(stat.S_IFDIR
|
|
12019
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
12020
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11671
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11672
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11673
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
12021
11674
|
returnval.update({lcfi: member.filename})
|
|
12022
11675
|
if(not verbose):
|
|
12023
11676
|
VerbosePrintOut(member.filename)
|
|
@@ -12113,18 +11766,18 @@ if(py7zr_support):
|
|
|
12113
11766
|
else:
|
|
12114
11767
|
fname = "./"+member.filename
|
|
12115
11768
|
if(not member.is_directory):
|
|
12116
|
-
fpremode = int(stat.S_IFREG
|
|
11769
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
12117
11770
|
elif(member.is_directory):
|
|
12118
|
-
fpremode = int(stat.S_IFDIR
|
|
11771
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
12119
11772
|
fwinattributes = int(0)
|
|
12120
11773
|
if(member.is_directory):
|
|
12121
|
-
fmode = int(stat.S_IFDIR
|
|
12122
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
12123
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11774
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11775
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11776
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
12124
11777
|
else:
|
|
12125
|
-
fmode = int(stat.S_IFLNK
|
|
12126
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
12127
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11778
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11779
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11780
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
12128
11781
|
returnval.update({lcfi: member.filename})
|
|
12129
11782
|
if(not verbose):
|
|
12130
11783
|
VerbosePrintOut(member.filename)
|