PyFoxFile 0.24.2__py3-none-any.whl → 0.24.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyfoxfile-0.24.2.data → pyfoxfile-0.24.6.data}/scripts/foxfile.py +32 -52
- {pyfoxfile-0.24.2.dist-info → pyfoxfile-0.24.6.dist-info}/METADATA +1 -1
- pyfoxfile-0.24.6.dist-info/RECORD +10 -0
- pyfoxfile.py +312 -658
- pyfoxfile-0.24.2.dist-info/RECORD +0 -10
- {pyfoxfile-0.24.2.data → pyfoxfile-0.24.6.data}/scripts/foxneofile.py +0 -0
- {pyfoxfile-0.24.2.data → pyfoxfile-0.24.6.data}/scripts/neofoxfile.py +0 -0
- {pyfoxfile-0.24.2.dist-info → pyfoxfile-0.24.6.dist-info}/WHEEL +0 -0
- {pyfoxfile-0.24.2.dist-info → pyfoxfile-0.24.6.dist-info}/licenses/LICENSE +0 -0
- {pyfoxfile-0.24.2.dist-info → pyfoxfile-0.24.6.dist-info}/top_level.txt +0 -0
- {pyfoxfile-0.24.2.dist-info → pyfoxfile-0.24.6.dist-info}/zip-safe +0 -0
pyfoxfile.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
|
|
15
15
|
Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
|
|
16
16
|
|
|
17
|
-
$FileInfo: pyfoxfile.py - Last Update:
|
|
17
|
+
$FileInfo: pyfoxfile.py - Last Update: 11/3/2025 Ver. 0.24.6 RC 1 - Author: cooldude2k $
|
|
18
18
|
'''
|
|
19
19
|
|
|
20
20
|
from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
|
|
@@ -33,10 +33,10 @@ import socket
|
|
|
33
33
|
import struct
|
|
34
34
|
import hashlib
|
|
35
35
|
import inspect
|
|
36
|
-
import datetime
|
|
37
36
|
import logging
|
|
38
37
|
import zipfile
|
|
39
38
|
import binascii
|
|
39
|
+
import datetime
|
|
40
40
|
import platform
|
|
41
41
|
from io import StringIO, BytesIO
|
|
42
42
|
from collections import namedtuple
|
|
@@ -413,7 +413,12 @@ def is_only_nonprintable(var):
|
|
|
413
413
|
__file_format_multi_dict__ = {}
|
|
414
414
|
__file_format_default__ = "FoxFile"
|
|
415
415
|
__include_defaults__ = True
|
|
416
|
-
__use_inmemfile__ =
|
|
416
|
+
__use_inmemfile__ = True
|
|
417
|
+
__use_spoolfile__ = False
|
|
418
|
+
__use_spooldir__ = tempfile.gettempdir()
|
|
419
|
+
BYTES_PER_MiB = 1024 * 1024
|
|
420
|
+
DEFAULT_SPOOL_MAX = 8 * BYTES_PER_MiB
|
|
421
|
+
__spoolfile_size__ = DEFAULT_SPOOL_MAX
|
|
417
422
|
__program_name__ = "Py"+__file_format_default__
|
|
418
423
|
__use_env_file__ = True
|
|
419
424
|
__use_ini_file__ = True
|
|
@@ -450,6 +455,8 @@ if __use_ini_file__ and os.path.exists(__config_file__):
|
|
|
450
455
|
__program_name__ = decode_unicode_escape(config.get('config', 'proname'))
|
|
451
456
|
__include_defaults__ = config.getboolean('config', 'includedef')
|
|
452
457
|
__use_inmemfile__ = config.getboolean('config', 'inmemfile')
|
|
458
|
+
__use_spoolfile__ = config.getboolean('config', 'usespoolfile')
|
|
459
|
+
__spoolfile_size__ = config.getint('config', 'spoolfilesize')
|
|
453
460
|
# Loop through all sections
|
|
454
461
|
for section in config.sections():
|
|
455
462
|
if section == "config":
|
|
@@ -543,6 +550,8 @@ elif __use_json_file__ and os.path.exists(__config_file__):
|
|
|
543
550
|
__program_name__ = decode_unicode_escape(_get(cfg_config, 'proname', ''))
|
|
544
551
|
__include_defaults__ = _to_bool(_get(cfg_config, 'includedef', False))
|
|
545
552
|
__use_inmemfile__ = _to_bool(_get(cfg_config, 'inmemfile', False))
|
|
553
|
+
__use_spoolfile__ = _to_bool(_get(cfg_config, 'usespoolfile', False))
|
|
554
|
+
__spoolfile_size__ = _to_int(_get(cfg_config, 'spoolfilesize', DEFAULT_SPOOL_MAX))
|
|
546
555
|
|
|
547
556
|
# --- iterate format sections (everything except "config") ---
|
|
548
557
|
required_keys = [
|
|
@@ -628,13 +637,15 @@ __use_alt_inode__ = __file_format_multi_dict__[__file_format_default__]['use_alt
|
|
|
628
637
|
__file_format_extension__ = __file_format_multi_dict__[__file_format_default__]['format_extension']
|
|
629
638
|
__file_format_dict__ = __file_format_multi_dict__[__file_format_default__]
|
|
630
639
|
__project__ = __program_name__
|
|
640
|
+
__program_alt_name__ = __program_name__
|
|
631
641
|
__project_url__ = "https://github.com/GameMaker2k/PyFoxFile"
|
|
632
|
-
|
|
633
|
-
|
|
642
|
+
__project_release_url__ = __project_url__+"/releases/latest"
|
|
643
|
+
__version_info__ = (0, 24, 6, "RC 1", 1)
|
|
644
|
+
__version_date_info__ = (2025, 11, 3, "RC 1", 1)
|
|
634
645
|
__version_date__ = str(__version_date_info__[0]) + "." + str(
|
|
635
646
|
__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2)
|
|
636
647
|
__revision__ = __version_info__[3]
|
|
637
|
-
__revision_id__ = "$Id:
|
|
648
|
+
__revision_id__ = "$Id: 05c7600a19a0abcabf9ecc0656c3a8d7fdee6a32 $"
|
|
638
649
|
if(__version_info__[4] is not None):
|
|
639
650
|
__version_date_plusrc__ = __version_date__ + \
|
|
640
651
|
"-" + str(__version_date_info__[4])
|
|
@@ -646,6 +657,78 @@ if(__version_info__[3] is not None):
|
|
|
646
657
|
if(__version_info__[3] is None):
|
|
647
658
|
__version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2])
|
|
648
659
|
|
|
660
|
+
# From: https://stackoverflow.com/a/28568003
|
|
661
|
+
# By Phaxmohdem
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def versiontuple(v):
|
|
665
|
+
filled = []
|
|
666
|
+
for point in v.split("."):
|
|
667
|
+
filled.append(point.zfill(8))
|
|
668
|
+
return tuple(filled)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def version_check(myvercheck, newvercheck):
|
|
672
|
+
vercheck = 0
|
|
673
|
+
try:
|
|
674
|
+
from packaging import version
|
|
675
|
+
vercheck = 1
|
|
676
|
+
except ImportError:
|
|
677
|
+
try:
|
|
678
|
+
from distutils.version import LooseVersion, StrictVersion
|
|
679
|
+
vercheck = 2
|
|
680
|
+
except ImportError:
|
|
681
|
+
try:
|
|
682
|
+
from pkg_resources import parse_version
|
|
683
|
+
vercheck = 3
|
|
684
|
+
except ImportError:
|
|
685
|
+
return 5
|
|
686
|
+
# print(myvercheck, newvercheck)
|
|
687
|
+
if (vercheck == 1):
|
|
688
|
+
if (version.parse(myvercheck) == version.parse(newvercheck)):
|
|
689
|
+
return 0
|
|
690
|
+
elif (version.parse(myvercheck) < version.parse(newvercheck)):
|
|
691
|
+
return 1
|
|
692
|
+
elif (version.parse(myvercheck) > version.parse(newvercheck)):
|
|
693
|
+
return 2
|
|
694
|
+
else:
|
|
695
|
+
return 3
|
|
696
|
+
elif (vercheck == 2):
|
|
697
|
+
if (StrictVersion(myvercheck) == StrictVersion(newvercheck)):
|
|
698
|
+
return 0
|
|
699
|
+
elif (StrictVersion(myvercheck) < StrictVersion(newvercheck)):
|
|
700
|
+
return 1
|
|
701
|
+
elif (StrictVersion(myvercheck) > StrictVersion(newvercheck)):
|
|
702
|
+
return 2
|
|
703
|
+
else:
|
|
704
|
+
return 3
|
|
705
|
+
elif (vercheck == 3):
|
|
706
|
+
if (parse_version(myvercheck) == parse_version(newvercheck)):
|
|
707
|
+
return 0
|
|
708
|
+
elif (parse_version(myvercheck) < parse_version(newvercheck)):
|
|
709
|
+
return 1
|
|
710
|
+
elif (parse_version(myvercheck) > parse_version(newvercheck)):
|
|
711
|
+
return 2
|
|
712
|
+
else:
|
|
713
|
+
return 3
|
|
714
|
+
else:
|
|
715
|
+
if (versiontuple(myvercheck) == versiontuple(newvercheck)):
|
|
716
|
+
return 0
|
|
717
|
+
elif (versiontuple(myvercheck) < versiontuple(newvercheck)):
|
|
718
|
+
return 1
|
|
719
|
+
elif (versiontuple(myvercheck) > versiontuple(newvercheck)):
|
|
720
|
+
return 2
|
|
721
|
+
else:
|
|
722
|
+
return 3
|
|
723
|
+
return 4
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def check_version_number(myversion=__version__, proname=__program_alt_name__, newverurl=__project_release_url__):
|
|
727
|
+
prevercheck = download_from_url(newverurl, geturls_headers, geturls_cj)
|
|
728
|
+
newvercheck = re.findall(proname + " ([0-9\\.]+)<\\/a\\>", prevercheck['Content'].decode("UTF-8"))[0]
|
|
729
|
+
myvercheck = re.findall("([0-9\\.]+)", myversion)[0]
|
|
730
|
+
return version_check(myvercheck, newvercheck)
|
|
731
|
+
|
|
649
732
|
# ===== Module-level type code table & helpers (reuse anywhere) =====
|
|
650
733
|
|
|
651
734
|
FT = {
|
|
@@ -746,6 +829,7 @@ try:
|
|
|
746
829
|
compressionsupport.append("lz4")
|
|
747
830
|
except ImportError:
|
|
748
831
|
pass
|
|
832
|
+
'''
|
|
749
833
|
try:
|
|
750
834
|
import lzo
|
|
751
835
|
compressionsupport.append("lzo")
|
|
@@ -753,6 +837,7 @@ try:
|
|
|
753
837
|
except ImportError:
|
|
754
838
|
lzo = None
|
|
755
839
|
pass
|
|
840
|
+
'''
|
|
756
841
|
try:
|
|
757
842
|
import zstandard
|
|
758
843
|
compressionsupport.append("zst")
|
|
@@ -812,11 +897,13 @@ if('lzo' in compressionsupport):
|
|
|
812
897
|
compressionlistalt.append('lzo')
|
|
813
898
|
outextlist.append('lzo')
|
|
814
899
|
outextlistwd.append('.lzo')
|
|
900
|
+
'''
|
|
815
901
|
if('lzop' in compressionsupport):
|
|
816
902
|
compressionlist.append('lzop')
|
|
817
903
|
compressionlistalt.append('lzop')
|
|
818
904
|
outextlist.append('lzop')
|
|
819
905
|
outextlistwd.append('.lzop')
|
|
906
|
+
'''
|
|
820
907
|
if('lzma' in compressionsupport):
|
|
821
908
|
compressionlist.append('lzma')
|
|
822
909
|
compressionlistalt.append('lzma')
|
|
@@ -1962,7 +2049,7 @@ def _normalize_initial_data(data, isbytes, encoding, errors=None):
|
|
|
1962
2049
|
|
|
1963
2050
|
|
|
1964
2051
|
def MkTempFile(data=None,
|
|
1965
|
-
inmem=
|
|
2052
|
+
inmem=__use_inmemfile__,
|
|
1966
2053
|
isbytes=True,
|
|
1967
2054
|
prefix="",
|
|
1968
2055
|
delete=True,
|
|
@@ -1970,9 +2057,9 @@ def MkTempFile(data=None,
|
|
|
1970
2057
|
newline=None, # text mode only; in-memory objects ignore newline semantics
|
|
1971
2058
|
dir=None,
|
|
1972
2059
|
suffix="",
|
|
1973
|
-
use_spool=
|
|
1974
|
-
spool_max=
|
|
1975
|
-
spool_dir=
|
|
2060
|
+
use_spool=__use_spoolfile__,
|
|
2061
|
+
spool_max=__spoolfile_size__,
|
|
2062
|
+
spool_dir=__use_spooldir__:
|
|
1976
2063
|
"""
|
|
1977
2064
|
Return a file-like handle with consistent behavior on Py2.7 and Py3.x.
|
|
1978
2065
|
|
|
@@ -2415,7 +2502,7 @@ class ZlibFile(object):
|
|
|
2415
2502
|
|
|
2416
2503
|
def __init__(self, file_path=None, fileobj=None, mode='rb', level=6, wbits=15,
|
|
2417
2504
|
encoding=None, errors=None, newline=None,
|
|
2418
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2505
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2419
2506
|
|
|
2420
2507
|
if file_path is None and fileobj is None:
|
|
2421
2508
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -2902,7 +2989,7 @@ class GzipFile(object):
|
|
|
2902
2989
|
|
|
2903
2990
|
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
2904
2991
|
level=6, encoding=None, errors=None, newline=None,
|
|
2905
|
-
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=
|
|
2992
|
+
tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
|
|
2906
2993
|
|
|
2907
2994
|
if file_path is None and fileobj is None:
|
|
2908
2995
|
raise ValueError("Either file_path or fileobj must be provided")
|
|
@@ -3326,519 +3413,6 @@ def gzip_decompress_bytes_all_members(blob):
|
|
|
3326
3413
|
"""
|
|
3327
3414
|
return _gzip_decompress_multimember(bytes(blob))
|
|
3328
3415
|
|
|
3329
|
-
|
|
3330
|
-
# ---------- Simple LZO container (NOT real .lzop) ----------
|
|
3331
|
-
# File layout (concatenated members allowed):
|
|
3332
|
-
# [MAGIC 8B] [FLAGS 1B] [ULEN 8B] [CRC32 4B] [CCHUNK...] | repeat...
|
|
3333
|
-
# where:
|
|
3334
|
-
# MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3335
|
-
# FLAGS = bit0: 1 => member has ULEN+CRC, 0 => no header (legacy)
|
|
3336
|
-
# ULEN = uncompressed length (u64 BE)
|
|
3337
|
-
# CRC32 = CRC32 of uncompressed data (u32 BE)
|
|
3338
|
-
# CCHUNK = one or more compressed chunks:
|
|
3339
|
-
# [u32BE chunk_size][chunk_data] ... then a zero-size u32 terminator
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
class LzopFile(object):
|
|
3343
|
-
MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
|
|
3344
|
-
FLAG_HAS_UHDR = 0x01
|
|
3345
|
-
RAW_CHUNK = 256 * 1024 # 256 KiB per raw (pre-compress) chunk
|
|
3346
|
-
|
|
3347
|
-
def __init__(self, file_path=None, fileobj=None, mode='rb',
|
|
3348
|
-
level=9, encoding=None, errors=None, newline=None,
|
|
3349
|
-
write_header=True,
|
|
3350
|
-
tolerant_read=False, scan_bytes=(64 << 10),
|
|
3351
|
-
spool_threshold=(8 << 20)):
|
|
3352
|
-
"""
|
|
3353
|
-
Custom LZO file (NOT the lzop(1) format).
|
|
3354
|
-
- streaming write/read, supports concatenated members
|
|
3355
|
-
- optional per-member header (uncompressed length + CRC32)
|
|
3356
|
-
- spooled reads to limit RAM, strict text mode with newline control
|
|
3357
|
-
- tolerant_read: scan forward (up to scan_bytes) to first MAGIC
|
|
3358
|
-
|
|
3359
|
-
:param write_header: if True, include ULEN+CRC32 per member
|
|
3360
|
-
:param tolerant_read: skip leading junk up to scan_bytes to find MAGIC
|
|
3361
|
-
:param scan_bytes: max bytes to scan when tolerant_read=True
|
|
3362
|
-
:param spool_threshold: SpooledTemporaryFile RAM threshold before spill
|
|
3363
|
-
"""
|
|
3364
|
-
if lzo is None:
|
|
3365
|
-
raise ImportError("python-lzo is required for LzopFile")
|
|
3366
|
-
|
|
3367
|
-
if file_path is None and fileobj is None:
|
|
3368
|
-
raise ValueError("Either file_path or fileobj must be provided")
|
|
3369
|
-
if file_path is not None and fileobj is not None:
|
|
3370
|
-
raise ValueError("Only one of file_path or fileobj should be provided")
|
|
3371
|
-
|
|
3372
|
-
if 'b' not in mode and 't' not in mode:
|
|
3373
|
-
mode += 'b'
|
|
3374
|
-
if 'x' in mode and PY2:
|
|
3375
|
-
raise ValueError("Exclusive creation mode 'x' is not supported on Python 2")
|
|
3376
|
-
|
|
3377
|
-
self.file_path = file_path
|
|
3378
|
-
self.file = fileobj
|
|
3379
|
-
self.mode = mode
|
|
3380
|
-
self.level = int(level) # effective: 1 or 9 (clamped)
|
|
3381
|
-
self.encoding = encoding
|
|
3382
|
-
self.errors = errors
|
|
3383
|
-
self.newline = newline
|
|
3384
|
-
self._text_mode = ('t' in mode)
|
|
3385
|
-
|
|
3386
|
-
self._write_header = bool(write_header)
|
|
3387
|
-
|
|
3388
|
-
# Config (read path)
|
|
3389
|
-
self.tolerant_read = bool(tolerant_read)
|
|
3390
|
-
self.scan_bytes = int(scan_bytes)
|
|
3391
|
-
self.spool_threshold = int(spool_threshold)
|
|
3392
|
-
|
|
3393
|
-
# Write state
|
|
3394
|
-
self._crc = 0
|
|
3395
|
-
self._ulen = 0
|
|
3396
|
-
self._open_member = False
|
|
3397
|
-
self._member_header_pos = None # position *after* ULEN+CRC placeholders
|
|
3398
|
-
|
|
3399
|
-
# Read state
|
|
3400
|
-
self._spool = None
|
|
3401
|
-
self._text_reader = None
|
|
3402
|
-
self._position = 0
|
|
3403
|
-
self.closed = False
|
|
3404
|
-
|
|
3405
|
-
internal_mode = mode.replace('t', 'b')
|
|
3406
|
-
if self.file is None:
|
|
3407
|
-
if 'x' in internal_mode and os.path.exists(file_path):
|
|
3408
|
-
raise IOError("File exists: '{}'".format(file_path))
|
|
3409
|
-
self.file = open(file_path, internal_mode)
|
|
3410
|
-
else:
|
|
3411
|
-
if 'r' in internal_mode and not hasattr(self.file, 'read'):
|
|
3412
|
-
raise ValueError("fileobj must support read() in read mode")
|
|
3413
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')) and not hasattr(self.file, 'write'):
|
|
3414
|
-
raise ValueError("fileobj must support write() in write/append mode")
|
|
3415
|
-
|
|
3416
|
-
self._fp = self.file
|
|
3417
|
-
if any(ch in internal_mode for ch in ('w', 'a', 'x')):
|
|
3418
|
-
# Start a new member at EOF for append
|
|
3419
|
-
if 'a' in internal_mode:
|
|
3420
|
-
try:
|
|
3421
|
-
self.file.seek(0, os.SEEK_END)
|
|
3422
|
-
except Exception:
|
|
3423
|
-
pass
|
|
3424
|
-
# Defer writing header until first write so empty files don’t get empty members
|
|
3425
|
-
elif 'r' in internal_mode:
|
|
3426
|
-
self._load_all_members_spooled()
|
|
3427
|
-
else:
|
|
3428
|
-
raise ValueError("Unsupported mode: {}".format(mode))
|
|
3429
|
-
|
|
3430
|
-
# ---------- helpers ----------
|
|
3431
|
-
@property
|
|
3432
|
-
def name(self):
|
|
3433
|
-
return self.file_path
|
|
3434
|
-
|
|
3435
|
-
def readable(self):
|
|
3436
|
-
return 'r' in self.mode
|
|
3437
|
-
|
|
3438
|
-
def writable(self):
|
|
3439
|
-
return any(ch in self.mode for ch in ('w', 'a', 'x'))
|
|
3440
|
-
|
|
3441
|
-
def seekable(self):
|
|
3442
|
-
return True if self._spool is not None else bool(getattr(self.file, 'seek', None))
|
|
3443
|
-
|
|
3444
|
-
def _normalize_newlines_for_write(self, s):
|
|
3445
|
-
nl = self.newline if self.newline is not None else "\n"
|
|
3446
|
-
return s.replace("\r\n", "\n").replace("\r", "\n").replace("\n", nl)
|
|
3447
|
-
|
|
3448
|
-
def _reader(self):
|
|
3449
|
-
return self._text_reader if self._text_mode else self._spool
|
|
3450
|
-
|
|
3451
|
-
# ---------- Write path ----------
|
|
3452
|
-
def _ensure_member_header(self):
|
|
3453
|
-
if self._open_member:
|
|
3454
|
-
return
|
|
3455
|
-
flags = self.FLAG_HAS_UHDR if self._write_header else 0
|
|
3456
|
-
self.file.write(self.MAGIC)
|
|
3457
|
-
self.file.write(struct.pack(">B", flags))
|
|
3458
|
-
if self._write_header:
|
|
3459
|
-
# placeholders for ULEN+CRC; we’ll backfill on finalize
|
|
3460
|
-
self.file.write(struct.pack(">Q", 0))
|
|
3461
|
-
self.file.write(struct.pack(">I", 0))
|
|
3462
|
-
# position *after* ULEN+CRC placeholders (or after FLAGS if no header)
|
|
3463
|
-
self._member_header_pos = self.file.tell()
|
|
3464
|
-
self._open_member = True
|
|
3465
|
-
# reset member stats
|
|
3466
|
-
self._crc = 0
|
|
3467
|
-
self._ulen = 0
|
|
3468
|
-
|
|
3469
|
-
def write(self, data):
|
|
3470
|
-
if 'r' in self.mode:
|
|
3471
|
-
raise IOError("File not open for writing")
|
|
3472
|
-
|
|
3473
|
-
if self._text_mode:
|
|
3474
|
-
enc = self.encoding or 'UTF-8'
|
|
3475
|
-
errs = self.errors or 'strict'
|
|
3476
|
-
if not isinstance(data, text_type):
|
|
3477
|
-
raise TypeError("write() expects text (unicode/str) in text mode")
|
|
3478
|
-
data = self._normalize_newlines_for_write(data).encode(enc, errs)
|
|
3479
|
-
else:
|
|
3480
|
-
if not isinstance(data, binary_types):
|
|
3481
|
-
raise TypeError("write() expects bytes-like in binary mode")
|
|
3482
|
-
|
|
3483
|
-
# Normalize Py3 memoryview / Py2 bytearray
|
|
3484
|
-
if (not PY2) and isinstance(data, memoryview):
|
|
3485
|
-
data = data.tobytes()
|
|
3486
|
-
elif PY2 and isinstance(data, bytearray):
|
|
3487
|
-
data = bytes(data)
|
|
3488
|
-
|
|
3489
|
-
if not data:
|
|
3490
|
-
return 0
|
|
3491
|
-
|
|
3492
|
-
# Begin member and write header on first write
|
|
3493
|
-
self._ensure_member_header()
|
|
3494
|
-
|
|
3495
|
-
# Update integrity stats
|
|
3496
|
-
self._crc = _crc32u(data, self._crc)
|
|
3497
|
-
self._ulen += len(data)
|
|
3498
|
-
|
|
3499
|
-
# Stream in RAW_CHUNK-sized pieces. Each piece becomes one compressed chunk record.
|
|
3500
|
-
mv = memoryview(data)
|
|
3501
|
-
# clamp level to {1, 9}
|
|
3502
|
-
lvl = 9 if self.level >= 9 else 1
|
|
3503
|
-
for off in range(0, len(data), self.RAW_CHUNK):
|
|
3504
|
-
raw = mv[off:off + self.RAW_CHUNK].tobytes()
|
|
3505
|
-
c = lzo.compress(raw, lvl)
|
|
3506
|
-
self.file.write(struct.pack(">I", len(c)))
|
|
3507
|
-
self.file.write(c)
|
|
3508
|
-
|
|
3509
|
-
return len(data)
|
|
3510
|
-
|
|
3511
|
-
def _flush_member_only(self):
|
|
3512
|
-
"""Finalize the current member: write terminator and backfill header."""
|
|
3513
|
-
if not self._open_member:
|
|
3514
|
-
return
|
|
3515
|
-
# write zero-length chunk terminator
|
|
3516
|
-
self.file.write(struct.pack(">I", 0))
|
|
3517
|
-
if self._write_header:
|
|
3518
|
-
# ULEN is at (_member_header_pos - 12), CRC at (_member_header_pos - 4)
|
|
3519
|
-
ulen_pos = self._member_header_pos - 12
|
|
3520
|
-
crc_pos = self._member_header_pos - 4
|
|
3521
|
-
cur = self.file.tell()
|
|
3522
|
-
# backfill ULEN
|
|
3523
|
-
self.file.seek(ulen_pos, os.SEEK_SET)
|
|
3524
|
-
self.file.write(struct.pack(">Q", self._ulen))
|
|
3525
|
-
# backfill CRC32
|
|
3526
|
-
self.file.seek(crc_pos, os.SEEK_SET)
|
|
3527
|
-
self.file.write(struct.pack(">I", self._crc))
|
|
3528
|
-
# restore position
|
|
3529
|
-
self.file.seek(cur, os.SEEK_SET)
|
|
3530
|
-
# reset for potential new member
|
|
3531
|
-
self._open_member = False
|
|
3532
|
-
self._crc = 0
|
|
3533
|
-
self._ulen = 0
|
|
3534
|
-
self._member_header_pos = None
|
|
3535
|
-
|
|
3536
|
-
def flush(self):
|
|
3537
|
-
if self.closed:
|
|
3538
|
-
return
|
|
3539
|
-
# finalize any open member
|
|
3540
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3541
|
-
self._flush_member_only()
|
|
3542
|
-
if hasattr(self.file, 'flush'):
|
|
3543
|
-
self.file.flush()
|
|
3544
|
-
|
|
3545
|
-
def close(self):
|
|
3546
|
-
if self.closed:
|
|
3547
|
-
return
|
|
3548
|
-
try:
|
|
3549
|
-
# Ensure a clean member terminator & header backfill if needed
|
|
3550
|
-
if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
|
|
3551
|
-
self._flush_member_only()
|
|
3552
|
-
if hasattr(self.file, 'flush'):
|
|
3553
|
-
try:
|
|
3554
|
-
self.file.flush()
|
|
3555
|
-
except Exception:
|
|
3556
|
-
pass
|
|
3557
|
-
finally:
|
|
3558
|
-
if self.file_path and self.file is not None:
|
|
3559
|
-
try:
|
|
3560
|
-
self.file.close()
|
|
3561
|
-
except Exception:
|
|
3562
|
-
pass
|
|
3563
|
-
# tear down read handles
|
|
3564
|
-
try:
|
|
3565
|
-
if self._text_reader is not None:
|
|
3566
|
-
self._text_reader.detach()
|
|
3567
|
-
except Exception:
|
|
3568
|
-
pass
|
|
3569
|
-
try:
|
|
3570
|
-
if self._spool is not None:
|
|
3571
|
-
self._spool.close()
|
|
3572
|
-
except Exception:
|
|
3573
|
-
pass
|
|
3574
|
-
self.closed = True
|
|
3575
|
-
|
|
3576
|
-
# ---------- Read path (spooled, multi-member, tolerant scan) ----------
|
|
3577
|
-
def _load_all_members_spooled(self):
|
|
3578
|
-
# Seek to start if possible
|
|
3579
|
-
try:
|
|
3580
|
-
self.file.seek(0)
|
|
3581
|
-
except Exception:
|
|
3582
|
-
pass
|
|
3583
|
-
|
|
3584
|
-
self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
|
|
3585
|
-
|
|
3586
|
-
def read_exact(n, abs_off_ref):
|
|
3587
|
-
"""Read exactly n bytes, updating abs_off_ref[0]."""
|
|
3588
|
-
b = b""
|
|
3589
|
-
while len(b) < n:
|
|
3590
|
-
part = self.file.read(n - len(b))
|
|
3591
|
-
if not part:
|
|
3592
|
-
break
|
|
3593
|
-
b += part
|
|
3594
|
-
abs_off_ref[0] += len(part)
|
|
3595
|
-
return b
|
|
3596
|
-
|
|
3597
|
-
CHUNK = 1 << 20
|
|
3598
|
-
abs_off = [0] # track absolute file offset
|
|
3599
|
-
scanned = 0
|
|
3600
|
-
|
|
3601
|
-
while True:
|
|
3602
|
-
# Locate MAGIC (support tolerant scan across chunk boundaries)
|
|
3603
|
-
head = read_exact(len(self.MAGIC), abs_off)
|
|
3604
|
-
if not head:
|
|
3605
|
-
break # EOF
|
|
3606
|
-
if head != self.MAGIC:
|
|
3607
|
-
# Tolerant scan: slide-by-one until found or limit exceeded
|
|
3608
|
-
buf = head
|
|
3609
|
-
while True:
|
|
3610
|
-
if self.tolerant_read and scanned < self.scan_bytes:
|
|
3611
|
-
nxt = read_exact(1, abs_off)
|
|
3612
|
-
if not nxt:
|
|
3613
|
-
# EOF without finding magic
|
|
3614
|
-
raise ValueError("Invalid LZO container: magic not found before EOF")
|
|
3615
|
-
buf = buf[1:] + nxt
|
|
3616
|
-
scanned += 1
|
|
3617
|
-
if buf == self.MAGIC:
|
|
3618
|
-
break
|
|
3619
|
-
continue
|
|
3620
|
-
raise ValueError("Invalid LZO container magic near offset {}".format(abs_off[0] - len(buf)))
|
|
3621
|
-
# found MAGIC; proceed
|
|
3622
|
-
|
|
3623
|
-
# FLAGS
|
|
3624
|
-
f_b = read_exact(1, abs_off)
|
|
3625
|
-
if len(f_b) != 1:
|
|
3626
|
-
raise ValueError("Truncated header (flags) at offset {}".format(abs_off[0]))
|
|
3627
|
-
flags = ord(f_b) if PY2 else f_b[0]
|
|
3628
|
-
|
|
3629
|
-
# Optional ULEN/CRC
|
|
3630
|
-
ulen = None
|
|
3631
|
-
expect_crc = None
|
|
3632
|
-
if flags & self.FLAG_HAS_UHDR:
|
|
3633
|
-
ulen_b = read_exact(8, abs_off)
|
|
3634
|
-
crc_b = read_exact(4, abs_off)
|
|
3635
|
-
if len(ulen_b) != 8 or len(crc_b) != 4:
|
|
3636
|
-
raise ValueError("Truncated ULEN/CRC header at offset {}".format(abs_off[0]))
|
|
3637
|
-
ulen = struct.unpack(">Q", ulen_b)[0]
|
|
3638
|
-
expect_crc = struct.unpack(">I", crc_b)[0]
|
|
3639
|
-
|
|
3640
|
-
# Chunk loop
|
|
3641
|
-
m_crc = 0
|
|
3642
|
-
m_len = 0
|
|
3643
|
-
while True:
|
|
3644
|
-
sz_b = read_exact(4, abs_off)
|
|
3645
|
-
if len(sz_b) != 4:
|
|
3646
|
-
raise ValueError("Truncated chunk size at offset {}".format(abs_off[0]))
|
|
3647
|
-
csz = struct.unpack(">I", sz_b)[0]
|
|
3648
|
-
if csz == 0:
|
|
3649
|
-
break # end of member
|
|
3650
|
-
cdata = read_exact(csz, abs_off)
|
|
3651
|
-
if len(cdata) != csz:
|
|
3652
|
-
raise ValueError("Truncated chunk payload at offset {}".format(abs_off[0]))
|
|
3653
|
-
try:
|
|
3654
|
-
raw = lzo.decompress(cdata)
|
|
3655
|
-
except Exception as e:
|
|
3656
|
-
raise ValueError("LZO decompression error at offset {}: {}".format(abs_off[0], e))
|
|
3657
|
-
self._spool.write(raw)
|
|
3658
|
-
m_len += len(raw)
|
|
3659
|
-
m_crc = _crc32u(raw, m_crc)
|
|
3660
|
-
|
|
3661
|
-
# Validate member integrity if header present
|
|
3662
|
-
if ulen is not None and m_len != ulen:
|
|
3663
|
-
raise ValueError("Member length mismatch ({} != {})".format(m_len, ulen))
|
|
3664
|
-
if expect_crc is not None and m_crc != expect_crc:
|
|
3665
|
-
raise ValueError("Member CRC32 mismatch (got 0x{:08x}, want 0x{:08x})"
|
|
3666
|
-
.format(m_crc, expect_crc))
|
|
3667
|
-
|
|
3668
|
-
# Prepare read handles
|
|
3669
|
-
try:
|
|
3670
|
-
self._spool.seek(0)
|
|
3671
|
-
except Exception:
|
|
3672
|
-
pass
|
|
3673
|
-
|
|
3674
|
-
if self._text_mode:
|
|
3675
|
-
enc = self.encoding or 'UTF-8'
|
|
3676
|
-
errs = self.errors or 'strict'
|
|
3677
|
-
# newline=None => universal newline translation; exact string if provided
|
|
3678
|
-
self._text_reader = io.TextIOWrapper(self._spool, encoding=enc, errors=errs, newline=self.newline)
|
|
3679
|
-
try:
|
|
3680
|
-
self._text_reader.seek(0)
|
|
3681
|
-
except Exception:
|
|
3682
|
-
pass
|
|
3683
|
-
|
|
3684
|
-
self._position = 0
|
|
3685
|
-
|
|
3686
|
-
# ---------- Buffered read API (delegates to spool/text wrapper) ----------
|
|
3687
|
-
def read(self, size=-1):
|
|
3688
|
-
if self.closed:
|
|
3689
|
-
raise ValueError("I/O operation on closed file")
|
|
3690
|
-
if 'r' not in self.mode:
|
|
3691
|
-
raise IOError("File not open for reading")
|
|
3692
|
-
r = self._reader()
|
|
3693
|
-
if r is None:
|
|
3694
|
-
raise IOError("Reader not initialized")
|
|
3695
|
-
out = r.read() if (size is None or size < 0) else r.read(int(size))
|
|
3696
|
-
try:
|
|
3697
|
-
self._position = r.tell()
|
|
3698
|
-
except Exception:
|
|
3699
|
-
pass
|
|
3700
|
-
return out
|
|
3701
|
-
|
|
3702
|
-
def readline(self, size=-1):
|
|
3703
|
-
if self.closed:
|
|
3704
|
-
raise ValueError("I/O operation on closed file")
|
|
3705
|
-
if 'r' not in self.mode:
|
|
3706
|
-
raise IOError("File not open for reading")
|
|
3707
|
-
r = self._reader()
|
|
3708
|
-
if r is None:
|
|
3709
|
-
raise IOError("Reader not initialized")
|
|
3710
|
-
out = r.readline() if (size is None or size < 0) else r.readline(int(size))
|
|
3711
|
-
try:
|
|
3712
|
-
self._position = r.tell()
|
|
3713
|
-
except Exception:
|
|
3714
|
-
pass
|
|
3715
|
-
if not self._text_mode and out is None:
|
|
3716
|
-
return b""
|
|
3717
|
-
if self._text_mode and out is None:
|
|
3718
|
-
return text_type("")
|
|
3719
|
-
return out
|
|
3720
|
-
|
|
3721
|
-
def __iter__(self):
|
|
3722
|
-
return self
|
|
3723
|
-
|
|
3724
|
-
def __next__(self):
|
|
3725
|
-
line = self.readline()
|
|
3726
|
-
if (self._text_mode and line == "") or (not self._text_mode and line == b""):
|
|
3727
|
-
raise StopIteration
|
|
3728
|
-
return line
|
|
3729
|
-
|
|
3730
|
-
if PY2:
|
|
3731
|
-
next = __next__
|
|
3732
|
-
|
|
3733
|
-
def seek(self, offset, whence=0):
|
|
3734
|
-
if self.closed:
|
|
3735
|
-
raise ValueError("I/O operation on closed file")
|
|
3736
|
-
if 'r' not in self.mode:
|
|
3737
|
-
raise IOError("File not open for reading")
|
|
3738
|
-
r = self._reader()
|
|
3739
|
-
if r is None:
|
|
3740
|
-
raise IOError("Reader not initialized")
|
|
3741
|
-
newpos = r.seek(int(offset), int(whence))
|
|
3742
|
-
self._position = newpos
|
|
3743
|
-
return newpos
|
|
3744
|
-
|
|
3745
|
-
def tell(self):
|
|
3746
|
-
if self._reader() is not None:
|
|
3747
|
-
try:
|
|
3748
|
-
self._position = self._reader().tell()
|
|
3749
|
-
except Exception:
|
|
3750
|
-
pass
|
|
3751
|
-
return self._position
|
|
3752
|
-
|
|
3753
|
-
# ---------- Misc ----------
|
|
3754
|
-
def fileno(self):
|
|
3755
|
-
if hasattr(self.file, 'fileno'):
|
|
3756
|
-
return self.file.fileno()
|
|
3757
|
-
raise OSError("Underlying file object does not support fileno()")
|
|
3758
|
-
|
|
3759
|
-
def isatty(self):
|
|
3760
|
-
return bool(getattr(self.file, 'isatty', lambda: False)())
|
|
3761
|
-
|
|
3762
|
-
def truncate(self, size=None):
|
|
3763
|
-
# Prevent corruption of compressed streams
|
|
3764
|
-
raise OSError("truncate() is not supported for compressed streams")
|
|
3765
|
-
|
|
3766
|
-
# ---------- Convenience constructors ----------
|
|
3767
|
-
@classmethod
|
|
3768
|
-
def open(cls, path, mode='rb', **kw):
|
|
3769
|
-
"""
|
|
3770
|
-
Mirror built-in open() but for LzopFile.
|
|
3771
|
-
Example:
|
|
3772
|
-
with LzopFile.open("data.lzo", "rt", encoding="utf-8") as f:
|
|
3773
|
-
print(f.readline())
|
|
3774
|
-
"""
|
|
3775
|
-
return cls(file_path=path, mode=mode, **kw)
|
|
3776
|
-
|
|
3777
|
-
@classmethod
|
|
3778
|
-
def from_fileobj(cls, fileobj, mode='rb', **kw):
|
|
3779
|
-
"""
|
|
3780
|
-
Wrap an existing file-like object (caller retains ownership).
|
|
3781
|
-
"""
|
|
3782
|
-
return cls(fileobj=fileobj, mode=mode, **kw)
|
|
3783
|
-
|
|
3784
|
-
@classmethod
|
|
3785
|
-
def from_bytes(cls, data, mode='rb', **kw):
|
|
3786
|
-
"""
|
|
3787
|
-
Read from an in-memory bytes buffer.
|
|
3788
|
-
Example:
|
|
3789
|
-
f = LzopFile.from_bytes(blob, mode='rt', encoding='utf-8', tolerant_read=True)
|
|
3790
|
-
text = f.read()
|
|
3791
|
-
"""
|
|
3792
|
-
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
3793
|
-
raise TypeError("from_bytes() expects a bytes-like object")
|
|
3794
|
-
bio = io.BytesIO(bytes(data) if not isinstance(data, bytes) else data)
|
|
3795
|
-
return cls(fileobj=bio, mode=mode, **kw)
|
|
3796
|
-
|
|
3797
|
-
# compatibility aliases for unwrapping utilities
|
|
3798
|
-
@property
|
|
3799
|
-
def fileobj(self):
|
|
3800
|
-
return self.file
|
|
3801
|
-
|
|
3802
|
-
@property
|
|
3803
|
-
def myfileobj(self):
|
|
3804
|
-
return self.file
|
|
3805
|
-
|
|
3806
|
-
# ---------- Top-level helpers ----------
|
|
3807
|
-
def lzop_compress_bytes(payload, level=9, text=False, **kw):
|
|
3808
|
-
"""
|
|
3809
|
-
Compress 'payload' into a single LZO member (our custom container) and return bytes.
|
|
3810
|
-
- text=True: 'payload' is text; encoding/newline/errors handled via LzopFile('wt')
|
|
3811
|
-
- text=False: 'payload' is bytes-like; written via LzopFile('wb')
|
|
3812
|
-
Kwargs forwarded: write_header (default True), newline/encoding/errors, etc.
|
|
3813
|
-
"""
|
|
3814
|
-
bio = io.BytesIO()
|
|
3815
|
-
mode = 'wt' if text else 'wb'
|
|
3816
|
-
f = LzopFile(fileobj=bio, mode=mode, level=level, **kw)
|
|
3817
|
-
try:
|
|
3818
|
-
f.write(payload)
|
|
3819
|
-
f.flush() # finalize member (writes terminator + backfills header)
|
|
3820
|
-
finally:
|
|
3821
|
-
f.close()
|
|
3822
|
-
return bio.getvalue()
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
def lzop_decompress_bytes(blob, mode='rb', tolerant_read=False, scan_bytes=(64 << 10),
|
|
3826
|
-
spool_threshold=(8 << 20), **kw):
|
|
3827
|
-
"""
|
|
3828
|
-
Decompress bytes produced by this custom container.
|
|
3829
|
-
- mode='rb' -> returns bytes; mode='rt' -> returns text (set encoding/errors/newline in kw)
|
|
3830
|
-
- tolerant_read/scan_bytes/spool_threshold forwarded to LzopFile
|
|
3831
|
-
"""
|
|
3832
|
-
if not isinstance(blob, (bytes, bytearray, memoryview)):
|
|
3833
|
-
raise TypeError("lzop_decompress_bytes() expects a bytes-like object")
|
|
3834
|
-
f = LzopFile.from_bytes(blob, mode=mode, tolerant_read=tolerant_read,
|
|
3835
|
-
scan_bytes=scan_bytes, spool_threshold=spool_threshold, **kw)
|
|
3836
|
-
try:
|
|
3837
|
-
return f.read()
|
|
3838
|
-
finally:
|
|
3839
|
-
f.close()
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
3416
|
def TarFileCheck(infile):
|
|
3843
3417
|
try:
|
|
3844
3418
|
if is_tarfile(infile):
|
|
@@ -4321,6 +3895,49 @@ _CRC_SPECS = {
|
|
|
4321
3895
|
"crc64_iso": CRCSpec(64, 0x000000000000001B, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, True, True),
|
|
4322
3896
|
}
|
|
4323
3897
|
|
|
3898
|
+
# --- helpers --------------------------------------------------------------
|
|
3899
|
+
|
|
3900
|
+
try:
|
|
3901
|
+
# Python 2 may not have algorithms_available
|
|
3902
|
+
_ALGORITHMS_AVAILABLE = set(hashlib.algorithms_available)
|
|
3903
|
+
except AttributeError:
|
|
3904
|
+
_ALGORITHMS_AVAILABLE = set(getattr(hashlib, "algorithms", []))
|
|
3905
|
+
|
|
3906
|
+
|
|
3907
|
+
def _coerce_bytes(data):
|
|
3908
|
+
"""Return `data` as a bytes object (Py2 / Py3)."""
|
|
3909
|
+
if isinstance(data, memoryview):
|
|
3910
|
+
# Py3 has .tobytes(), Py2 falls back to bytes()
|
|
3911
|
+
try:
|
|
3912
|
+
return data.tobytes()
|
|
3913
|
+
except AttributeError:
|
|
3914
|
+
return bytes(data)
|
|
3915
|
+
|
|
3916
|
+
if isinstance(data, bytearray):
|
|
3917
|
+
return bytes(data)
|
|
3918
|
+
|
|
3919
|
+
if not isinstance(data, bytes):
|
|
3920
|
+
# E.g. list of ints, unicode, etc.
|
|
3921
|
+
return bytes(bytearray(data))
|
|
3922
|
+
|
|
3923
|
+
return data
|
|
3924
|
+
|
|
3925
|
+
|
|
3926
|
+
def _bytes_to_int(b):
|
|
3927
|
+
"""Big-endian bytes -> int, Py2/3 safe."""
|
|
3928
|
+
if not isinstance(b, (bytes, bytearray)):
|
|
3929
|
+
b = _coerce_bytes(b)
|
|
3930
|
+
|
|
3931
|
+
value = 0
|
|
3932
|
+
for ch in b:
|
|
3933
|
+
if not isinstance(ch, int): # Py2: ch is a 1-char string
|
|
3934
|
+
ch = ord(ch)
|
|
3935
|
+
value = (value << 8) | ch
|
|
3936
|
+
return value
|
|
3937
|
+
|
|
3938
|
+
|
|
3939
|
+
# --- your existing CRCContext (unchanged) ---------------------------------
|
|
3940
|
+
|
|
4324
3941
|
class CRCContext(object):
|
|
4325
3942
|
__slots__ = ("spec", "table", "mask", "shift", "crc")
|
|
4326
3943
|
|
|
@@ -4334,10 +3951,11 @@ class CRCContext(object):
|
|
|
4334
3951
|
def update(self, data):
|
|
4335
3952
|
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
4336
3953
|
data = bytes(bytearray(data))
|
|
3954
|
+
buf = _mv_tobytes(memoryview(data))
|
|
4337
3955
|
if self.spec.refin:
|
|
4338
3956
|
c = self.crc
|
|
4339
3957
|
tbl = self.table
|
|
4340
|
-
for b in
|
|
3958
|
+
for b in buf:
|
|
4341
3959
|
if not isinstance(b, int): # Py2
|
|
4342
3960
|
b = ord(b)
|
|
4343
3961
|
c = tbl[(c ^ b) & 0xFF] ^ (c >> 8)
|
|
@@ -4347,7 +3965,7 @@ class CRCContext(object):
|
|
|
4347
3965
|
tbl = self.table
|
|
4348
3966
|
sh = self.shift
|
|
4349
3967
|
msk = self.mask
|
|
4350
|
-
for b in
|
|
3968
|
+
for b in buf:
|
|
4351
3969
|
if not isinstance(b, int):
|
|
4352
3970
|
b = ord(b)
|
|
4353
3971
|
c = tbl[((c >> sh) ^ b) & 0xFF] ^ ((c << 8) & msk)
|
|
@@ -4364,6 +3982,82 @@ class CRCContext(object):
|
|
|
4364
3982
|
width_hex = (self.spec.width + 3) // 4
|
|
4365
3983
|
return format(self.digest_int(), "0{}x".format(width_hex)).lower()
|
|
4366
3984
|
|
|
3985
|
+
|
|
3986
|
+
# --- hashlib-backed implementation ---------------------------------------
|
|
3987
|
+
|
|
3988
|
+
class _HashlibCRCWrapper(object):
|
|
3989
|
+
"""
|
|
3990
|
+
Wrap a hashlib object to present the same interface as CRCContext
|
|
3991
|
+
(update, digest_int, hexdigest).
|
|
3992
|
+
|
|
3993
|
+
Assumes the hashlib algorithm already implements the exact CRC
|
|
3994
|
+
specification (refin/refout/xorout/etc.).
|
|
3995
|
+
"""
|
|
3996
|
+
__slots__ = ("_h", "spec", "mask", "width_hex")
|
|
3997
|
+
|
|
3998
|
+
def __init__(self, algo_name, spec):
|
|
3999
|
+
self._h = hashlib.new(algo_name)
|
|
4000
|
+
self.spec = spec
|
|
4001
|
+
self.mask = (1 << spec.width) - 1
|
|
4002
|
+
self.width_hex = (spec.width + 3) // 4
|
|
4003
|
+
|
|
4004
|
+
def update(self, data):
|
|
4005
|
+
self._h.update(_coerce_bytes(data))
|
|
4006
|
+
return self
|
|
4007
|
+
|
|
4008
|
+
def digest_int(self):
|
|
4009
|
+
# Convert final digest bytes to an integer and mask to width
|
|
4010
|
+
value = _bytes_to_int(self._h.digest())
|
|
4011
|
+
return value & self.mask
|
|
4012
|
+
|
|
4013
|
+
def hexdigest(self):
|
|
4014
|
+
h = self._h.hexdigest().lower()
|
|
4015
|
+
# Normalize to the same number of hex digits as CRCContext
|
|
4016
|
+
if len(h) < self.width_hex:
|
|
4017
|
+
h = ("0" * (self.width_hex - len(h))) + h
|
|
4018
|
+
elif len(h) > self.width_hex:
|
|
4019
|
+
h = h[-self.width_hex:]
|
|
4020
|
+
return h
|
|
4021
|
+
|
|
4022
|
+
|
|
4023
|
+
# --- public class: choose hashlib or fallback -----------------------------
|
|
4024
|
+
|
|
4025
|
+
class CRC(object):
|
|
4026
|
+
"""
|
|
4027
|
+
CRC wrapper that uses hashlib if available, otherwise falls back to
|
|
4028
|
+
the pure-Python CRCContext.
|
|
4029
|
+
|
|
4030
|
+
spec.hashlib_name (preferred) or spec.name is used as the hashlib
|
|
4031
|
+
algorithm name, e.g. 'crc32', 'crc32c', etc.
|
|
4032
|
+
"""
|
|
4033
|
+
|
|
4034
|
+
__slots__ = ("spec", "_impl")
|
|
4035
|
+
|
|
4036
|
+
def __init__(self, spec):
|
|
4037
|
+
self.spec = spec
|
|
4038
|
+
|
|
4039
|
+
algo_name = getattr(spec, "hashlib_name", None) or getattr(spec, "name", None)
|
|
4040
|
+
impl = None
|
|
4041
|
+
|
|
4042
|
+
if algo_name and algo_name in _ALGORITHMS_AVAILABLE:
|
|
4043
|
+
# Use hashlib-backed implementation
|
|
4044
|
+
impl = _HashlibCRCWrapper(algo_name, spec)
|
|
4045
|
+
else:
|
|
4046
|
+
# Fallback to your pure-Python implementation
|
|
4047
|
+
impl = CRCContext(spec)
|
|
4048
|
+
|
|
4049
|
+
self._impl = impl
|
|
4050
|
+
|
|
4051
|
+
def update(self, data):
|
|
4052
|
+
self._impl.update(data)
|
|
4053
|
+
return self
|
|
4054
|
+
|
|
4055
|
+
def digest_int(self):
|
|
4056
|
+
return self._impl.digest_int()
|
|
4057
|
+
|
|
4058
|
+
def hexdigest(self):
|
|
4059
|
+
return self._impl.hexdigest()
|
|
4060
|
+
|
|
4367
4061
|
def crc_context_from_name(name_norm):
|
|
4368
4062
|
spec = _CRC_SPECS.get(name_norm)
|
|
4369
4063
|
if spec is None:
|
|
@@ -5231,7 +4925,7 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
|
|
|
5231
4925
|
cfcontents.close()
|
|
5232
4926
|
fcontents.seek(0, 0)
|
|
5233
4927
|
fccs = GetFileChecksum(
|
|
5234
|
-
fcontents
|
|
4928
|
+
fcontents, HeaderOut[-3].lower(), False, formatspecs)
|
|
5235
4929
|
fcontentend = fp.tell()
|
|
5236
4930
|
if(re.findall("^\\+([0-9]+)", fseeknextfile)):
|
|
5237
4931
|
fseeknextasnum = int(fseeknextfile.replace("+", ""))
|
|
@@ -7090,9 +6784,9 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7090
6784
|
if(verbose):
|
|
7091
6785
|
VerbosePrintOut(fname)
|
|
7092
6786
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7093
|
-
fpremode = int(stat.S_IFDIR
|
|
6787
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7094
6788
|
else:
|
|
7095
|
-
fpremode = int(stat.S_IFREG
|
|
6789
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7096
6790
|
flinkcount = 0
|
|
7097
6791
|
ftype = 0
|
|
7098
6792
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
@@ -7121,37 +6815,42 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
|
|
|
7121
6815
|
fbtime = format(
|
|
7122
6816
|
int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower()
|
|
7123
6817
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
7124
|
-
fwinattributes = format(int(zipinfo.external_attr), 'x').lower()
|
|
6818
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7125
6819
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7126
|
-
fmode = format(int(stat.S_IFDIR
|
|
7127
|
-
fchmode = stat.S_IMODE(int(stat.S_IFDIR
|
|
7128
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFDIR
|
|
6820
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6821
|
+
fchmode = stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))
|
|
6822
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))
|
|
7129
6823
|
else:
|
|
7130
|
-
fmode = format(int(stat.S_IFREG
|
|
7131
|
-
fchmode = stat.S_IMODE(int(stat.S_IFREG
|
|
7132
|
-
ftypemod = stat.S_IFMT(int(stat.S_IFREG
|
|
6824
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6825
|
+
fchmode = stat.S_IMODE(int(stat.S_IFREG | 0x1b6))
|
|
6826
|
+
ftypemod = stat.S_IFMT(int(stat.S_IFREG | 0x1b6))
|
|
7133
6827
|
elif(zipinfo.create_system == 3):
|
|
7134
|
-
fwinattributes = format(int(
|
|
7135
|
-
|
|
7136
|
-
|
|
7137
|
-
|
|
7138
|
-
|
|
7139
|
-
|
|
7140
|
-
|
|
7141
|
-
|
|
7142
|
-
|
|
7143
|
-
|
|
7144
|
-
|
|
7145
|
-
|
|
7146
|
-
|
|
6828
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
6829
|
+
fmode = format(int((zipinfo.external_attr >> 16) & 0xFFFF), 'x').lower()
|
|
6830
|
+
prefmode = int((zipinfo.external_attr >> 16) & 0xFFFF)
|
|
6831
|
+
if (prefmode == 0):
|
|
6832
|
+
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
6833
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6834
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
6835
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6836
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6837
|
+
else:
|
|
6838
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6839
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
6840
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6841
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6842
|
+
fchmode = stat.S_IMODE(prefmode)
|
|
6843
|
+
ftypemod = stat.S_IFMT(prefmode)
|
|
6844
|
+
else:
|
|
6845
|
+
fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
|
|
7147
6846
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
7148
|
-
fmode = format(int(stat.S_IFDIR
|
|
7149
|
-
prefmode = int(stat.S_IFDIR
|
|
6847
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
6848
|
+
prefmode = int(stat.S_IFDIR | 0x1ff)
|
|
7150
6849
|
fchmode = stat.S_IMODE(prefmode)
|
|
7151
6850
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7152
6851
|
else:
|
|
7153
|
-
fmode = format(int(stat.S_IFREG
|
|
7154
|
-
prefmode = int(stat.S_IFREG
|
|
6852
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
6853
|
+
prefmode = int(stat.S_IFREG | 0x1b6)
|
|
7155
6854
|
fchmode = stat.S_IMODE(prefmode)
|
|
7156
6855
|
ftypemod = stat.S_IFMT(prefmode)
|
|
7157
6856
|
fcompression = ""
|
|
@@ -7333,11 +7032,11 @@ if(rarfile_support):
|
|
|
7333
7032
|
if(is_unix and member.external_attr != 0):
|
|
7334
7033
|
fpremode = int(member.external_attr)
|
|
7335
7034
|
elif(member.is_file()):
|
|
7336
|
-
fpremode = int(stat.S_IFREG
|
|
7035
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7337
7036
|
elif(member.is_symlink()):
|
|
7338
|
-
fpremode = int(stat.S_IFLNK
|
|
7037
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
7339
7038
|
elif(member.is_dir()):
|
|
7340
|
-
fpremode = int(stat.S_IFDIR
|
|
7039
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7341
7040
|
if(is_windows and member.external_attr != 0):
|
|
7342
7041
|
fwinattributes = format(int(member.external_attr), 'x').lower()
|
|
7343
7042
|
else:
|
|
@@ -7390,23 +7089,23 @@ if(rarfile_support):
|
|
|
7390
7089
|
ftypemod = format(
|
|
7391
7090
|
int(stat.S_IFMT(member.external_attr)), 'x').lower()
|
|
7392
7091
|
elif(member.is_file()):
|
|
7393
|
-
fmode = format(int(stat.S_IFREG
|
|
7092
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7394
7093
|
fchmode = format(
|
|
7395
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7094
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7396
7095
|
ftypemod = format(
|
|
7397
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7096
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7398
7097
|
elif(member.is_symlink()):
|
|
7399
|
-
fmode = format(int(stat.S_IFLNK
|
|
7098
|
+
fmode = format(int(stat.S_IFLNK | 0x1b6), 'x').lower()
|
|
7400
7099
|
fchmode = format(
|
|
7401
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7100
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7402
7101
|
ftypemod = format(
|
|
7403
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7102
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7404
7103
|
elif(member.is_dir()):
|
|
7405
|
-
fmode = format(int(stat.S_IFDIR
|
|
7104
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7406
7105
|
fchmode = format(
|
|
7407
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7106
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7408
7107
|
ftypemod = format(
|
|
7409
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7108
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7410
7109
|
try:
|
|
7411
7110
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7412
7111
|
except AttributeError:
|
|
@@ -7510,11 +7209,11 @@ if(rarfile_support):
|
|
|
7510
7209
|
return fp
|
|
7511
7210
|
|
|
7512
7211
|
if(not py7zr_support):
|
|
7513
|
-
def
|
|
7212
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7514
7213
|
return False
|
|
7515
7214
|
|
|
7516
7215
|
if(py7zr_support):
|
|
7517
|
-
def
|
|
7216
|
+
def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
|
|
7518
7217
|
if(not hasattr(fp, "write")):
|
|
7519
7218
|
return False
|
|
7520
7219
|
if(verbose):
|
|
@@ -7557,9 +7256,9 @@ if(py7zr_support):
|
|
|
7557
7256
|
if(verbose):
|
|
7558
7257
|
VerbosePrintOut(fname)
|
|
7559
7258
|
if(not member.is_directory):
|
|
7560
|
-
fpremode = int(stat.S_IFREG
|
|
7259
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
7561
7260
|
elif(member.is_directory):
|
|
7562
|
-
fpremode = int(stat.S_IFDIR
|
|
7261
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
7563
7262
|
fwinattributes = format(int(0), 'x').lower()
|
|
7564
7263
|
fcompression = ""
|
|
7565
7264
|
fcsize = format(int(0), 'x').lower()
|
|
@@ -7583,17 +7282,17 @@ if(py7zr_support):
|
|
|
7583
7282
|
fctime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7584
7283
|
fbtime = format(int(member.creationtime.timestamp()), 'x').lower()
|
|
7585
7284
|
if(member.is_directory):
|
|
7586
|
-
fmode = format(int(stat.S_IFDIR
|
|
7285
|
+
fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
|
|
7587
7286
|
fchmode = format(
|
|
7588
|
-
int(stat.S_IMODE(int(stat.S_IFDIR
|
|
7287
|
+
int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7589
7288
|
ftypemod = format(
|
|
7590
|
-
int(stat.S_IFMT(int(stat.S_IFDIR
|
|
7289
|
+
int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
|
|
7591
7290
|
else:
|
|
7592
|
-
fmode = format(int(stat.S_IFREG
|
|
7291
|
+
fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
|
|
7593
7292
|
fchmode = format(
|
|
7594
|
-
int(stat.S_IMODE(int(stat.S_IFREG
|
|
7293
|
+
int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7595
7294
|
ftypemod = format(
|
|
7596
|
-
int(stat.S_IFMT(int(stat.S_IFREG
|
|
7295
|
+
int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
|
|
7597
7296
|
try:
|
|
7598
7297
|
fuid = format(int(os.getuid()), 'x').lower()
|
|
7599
7298
|
except AttributeError:
|
|
@@ -8232,11 +7931,11 @@ def AppendFilesWithContentFromRarFileToStackedOutFile(infiles, outfile, fmttype=
|
|
|
8232
7931
|
return returnout
|
|
8233
7932
|
|
|
8234
7933
|
if(not py7zr_support):
|
|
8235
|
-
def
|
|
7934
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8236
7935
|
return False
|
|
8237
7936
|
|
|
8238
7937
|
if(py7zr_support):
|
|
8239
|
-
def
|
|
7938
|
+
def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8240
7939
|
if(IsNestedDict(formatspecs) and fmttype=="auto" and
|
|
8241
7940
|
(outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
|
|
8242
7941
|
get_in_ext = os.path.splitext(outfile)
|
|
@@ -8278,7 +7977,7 @@ if(py7zr_support):
|
|
|
8278
7977
|
fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
|
|
8279
7978
|
except PermissionError:
|
|
8280
7979
|
return False
|
|
8281
|
-
|
|
7980
|
+
AppendFilesWithContentFromSevenZipFile(infiles, fp, extradata, jsondata, compression,
|
|
8282
7981
|
compresswholefile, compressionlevel, compressionuselist, checksumtype, formatspecs, verbose)
|
|
8283
7982
|
if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
|
|
8284
7983
|
fp = CompressOpenFileAlt(
|
|
@@ -8316,12 +8015,12 @@ if(py7zr_support):
|
|
|
8316
8015
|
fp.close()
|
|
8317
8016
|
return True
|
|
8318
8017
|
|
|
8319
|
-
def
|
|
8018
|
+
def AppendFilesWithContentFromSevenZipFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
|
|
8320
8019
|
if not isinstance(infiles, list):
|
|
8321
8020
|
infiles = [infiles]
|
|
8322
8021
|
returnout = False
|
|
8323
8022
|
for infileslist in infiles:
|
|
8324
|
-
returnout =
|
|
8023
|
+
returnout = AppendFilesWithContentFromSevenZipFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
|
|
8325
8024
|
if(not returnout):
|
|
8326
8025
|
break
|
|
8327
8026
|
else:
|
|
@@ -9025,8 +8724,6 @@ def CheckCompressionSubType(infile, formatspecs=__file_format_multi_dict__, file
|
|
|
9025
8724
|
fp = pyzstd.zstdfile.ZstdFile(infile, mode="rb")
|
|
9026
8725
|
else:
|
|
9027
8726
|
return Flase
|
|
9028
|
-
elif((compresscheck == "lzo" or compresscheck == "lzop") and compresscheck in compressionsupport):
|
|
9029
|
-
fp = LzopFile(infile, mode="rb")
|
|
9030
8727
|
elif((compresscheck == "lzma" or compresscheck == "xz") and compresscheck in compressionsupport):
|
|
9031
8728
|
fp = lzma.open(infile, "rb")
|
|
9032
8729
|
elif(compresscheck == "zlib" and compresscheck in compressionsupport):
|
|
@@ -9149,8 +8846,6 @@ def UncompressFileAlt(fp, formatspecs=__file_format_multi_dict__, filestart=0,
|
|
|
9149
8846
|
return False
|
|
9150
8847
|
elif kind == "lz4" and "lz4" in compressionsupport:
|
|
9151
8848
|
wrapped = lz4.frame.LZ4FrameFile(src, mode="rb")
|
|
9152
|
-
elif kind in ("lzo","lzop") and (("lzo" in compressionsupport) or ("lzop" in compressionsupport)):
|
|
9153
|
-
wrapped = LzopFile(fileobj=src, mode="rb")
|
|
9154
8849
|
elif kind == "zlib" and "zlib" in compressionsupport:
|
|
9155
8850
|
wrapped = ZlibFile(fileobj=src, mode="rb")
|
|
9156
8851
|
else:
|
|
@@ -9222,8 +8917,6 @@ def UncompressFile(infile, formatspecs=__file_format_multi_dict__, mode="rb",
|
|
|
9222
8917
|
return False
|
|
9223
8918
|
elif (compresscheck == "lz4" and "lz4" in compressionsupport):
|
|
9224
8919
|
fp = lz4.frame.open(infile, mode)
|
|
9225
|
-
elif ((compresscheck == "lzo" or compresscheck == "lzop") and "lzop" in compressionsupport):
|
|
9226
|
-
fp = LzopFile(infile, mode=mode)
|
|
9227
8920
|
elif ((compresscheck == "lzma" or compresscheck == "xz") and "xz" in compressionsupport):
|
|
9228
8921
|
fp = lzma.open(infile, mode)
|
|
9229
8922
|
elif (compresscheck == "zlib" and "zlib" in compressionsupport):
|
|
@@ -9791,7 +9484,7 @@ def fast_copy(infp, outfp, bufsize=1 << 20):
|
|
|
9791
9484
|
outfp.write(data)
|
|
9792
9485
|
|
|
9793
9486
|
|
|
9794
|
-
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=
|
|
9487
|
+
def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
|
|
9795
9488
|
"""
|
|
9796
9489
|
Copy a disk file into an mmap-backed destination (FileLikeAdapter).
|
|
9797
9490
|
Falls back to buffered copy if the source cannot be mmapped.
|
|
@@ -10017,9 +9710,6 @@ def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None,
|
|
|
10017
9710
|
elif (fextname == ".lz4" and "lz4" in compressionsupport):
|
|
10018
9711
|
outfp = FileLikeAdapter(lz4.frame.open(outfile, mode, compression_level=compressionlevel), mode="wb")
|
|
10019
9712
|
|
|
10020
|
-
elif (fextname == ".lzo" and "lzop" in compressionsupport):
|
|
10021
|
-
outfp = FileLikeAdapter(LzopFile(outfile, mode=mode, level=compressionlevel), mode="wb")
|
|
10022
|
-
|
|
10023
9713
|
elif (fextname == ".lzma" and "lzma" in compressionsupport):
|
|
10024
9714
|
try:
|
|
10025
9715
|
outfp = FileLikeAdapter(
|
|
@@ -10141,7 +9831,7 @@ if(not py7zr_support):
|
|
|
10141
9831
|
|
|
10142
9832
|
if(py7zr_support):
|
|
10143
9833
|
def PackFoxFileFromSevenZipFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
10144
|
-
return
|
|
9834
|
+
return AppendFilesWithContentFromSevenZipFileToOutFile(infile, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, returnfp)
|
|
10145
9835
|
|
|
10146
9836
|
|
|
10147
9837
|
def PackFoxFileFromInFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
|
|
@@ -10249,14 +9939,6 @@ def FoxFileValidate(infile, fmttype="auto", filestart=0,
|
|
|
10249
9939
|
checkcompressfile = CheckCompressionSubType(fp, formatspecs, filestart, True)
|
|
10250
9940
|
if(IsNestedDict(formatspecs) and checkcompressfile in formatspecs):
|
|
10251
9941
|
formatspecs = formatspecs[checkcompressfile]
|
|
10252
|
-
if(checkcompressfile == "tarfile" and TarFileCheck(infile)):
|
|
10253
|
-
return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10254
|
-
elif(checkcompressfile == "zipfile" and zipfile.is_zipfile(infile)):
|
|
10255
|
-
return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10256
|
-
elif(rarfile_support and checkcompressfile == "rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
|
|
10257
|
-
return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10258
|
-
elif(py7zr_support and checkcompressfile == "7zipfile" and py7zr.is_7zfile(infile)):
|
|
10259
|
-
return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
|
|
10260
9942
|
elif(IsSingleDict(formatspecs) and checkcompressfile != formatspecs['format_magic']):
|
|
10261
9943
|
return False
|
|
10262
9944
|
elif(IsNestedDict(formatspecs) and checkcompressfile not in formatspecs):
|
|
@@ -10618,9 +10300,10 @@ def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__fi
|
|
|
10618
10300
|
while True:
|
|
10619
10301
|
if outstartfile >= outfsize: # stop when function signals False
|
|
10620
10302
|
break
|
|
10621
|
-
is_valid_file =
|
|
10303
|
+
is_valid_file = ArchiveFileValidate(infile, fmttype, outstartfile, formatspecs, seektoend, verbose, True)
|
|
10622
10304
|
if is_valid_file is False: # stop when function signals False
|
|
10623
10305
|
outretval.append(is_valid_file)
|
|
10306
|
+
break
|
|
10624
10307
|
else:
|
|
10625
10308
|
outretval.append(True)
|
|
10626
10309
|
infile = is_valid_file
|
|
@@ -11626,35 +11309,6 @@ def MultipleFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, s
|
|
|
11626
11309
|
return outretval
|
|
11627
11310
|
|
|
11628
11311
|
|
|
11629
|
-
def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, returnfp=False):
|
|
11630
|
-
outretval = []
|
|
11631
|
-
outstartfile = filestart
|
|
11632
|
-
outfsize = float('inf')
|
|
11633
|
-
while True:
|
|
11634
|
-
if outstartfile >= outfsize: # stop when function signals False
|
|
11635
|
-
break
|
|
11636
|
-
is_valid_file = FoxFileValidate(infile, fmttype, filestart, formatspecs, seektoend, verbose, True)
|
|
11637
|
-
if is_valid_file is False: # stop when function signals False
|
|
11638
|
-
outretval.append(is_valid_file)
|
|
11639
|
-
else:
|
|
11640
|
-
outretval.append(True)
|
|
11641
|
-
infile = is_valid_file
|
|
11642
|
-
outstartfile = infile.tell()
|
|
11643
|
-
try:
|
|
11644
|
-
infile.seek(0, 2)
|
|
11645
|
-
except OSError:
|
|
11646
|
-
SeekToEndOfFile(infile)
|
|
11647
|
-
except ValueError:
|
|
11648
|
-
SeekToEndOfFile(infile)
|
|
11649
|
-
outfsize = infile.tell()
|
|
11650
|
-
infile.seek(outstartfile, 0)
|
|
11651
|
-
if(returnfp):
|
|
11652
|
-
return infile
|
|
11653
|
-
else:
|
|
11654
|
-
infile.close()
|
|
11655
|
-
return outretval
|
|
11656
|
-
|
|
11657
|
-
|
|
11658
11312
|
def StackedFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, newstyle=False, returnfp=False):
|
|
11659
11313
|
outretval = []
|
|
11660
11314
|
outstartfile = filestart
|
|
@@ -11865,11 +11519,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11865
11519
|
if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
|
|
11866
11520
|
fwinattributes = int(zipinfo.external_attr)
|
|
11867
11521
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11868
|
-
fmode = int(stat.S_IFDIR
|
|
11869
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11870
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11522
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11523
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11524
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11871
11525
|
else:
|
|
11872
|
-
fmode = int(stat.S_IFREG
|
|
11526
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11873
11527
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11874
11528
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11875
11529
|
elif(zipinfo.create_system == 3):
|
|
@@ -11885,11 +11539,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
|
|
|
11885
11539
|
else:
|
|
11886
11540
|
fwinattributes = int(0)
|
|
11887
11541
|
if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
|
|
11888
|
-
fmode = int(stat.S_IFDIR
|
|
11889
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
11890
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11542
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11543
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11544
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
11891
11545
|
else:
|
|
11892
|
-
fmode = int(stat.S_IFREG
|
|
11546
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11893
11547
|
fchmode = int(stat.S_IMODE(fmode))
|
|
11894
11548
|
ftypemod = int(stat.S_IFMT(fmode))
|
|
11895
11549
|
returnval.update({lcfi: member.filename})
|
|
@@ -12000,11 +11654,11 @@ if(rarfile_support):
|
|
|
12000
11654
|
if(is_unix and member.external_attr != 0):
|
|
12001
11655
|
fpremode = int(member.external_attr)
|
|
12002
11656
|
elif(member.is_file()):
|
|
12003
|
-
fpremode = int(stat.S_IFREG
|
|
11657
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
12004
11658
|
elif(member.is_symlink()):
|
|
12005
|
-
fpremode = int(stat.S_IFLNK
|
|
11659
|
+
fpremode = int(stat.S_IFLNK | 0x1b6)
|
|
12006
11660
|
elif(member.is_dir()):
|
|
12007
|
-
fpremode = int(stat.S_IFDIR
|
|
11661
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
12008
11662
|
if(is_windows and member.external_attr != 0):
|
|
12009
11663
|
fwinattributes = int(member.external_attr)
|
|
12010
11664
|
else:
|
|
@@ -12014,17 +11668,17 @@ if(rarfile_support):
|
|
|
12014
11668
|
fchmode = int(stat.S_IMODE(member.external_attr))
|
|
12015
11669
|
ftypemod = int(stat.S_IFMT(member.external_attr))
|
|
12016
11670
|
elif(member.is_file()):
|
|
12017
|
-
fmode = int(stat.S_IFREG
|
|
12018
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFREG
|
|
12019
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG
|
|
11671
|
+
fmode = int(stat.S_IFREG | 0x1b6)
|
|
11672
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6)))
|
|
11673
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6)))
|
|
12020
11674
|
elif(member.is_symlink()):
|
|
12021
|
-
fmode = int(stat.S_IFLNK
|
|
12022
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
12023
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11675
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11676
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11677
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
12024
11678
|
elif(member.is_dir()):
|
|
12025
|
-
fmode = int(stat.S_IFDIR
|
|
12026
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
12027
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11679
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11680
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11681
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
12028
11682
|
returnval.update({lcfi: member.filename})
|
|
12029
11683
|
if(not verbose):
|
|
12030
11684
|
VerbosePrintOut(member.filename)
|
|
@@ -12120,18 +11774,18 @@ if(py7zr_support):
|
|
|
12120
11774
|
else:
|
|
12121
11775
|
fname = "./"+member.filename
|
|
12122
11776
|
if(not member.is_directory):
|
|
12123
|
-
fpremode = int(stat.S_IFREG
|
|
11777
|
+
fpremode = int(stat.S_IFREG | 0x1b6)
|
|
12124
11778
|
elif(member.is_directory):
|
|
12125
|
-
fpremode = int(stat.S_IFDIR
|
|
11779
|
+
fpremode = int(stat.S_IFDIR | 0x1ff)
|
|
12126
11780
|
fwinattributes = int(0)
|
|
12127
11781
|
if(member.is_directory):
|
|
12128
|
-
fmode = int(stat.S_IFDIR
|
|
12129
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR
|
|
12130
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR
|
|
11782
|
+
fmode = int(stat.S_IFDIR | 0x1ff)
|
|
11783
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
|
|
11784
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
|
|
12131
11785
|
else:
|
|
12132
|
-
fmode = int(stat.S_IFLNK
|
|
12133
|
-
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK
|
|
12134
|
-
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK
|
|
11786
|
+
fmode = int(stat.S_IFLNK | 0x1b6)
|
|
11787
|
+
fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
|
|
11788
|
+
ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
|
|
12135
11789
|
returnval.update({lcfi: member.filename})
|
|
12136
11790
|
if(not verbose):
|
|
12137
11791
|
VerbosePrintOut(member.filename)
|