PyFoxFile 0.24.2__py3-none-any.whl → 0.24.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pyfoxfile.py CHANGED
@@ -14,7 +14,7 @@
14
14
  Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15
15
  Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
16
 
17
- $FileInfo: pyfoxfile.py - Last Update: 10/31/2025 Ver. 0.24.2 RC 1 - Author: cooldude2k $
17
+ $FileInfo: pyfoxfile.py - Last Update: 11/3/2025 Ver. 0.24.6 RC 1 - Author: cooldude2k $
18
18
  '''
19
19
 
20
20
  from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
@@ -33,10 +33,10 @@ import socket
33
33
  import struct
34
34
  import hashlib
35
35
  import inspect
36
- import datetime
37
36
  import logging
38
37
  import zipfile
39
38
  import binascii
39
+ import datetime
40
40
  import platform
41
41
  from io import StringIO, BytesIO
42
42
  from collections import namedtuple
@@ -413,7 +413,12 @@ def is_only_nonprintable(var):
413
413
  __file_format_multi_dict__ = {}
414
414
  __file_format_default__ = "FoxFile"
415
415
  __include_defaults__ = True
416
- __use_inmemfile__ = False
416
+ __use_inmemfile__ = True
417
+ __use_spoolfile__ = False
418
+ __use_spooldir__ = tempfile.gettempdir()
419
+ BYTES_PER_MiB = 1024 * 1024
420
+ DEFAULT_SPOOL_MAX = 8 * BYTES_PER_MiB
421
+ __spoolfile_size__ = DEFAULT_SPOOL_MAX
417
422
  __program_name__ = "Py"+__file_format_default__
418
423
  __use_env_file__ = True
419
424
  __use_ini_file__ = True
@@ -450,6 +455,8 @@ if __use_ini_file__ and os.path.exists(__config_file__):
450
455
  __program_name__ = decode_unicode_escape(config.get('config', 'proname'))
451
456
  __include_defaults__ = config.getboolean('config', 'includedef')
452
457
  __use_inmemfile__ = config.getboolean('config', 'inmemfile')
458
+ __use_spoolfile__ = config.getboolean('config', 'usespoolfile')
459
+ __spoolfile_size__ = config.getint('config', 'spoolfilesize')
453
460
  # Loop through all sections
454
461
  for section in config.sections():
455
462
  if section == "config":
@@ -543,6 +550,8 @@ elif __use_json_file__ and os.path.exists(__config_file__):
543
550
  __program_name__ = decode_unicode_escape(_get(cfg_config, 'proname', ''))
544
551
  __include_defaults__ = _to_bool(_get(cfg_config, 'includedef', False))
545
552
  __use_inmemfile__ = _to_bool(_get(cfg_config, 'inmemfile', False))
553
+ __use_spoolfile__ = _to_bool(_get(cfg_config, 'usespoolfile', False))
554
+ __spoolfile_size__ = _to_int(_get(cfg_config, 'spoolfilesize', DEFAULT_SPOOL_MAX))
546
555
 
547
556
  # --- iterate format sections (everything except "config") ---
548
557
  required_keys = [
@@ -628,13 +637,15 @@ __use_alt_inode__ = __file_format_multi_dict__[__file_format_default__]['use_alt
628
637
  __file_format_extension__ = __file_format_multi_dict__[__file_format_default__]['format_extension']
629
638
  __file_format_dict__ = __file_format_multi_dict__[__file_format_default__]
630
639
  __project__ = __program_name__
640
+ __program_alt_name__ = __program_name__
631
641
  __project_url__ = "https://github.com/GameMaker2k/PyFoxFile"
632
- __version_info__ = (0, 24, 2, "RC 1", 1)
633
- __version_date_info__ = (2025, 10, 31, "RC 1", 1)
642
+ __project_release_url__ = __project_url__+"/releases/latest"
643
+ __version_info__ = (0, 24, 6, "RC 1", 1)
644
+ __version_date_info__ = (2025, 11, 3, "RC 1", 1)
634
645
  __version_date__ = str(__version_date_info__[0]) + "." + str(
635
646
  __version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2)
636
647
  __revision__ = __version_info__[3]
637
- __revision_id__ = "$Id: 7cfe613bdb101b0c0e630f6d2ac71b2efb6edd88 $"
648
+ __revision_id__ = "$Id: 05c7600a19a0abcabf9ecc0656c3a8d7fdee6a32 $"
638
649
  if(__version_info__[4] is not None):
639
650
  __version_date_plusrc__ = __version_date__ + \
640
651
  "-" + str(__version_date_info__[4])
@@ -646,6 +657,78 @@ if(__version_info__[3] is not None):
646
657
  if(__version_info__[3] is None):
647
658
  __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2])
648
659
 
660
+ # From: https://stackoverflow.com/a/28568003
661
+ # By Phaxmohdem
662
+
663
+
664
+ def versiontuple(v):
665
+ filled = []
666
+ for point in v.split("."):
667
+ filled.append(point.zfill(8))
668
+ return tuple(filled)
669
+
670
+
671
+ def version_check(myvercheck, newvercheck):
672
+ vercheck = 0
673
+ try:
674
+ from packaging import version
675
+ vercheck = 1
676
+ except ImportError:
677
+ try:
678
+ from distutils.version import LooseVersion, StrictVersion
679
+ vercheck = 2
680
+ except ImportError:
681
+ try:
682
+ from pkg_resources import parse_version
683
+ vercheck = 3
684
+ except ImportError:
685
+ return 5
686
+ # print(myvercheck, newvercheck)
687
+ if (vercheck == 1):
688
+ if (version.parse(myvercheck) == version.parse(newvercheck)):
689
+ return 0
690
+ elif (version.parse(myvercheck) < version.parse(newvercheck)):
691
+ return 1
692
+ elif (version.parse(myvercheck) > version.parse(newvercheck)):
693
+ return 2
694
+ else:
695
+ return 3
696
+ elif (vercheck == 2):
697
+ if (StrictVersion(myvercheck) == StrictVersion(newvercheck)):
698
+ return 0
699
+ elif (StrictVersion(myvercheck) < StrictVersion(newvercheck)):
700
+ return 1
701
+ elif (StrictVersion(myvercheck) > StrictVersion(newvercheck)):
702
+ return 2
703
+ else:
704
+ return 3
705
+ elif (vercheck == 3):
706
+ if (parse_version(myvercheck) == parse_version(newvercheck)):
707
+ return 0
708
+ elif (parse_version(myvercheck) < parse_version(newvercheck)):
709
+ return 1
710
+ elif (parse_version(myvercheck) > parse_version(newvercheck)):
711
+ return 2
712
+ else:
713
+ return 3
714
+ else:
715
+ if (versiontuple(myvercheck) == versiontuple(newvercheck)):
716
+ return 0
717
+ elif (versiontuple(myvercheck) < versiontuple(newvercheck)):
718
+ return 1
719
+ elif (versiontuple(myvercheck) > versiontuple(newvercheck)):
720
+ return 2
721
+ else:
722
+ return 3
723
+ return 4
724
+
725
+
726
+ def check_version_number(myversion=__version__, proname=__program_alt_name__, newverurl=__project_release_url__):
727
+ prevercheck = download_from_url(newverurl, geturls_headers, geturls_cj)
728
+ newvercheck = re.findall(proname + " ([0-9\\.]+)<\\/a\\>", prevercheck['Content'].decode("UTF-8"))[0]
729
+ myvercheck = re.findall("([0-9\\.]+)", myversion)[0]
730
+ return version_check(myvercheck, newvercheck)
731
+
649
732
  # ===== Module-level type code table & helpers (reuse anywhere) =====
650
733
 
651
734
  FT = {
@@ -746,6 +829,7 @@ try:
746
829
  compressionsupport.append("lz4")
747
830
  except ImportError:
748
831
  pass
832
+ '''
749
833
  try:
750
834
  import lzo
751
835
  compressionsupport.append("lzo")
@@ -753,6 +837,7 @@ try:
753
837
  except ImportError:
754
838
  lzo = None
755
839
  pass
840
+ '''
756
841
  try:
757
842
  import zstandard
758
843
  compressionsupport.append("zst")
@@ -812,11 +897,13 @@ if('lzo' in compressionsupport):
812
897
  compressionlistalt.append('lzo')
813
898
  outextlist.append('lzo')
814
899
  outextlistwd.append('.lzo')
900
+ '''
815
901
  if('lzop' in compressionsupport):
816
902
  compressionlist.append('lzop')
817
903
  compressionlistalt.append('lzop')
818
904
  outextlist.append('lzop')
819
905
  outextlistwd.append('.lzop')
906
+ '''
820
907
  if('lzma' in compressionsupport):
821
908
  compressionlist.append('lzma')
822
909
  compressionlistalt.append('lzma')
@@ -1962,7 +2049,7 @@ def _normalize_initial_data(data, isbytes, encoding, errors=None):
1962
2049
 
1963
2050
 
1964
2051
  def MkTempFile(data=None,
1965
- inmem=True,
2052
+ inmem=__use_inmemfile__,
1966
2053
  isbytes=True,
1967
2054
  prefix="",
1968
2055
  delete=True,
@@ -1970,9 +2057,9 @@ def MkTempFile(data=None,
1970
2057
  newline=None, # text mode only; in-memory objects ignore newline semantics
1971
2058
  dir=None,
1972
2059
  suffix="",
1973
- use_spool=False,
1974
- spool_max=8 * 1024 * 1024,
1975
- spool_dir=None):
2060
+ use_spool=__use_spoolfile__,
2061
+ spool_max=__spoolfile_size__,
2062
+ spool_dir=__use_spooldir__:
1976
2063
  """
1977
2064
  Return a file-like handle with consistent behavior on Py2.7 and Py3.x.
1978
2065
 
@@ -2415,7 +2502,7 @@ class ZlibFile(object):
2415
2502
 
2416
2503
  def __init__(self, file_path=None, fileobj=None, mode='rb', level=6, wbits=15,
2417
2504
  encoding=None, errors=None, newline=None,
2418
- tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=(8 << 20)):
2505
+ tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
2419
2506
 
2420
2507
  if file_path is None and fileobj is None:
2421
2508
  raise ValueError("Either file_path or fileobj must be provided")
@@ -2902,7 +2989,7 @@ class GzipFile(object):
2902
2989
 
2903
2990
  def __init__(self, file_path=None, fileobj=None, mode='rb',
2904
2991
  level=6, encoding=None, errors=None, newline=None,
2905
- tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=(8 << 20)):
2992
+ tolerant_read=False, scan_bytes=(64 << 10), spool_threshold=__spoolfile_size__):
2906
2993
 
2907
2994
  if file_path is None and fileobj is None:
2908
2995
  raise ValueError("Either file_path or fileobj must be provided")
@@ -3326,519 +3413,6 @@ def gzip_decompress_bytes_all_members(blob):
3326
3413
  """
3327
3414
  return _gzip_decompress_multimember(bytes(blob))
3328
3415
 
3329
-
3330
- # ---------- Simple LZO container (NOT real .lzop) ----------
3331
- # File layout (concatenated members allowed):
3332
- # [MAGIC 8B] [FLAGS 1B] [ULEN 8B] [CRC32 4B] [CCHUNK...] | repeat...
3333
- # where:
3334
- # MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
3335
- # FLAGS = bit0: 1 => member has ULEN+CRC, 0 => no header (legacy)
3336
- # ULEN = uncompressed length (u64 BE)
3337
- # CRC32 = CRC32 of uncompressed data (u32 BE)
3338
- # CCHUNK = one or more compressed chunks:
3339
- # [u32BE chunk_size][chunk_data] ... then a zero-size u32 terminator
3340
-
3341
-
3342
- class LzopFile(object):
3343
- MAGIC = b'\x89LZO\x0D\x0A\x1A\n'
3344
- FLAG_HAS_UHDR = 0x01
3345
- RAW_CHUNK = 256 * 1024 # 256 KiB per raw (pre-compress) chunk
3346
-
3347
- def __init__(self, file_path=None, fileobj=None, mode='rb',
3348
- level=9, encoding=None, errors=None, newline=None,
3349
- write_header=True,
3350
- tolerant_read=False, scan_bytes=(64 << 10),
3351
- spool_threshold=(8 << 20)):
3352
- """
3353
- Custom LZO file (NOT the lzop(1) format).
3354
- - streaming write/read, supports concatenated members
3355
- - optional per-member header (uncompressed length + CRC32)
3356
- - spooled reads to limit RAM, strict text mode with newline control
3357
- - tolerant_read: scan forward (up to scan_bytes) to first MAGIC
3358
-
3359
- :param write_header: if True, include ULEN+CRC32 per member
3360
- :param tolerant_read: skip leading junk up to scan_bytes to find MAGIC
3361
- :param scan_bytes: max bytes to scan when tolerant_read=True
3362
- :param spool_threshold: SpooledTemporaryFile RAM threshold before spill
3363
- """
3364
- if lzo is None:
3365
- raise ImportError("python-lzo is required for LzopFile")
3366
-
3367
- if file_path is None and fileobj is None:
3368
- raise ValueError("Either file_path or fileobj must be provided")
3369
- if file_path is not None and fileobj is not None:
3370
- raise ValueError("Only one of file_path or fileobj should be provided")
3371
-
3372
- if 'b' not in mode and 't' not in mode:
3373
- mode += 'b'
3374
- if 'x' in mode and PY2:
3375
- raise ValueError("Exclusive creation mode 'x' is not supported on Python 2")
3376
-
3377
- self.file_path = file_path
3378
- self.file = fileobj
3379
- self.mode = mode
3380
- self.level = int(level) # effective: 1 or 9 (clamped)
3381
- self.encoding = encoding
3382
- self.errors = errors
3383
- self.newline = newline
3384
- self._text_mode = ('t' in mode)
3385
-
3386
- self._write_header = bool(write_header)
3387
-
3388
- # Config (read path)
3389
- self.tolerant_read = bool(tolerant_read)
3390
- self.scan_bytes = int(scan_bytes)
3391
- self.spool_threshold = int(spool_threshold)
3392
-
3393
- # Write state
3394
- self._crc = 0
3395
- self._ulen = 0
3396
- self._open_member = False
3397
- self._member_header_pos = None # position *after* ULEN+CRC placeholders
3398
-
3399
- # Read state
3400
- self._spool = None
3401
- self._text_reader = None
3402
- self._position = 0
3403
- self.closed = False
3404
-
3405
- internal_mode = mode.replace('t', 'b')
3406
- if self.file is None:
3407
- if 'x' in internal_mode and os.path.exists(file_path):
3408
- raise IOError("File exists: '{}'".format(file_path))
3409
- self.file = open(file_path, internal_mode)
3410
- else:
3411
- if 'r' in internal_mode and not hasattr(self.file, 'read'):
3412
- raise ValueError("fileobj must support read() in read mode")
3413
- if any(ch in internal_mode for ch in ('w', 'a', 'x')) and not hasattr(self.file, 'write'):
3414
- raise ValueError("fileobj must support write() in write/append mode")
3415
-
3416
- self._fp = self.file
3417
- if any(ch in internal_mode for ch in ('w', 'a', 'x')):
3418
- # Start a new member at EOF for append
3419
- if 'a' in internal_mode:
3420
- try:
3421
- self.file.seek(0, os.SEEK_END)
3422
- except Exception:
3423
- pass
3424
- # Defer writing header until first write so empty files don’t get empty members
3425
- elif 'r' in internal_mode:
3426
- self._load_all_members_spooled()
3427
- else:
3428
- raise ValueError("Unsupported mode: {}".format(mode))
3429
-
3430
- # ---------- helpers ----------
3431
- @property
3432
- def name(self):
3433
- return self.file_path
3434
-
3435
- def readable(self):
3436
- return 'r' in self.mode
3437
-
3438
- def writable(self):
3439
- return any(ch in self.mode for ch in ('w', 'a', 'x'))
3440
-
3441
- def seekable(self):
3442
- return True if self._spool is not None else bool(getattr(self.file, 'seek', None))
3443
-
3444
- def _normalize_newlines_for_write(self, s):
3445
- nl = self.newline if self.newline is not None else "\n"
3446
- return s.replace("\r\n", "\n").replace("\r", "\n").replace("\n", nl)
3447
-
3448
- def _reader(self):
3449
- return self._text_reader if self._text_mode else self._spool
3450
-
3451
- # ---------- Write path ----------
3452
- def _ensure_member_header(self):
3453
- if self._open_member:
3454
- return
3455
- flags = self.FLAG_HAS_UHDR if self._write_header else 0
3456
- self.file.write(self.MAGIC)
3457
- self.file.write(struct.pack(">B", flags))
3458
- if self._write_header:
3459
- # placeholders for ULEN+CRC; we’ll backfill on finalize
3460
- self.file.write(struct.pack(">Q", 0))
3461
- self.file.write(struct.pack(">I", 0))
3462
- # position *after* ULEN+CRC placeholders (or after FLAGS if no header)
3463
- self._member_header_pos = self.file.tell()
3464
- self._open_member = True
3465
- # reset member stats
3466
- self._crc = 0
3467
- self._ulen = 0
3468
-
3469
- def write(self, data):
3470
- if 'r' in self.mode:
3471
- raise IOError("File not open for writing")
3472
-
3473
- if self._text_mode:
3474
- enc = self.encoding or 'UTF-8'
3475
- errs = self.errors or 'strict'
3476
- if not isinstance(data, text_type):
3477
- raise TypeError("write() expects text (unicode/str) in text mode")
3478
- data = self._normalize_newlines_for_write(data).encode(enc, errs)
3479
- else:
3480
- if not isinstance(data, binary_types):
3481
- raise TypeError("write() expects bytes-like in binary mode")
3482
-
3483
- # Normalize Py3 memoryview / Py2 bytearray
3484
- if (not PY2) and isinstance(data, memoryview):
3485
- data = data.tobytes()
3486
- elif PY2 and isinstance(data, bytearray):
3487
- data = bytes(data)
3488
-
3489
- if not data:
3490
- return 0
3491
-
3492
- # Begin member and write header on first write
3493
- self._ensure_member_header()
3494
-
3495
- # Update integrity stats
3496
- self._crc = _crc32u(data, self._crc)
3497
- self._ulen += len(data)
3498
-
3499
- # Stream in RAW_CHUNK-sized pieces. Each piece becomes one compressed chunk record.
3500
- mv = memoryview(data)
3501
- # clamp level to {1, 9}
3502
- lvl = 9 if self.level >= 9 else 1
3503
- for off in range(0, len(data), self.RAW_CHUNK):
3504
- raw = mv[off:off + self.RAW_CHUNK].tobytes()
3505
- c = lzo.compress(raw, lvl)
3506
- self.file.write(struct.pack(">I", len(c)))
3507
- self.file.write(c)
3508
-
3509
- return len(data)
3510
-
3511
- def _flush_member_only(self):
3512
- """Finalize the current member: write terminator and backfill header."""
3513
- if not self._open_member:
3514
- return
3515
- # write zero-length chunk terminator
3516
- self.file.write(struct.pack(">I", 0))
3517
- if self._write_header:
3518
- # ULEN is at (_member_header_pos - 12), CRC at (_member_header_pos - 4)
3519
- ulen_pos = self._member_header_pos - 12
3520
- crc_pos = self._member_header_pos - 4
3521
- cur = self.file.tell()
3522
- # backfill ULEN
3523
- self.file.seek(ulen_pos, os.SEEK_SET)
3524
- self.file.write(struct.pack(">Q", self._ulen))
3525
- # backfill CRC32
3526
- self.file.seek(crc_pos, os.SEEK_SET)
3527
- self.file.write(struct.pack(">I", self._crc))
3528
- # restore position
3529
- self.file.seek(cur, os.SEEK_SET)
3530
- # reset for potential new member
3531
- self._open_member = False
3532
- self._crc = 0
3533
- self._ulen = 0
3534
- self._member_header_pos = None
3535
-
3536
- def flush(self):
3537
- if self.closed:
3538
- return
3539
- # finalize any open member
3540
- if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
3541
- self._flush_member_only()
3542
- if hasattr(self.file, 'flush'):
3543
- self.file.flush()
3544
-
3545
- def close(self):
3546
- if self.closed:
3547
- return
3548
- try:
3549
- # Ensure a clean member terminator & header backfill if needed
3550
- if any(ch in self.mode for ch in ('w', 'a', 'x')) and self._open_member:
3551
- self._flush_member_only()
3552
- if hasattr(self.file, 'flush'):
3553
- try:
3554
- self.file.flush()
3555
- except Exception:
3556
- pass
3557
- finally:
3558
- if self.file_path and self.file is not None:
3559
- try:
3560
- self.file.close()
3561
- except Exception:
3562
- pass
3563
- # tear down read handles
3564
- try:
3565
- if self._text_reader is not None:
3566
- self._text_reader.detach()
3567
- except Exception:
3568
- pass
3569
- try:
3570
- if self._spool is not None:
3571
- self._spool.close()
3572
- except Exception:
3573
- pass
3574
- self.closed = True
3575
-
3576
- # ---------- Read path (spooled, multi-member, tolerant scan) ----------
3577
- def _load_all_members_spooled(self):
3578
- # Seek to start if possible
3579
- try:
3580
- self.file.seek(0)
3581
- except Exception:
3582
- pass
3583
-
3584
- self._spool = tempfile.SpooledTemporaryFile(max_size=self.spool_threshold)
3585
-
3586
- def read_exact(n, abs_off_ref):
3587
- """Read exactly n bytes, updating abs_off_ref[0]."""
3588
- b = b""
3589
- while len(b) < n:
3590
- part = self.file.read(n - len(b))
3591
- if not part:
3592
- break
3593
- b += part
3594
- abs_off_ref[0] += len(part)
3595
- return b
3596
-
3597
- CHUNK = 1 << 20
3598
- abs_off = [0] # track absolute file offset
3599
- scanned = 0
3600
-
3601
- while True:
3602
- # Locate MAGIC (support tolerant scan across chunk boundaries)
3603
- head = read_exact(len(self.MAGIC), abs_off)
3604
- if not head:
3605
- break # EOF
3606
- if head != self.MAGIC:
3607
- # Tolerant scan: slide-by-one until found or limit exceeded
3608
- buf = head
3609
- while True:
3610
- if self.tolerant_read and scanned < self.scan_bytes:
3611
- nxt = read_exact(1, abs_off)
3612
- if not nxt:
3613
- # EOF without finding magic
3614
- raise ValueError("Invalid LZO container: magic not found before EOF")
3615
- buf = buf[1:] + nxt
3616
- scanned += 1
3617
- if buf == self.MAGIC:
3618
- break
3619
- continue
3620
- raise ValueError("Invalid LZO container magic near offset {}".format(abs_off[0] - len(buf)))
3621
- # found MAGIC; proceed
3622
-
3623
- # FLAGS
3624
- f_b = read_exact(1, abs_off)
3625
- if len(f_b) != 1:
3626
- raise ValueError("Truncated header (flags) at offset {}".format(abs_off[0]))
3627
- flags = ord(f_b) if PY2 else f_b[0]
3628
-
3629
- # Optional ULEN/CRC
3630
- ulen = None
3631
- expect_crc = None
3632
- if flags & self.FLAG_HAS_UHDR:
3633
- ulen_b = read_exact(8, abs_off)
3634
- crc_b = read_exact(4, abs_off)
3635
- if len(ulen_b) != 8 or len(crc_b) != 4:
3636
- raise ValueError("Truncated ULEN/CRC header at offset {}".format(abs_off[0]))
3637
- ulen = struct.unpack(">Q", ulen_b)[0]
3638
- expect_crc = struct.unpack(">I", crc_b)[0]
3639
-
3640
- # Chunk loop
3641
- m_crc = 0
3642
- m_len = 0
3643
- while True:
3644
- sz_b = read_exact(4, abs_off)
3645
- if len(sz_b) != 4:
3646
- raise ValueError("Truncated chunk size at offset {}".format(abs_off[0]))
3647
- csz = struct.unpack(">I", sz_b)[0]
3648
- if csz == 0:
3649
- break # end of member
3650
- cdata = read_exact(csz, abs_off)
3651
- if len(cdata) != csz:
3652
- raise ValueError("Truncated chunk payload at offset {}".format(abs_off[0]))
3653
- try:
3654
- raw = lzo.decompress(cdata)
3655
- except Exception as e:
3656
- raise ValueError("LZO decompression error at offset {}: {}".format(abs_off[0], e))
3657
- self._spool.write(raw)
3658
- m_len += len(raw)
3659
- m_crc = _crc32u(raw, m_crc)
3660
-
3661
- # Validate member integrity if header present
3662
- if ulen is not None and m_len != ulen:
3663
- raise ValueError("Member length mismatch ({} != {})".format(m_len, ulen))
3664
- if expect_crc is not None and m_crc != expect_crc:
3665
- raise ValueError("Member CRC32 mismatch (got 0x{:08x}, want 0x{:08x})"
3666
- .format(m_crc, expect_crc))
3667
-
3668
- # Prepare read handles
3669
- try:
3670
- self._spool.seek(0)
3671
- except Exception:
3672
- pass
3673
-
3674
- if self._text_mode:
3675
- enc = self.encoding or 'UTF-8'
3676
- errs = self.errors or 'strict'
3677
- # newline=None => universal newline translation; exact string if provided
3678
- self._text_reader = io.TextIOWrapper(self._spool, encoding=enc, errors=errs, newline=self.newline)
3679
- try:
3680
- self._text_reader.seek(0)
3681
- except Exception:
3682
- pass
3683
-
3684
- self._position = 0
3685
-
3686
- # ---------- Buffered read API (delegates to spool/text wrapper) ----------
3687
- def read(self, size=-1):
3688
- if self.closed:
3689
- raise ValueError("I/O operation on closed file")
3690
- if 'r' not in self.mode:
3691
- raise IOError("File not open for reading")
3692
- r = self._reader()
3693
- if r is None:
3694
- raise IOError("Reader not initialized")
3695
- out = r.read() if (size is None or size < 0) else r.read(int(size))
3696
- try:
3697
- self._position = r.tell()
3698
- except Exception:
3699
- pass
3700
- return out
3701
-
3702
- def readline(self, size=-1):
3703
- if self.closed:
3704
- raise ValueError("I/O operation on closed file")
3705
- if 'r' not in self.mode:
3706
- raise IOError("File not open for reading")
3707
- r = self._reader()
3708
- if r is None:
3709
- raise IOError("Reader not initialized")
3710
- out = r.readline() if (size is None or size < 0) else r.readline(int(size))
3711
- try:
3712
- self._position = r.tell()
3713
- except Exception:
3714
- pass
3715
- if not self._text_mode and out is None:
3716
- return b""
3717
- if self._text_mode and out is None:
3718
- return text_type("")
3719
- return out
3720
-
3721
- def __iter__(self):
3722
- return self
3723
-
3724
- def __next__(self):
3725
- line = self.readline()
3726
- if (self._text_mode and line == "") or (not self._text_mode and line == b""):
3727
- raise StopIteration
3728
- return line
3729
-
3730
- if PY2:
3731
- next = __next__
3732
-
3733
- def seek(self, offset, whence=0):
3734
- if self.closed:
3735
- raise ValueError("I/O operation on closed file")
3736
- if 'r' not in self.mode:
3737
- raise IOError("File not open for reading")
3738
- r = self._reader()
3739
- if r is None:
3740
- raise IOError("Reader not initialized")
3741
- newpos = r.seek(int(offset), int(whence))
3742
- self._position = newpos
3743
- return newpos
3744
-
3745
- def tell(self):
3746
- if self._reader() is not None:
3747
- try:
3748
- self._position = self._reader().tell()
3749
- except Exception:
3750
- pass
3751
- return self._position
3752
-
3753
- # ---------- Misc ----------
3754
- def fileno(self):
3755
- if hasattr(self.file, 'fileno'):
3756
- return self.file.fileno()
3757
- raise OSError("Underlying file object does not support fileno()")
3758
-
3759
- def isatty(self):
3760
- return bool(getattr(self.file, 'isatty', lambda: False)())
3761
-
3762
- def truncate(self, size=None):
3763
- # Prevent corruption of compressed streams
3764
- raise OSError("truncate() is not supported for compressed streams")
3765
-
3766
- # ---------- Convenience constructors ----------
3767
- @classmethod
3768
- def open(cls, path, mode='rb', **kw):
3769
- """
3770
- Mirror built-in open() but for LzopFile.
3771
- Example:
3772
- with LzopFile.open("data.lzo", "rt", encoding="utf-8") as f:
3773
- print(f.readline())
3774
- """
3775
- return cls(file_path=path, mode=mode, **kw)
3776
-
3777
- @classmethod
3778
- def from_fileobj(cls, fileobj, mode='rb', **kw):
3779
- """
3780
- Wrap an existing file-like object (caller retains ownership).
3781
- """
3782
- return cls(fileobj=fileobj, mode=mode, **kw)
3783
-
3784
- @classmethod
3785
- def from_bytes(cls, data, mode='rb', **kw):
3786
- """
3787
- Read from an in-memory bytes buffer.
3788
- Example:
3789
- f = LzopFile.from_bytes(blob, mode='rt', encoding='utf-8', tolerant_read=True)
3790
- text = f.read()
3791
- """
3792
- if not isinstance(data, (bytes, bytearray, memoryview)):
3793
- raise TypeError("from_bytes() expects a bytes-like object")
3794
- bio = io.BytesIO(bytes(data) if not isinstance(data, bytes) else data)
3795
- return cls(fileobj=bio, mode=mode, **kw)
3796
-
3797
- # compatibility aliases for unwrapping utilities
3798
- @property
3799
- def fileobj(self):
3800
- return self.file
3801
-
3802
- @property
3803
- def myfileobj(self):
3804
- return self.file
3805
-
3806
- # ---------- Top-level helpers ----------
3807
- def lzop_compress_bytes(payload, level=9, text=False, **kw):
3808
- """
3809
- Compress 'payload' into a single LZO member (our custom container) and return bytes.
3810
- - text=True: 'payload' is text; encoding/newline/errors handled via LzopFile('wt')
3811
- - text=False: 'payload' is bytes-like; written via LzopFile('wb')
3812
- Kwargs forwarded: write_header (default True), newline/encoding/errors, etc.
3813
- """
3814
- bio = io.BytesIO()
3815
- mode = 'wt' if text else 'wb'
3816
- f = LzopFile(fileobj=bio, mode=mode, level=level, **kw)
3817
- try:
3818
- f.write(payload)
3819
- f.flush() # finalize member (writes terminator + backfills header)
3820
- finally:
3821
- f.close()
3822
- return bio.getvalue()
3823
-
3824
-
3825
- def lzop_decompress_bytes(blob, mode='rb', tolerant_read=False, scan_bytes=(64 << 10),
3826
- spool_threshold=(8 << 20), **kw):
3827
- """
3828
- Decompress bytes produced by this custom container.
3829
- - mode='rb' -> returns bytes; mode='rt' -> returns text (set encoding/errors/newline in kw)
3830
- - tolerant_read/scan_bytes/spool_threshold forwarded to LzopFile
3831
- """
3832
- if not isinstance(blob, (bytes, bytearray, memoryview)):
3833
- raise TypeError("lzop_decompress_bytes() expects a bytes-like object")
3834
- f = LzopFile.from_bytes(blob, mode=mode, tolerant_read=tolerant_read,
3835
- scan_bytes=scan_bytes, spool_threshold=spool_threshold, **kw)
3836
- try:
3837
- return f.read()
3838
- finally:
3839
- f.close()
3840
-
3841
-
3842
3416
  def TarFileCheck(infile):
3843
3417
  try:
3844
3418
  if is_tarfile(infile):
@@ -4321,6 +3895,49 @@ _CRC_SPECS = {
4321
3895
  "crc64_iso": CRCSpec(64, 0x000000000000001B, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, True, True),
4322
3896
  }
4323
3897
 
3898
+ # --- helpers --------------------------------------------------------------
3899
+
3900
+ try:
3901
+ # Python 2 may not have algorithms_available
3902
+ _ALGORITHMS_AVAILABLE = set(hashlib.algorithms_available)
3903
+ except AttributeError:
3904
+ _ALGORITHMS_AVAILABLE = set(getattr(hashlib, "algorithms", []))
3905
+
3906
+
3907
+ def _coerce_bytes(data):
3908
+ """Return `data` as a bytes object (Py2 / Py3)."""
3909
+ if isinstance(data, memoryview):
3910
+ # Py3 has .tobytes(), Py2 falls back to bytes()
3911
+ try:
3912
+ return data.tobytes()
3913
+ except AttributeError:
3914
+ return bytes(data)
3915
+
3916
+ if isinstance(data, bytearray):
3917
+ return bytes(data)
3918
+
3919
+ if not isinstance(data, bytes):
3920
+ # E.g. list of ints, unicode, etc.
3921
+ return bytes(bytearray(data))
3922
+
3923
+ return data
3924
+
3925
+
3926
+ def _bytes_to_int(b):
3927
+ """Big-endian bytes -> int, Py2/3 safe."""
3928
+ if not isinstance(b, (bytes, bytearray)):
3929
+ b = _coerce_bytes(b)
3930
+
3931
+ value = 0
3932
+ for ch in b:
3933
+ if not isinstance(ch, int): # Py2: ch is a 1-char string
3934
+ ch = ord(ch)
3935
+ value = (value << 8) | ch
3936
+ return value
3937
+
3938
+
3939
+ # --- your existing CRCContext (unchanged) ---------------------------------
3940
+
4324
3941
  class CRCContext(object):
4325
3942
  __slots__ = ("spec", "table", "mask", "shift", "crc")
4326
3943
 
@@ -4334,10 +3951,11 @@ class CRCContext(object):
4334
3951
  def update(self, data):
4335
3952
  if not isinstance(data, (bytes, bytearray, memoryview)):
4336
3953
  data = bytes(bytearray(data))
3954
+ buf = _mv_tobytes(memoryview(data))
4337
3955
  if self.spec.refin:
4338
3956
  c = self.crc
4339
3957
  tbl = self.table
4340
- for b in memoryview(data).tobytes():
3958
+ for b in buf:
4341
3959
  if not isinstance(b, int): # Py2
4342
3960
  b = ord(b)
4343
3961
  c = tbl[(c ^ b) & 0xFF] ^ (c >> 8)
@@ -4347,7 +3965,7 @@ class CRCContext(object):
4347
3965
  tbl = self.table
4348
3966
  sh = self.shift
4349
3967
  msk = self.mask
4350
- for b in memoryview(data).tobytes():
3968
+ for b in buf:
4351
3969
  if not isinstance(b, int):
4352
3970
  b = ord(b)
4353
3971
  c = tbl[((c >> sh) ^ b) & 0xFF] ^ ((c << 8) & msk)
@@ -4364,6 +3982,82 @@ class CRCContext(object):
4364
3982
  width_hex = (self.spec.width + 3) // 4
4365
3983
  return format(self.digest_int(), "0{}x".format(width_hex)).lower()
4366
3984
 
3985
+
3986
+ # --- hashlib-backed implementation ---------------------------------------
3987
+
3988
+ class _HashlibCRCWrapper(object):
3989
+ """
3990
+ Wrap a hashlib object to present the same interface as CRCContext
3991
+ (update, digest_int, hexdigest).
3992
+
3993
+ Assumes the hashlib algorithm already implements the exact CRC
3994
+ specification (refin/refout/xorout/etc.).
3995
+ """
3996
+ __slots__ = ("_h", "spec", "mask", "width_hex")
3997
+
3998
+ def __init__(self, algo_name, spec):
3999
+ self._h = hashlib.new(algo_name)
4000
+ self.spec = spec
4001
+ self.mask = (1 << spec.width) - 1
4002
+ self.width_hex = (spec.width + 3) // 4
4003
+
4004
+ def update(self, data):
4005
+ self._h.update(_coerce_bytes(data))
4006
+ return self
4007
+
4008
+ def digest_int(self):
4009
+ # Convert final digest bytes to an integer and mask to width
4010
+ value = _bytes_to_int(self._h.digest())
4011
+ return value & self.mask
4012
+
4013
+ def hexdigest(self):
4014
+ h = self._h.hexdigest().lower()
4015
+ # Normalize to the same number of hex digits as CRCContext
4016
+ if len(h) < self.width_hex:
4017
+ h = ("0" * (self.width_hex - len(h))) + h
4018
+ elif len(h) > self.width_hex:
4019
+ h = h[-self.width_hex:]
4020
+ return h
4021
+
4022
+
4023
+ # --- public class: choose hashlib or fallback -----------------------------
4024
+
4025
+ class CRC(object):
4026
+ """
4027
+ CRC wrapper that uses hashlib if available, otherwise falls back to
4028
+ the pure-Python CRCContext.
4029
+
4030
+ spec.hashlib_name (preferred) or spec.name is used as the hashlib
4031
+ algorithm name, e.g. 'crc32', 'crc32c', etc.
4032
+ """
4033
+
4034
+ __slots__ = ("spec", "_impl")
4035
+
4036
+ def __init__(self, spec):
4037
+ self.spec = spec
4038
+
4039
+ algo_name = getattr(spec, "hashlib_name", None) or getattr(spec, "name", None)
4040
+ impl = None
4041
+
4042
+ if algo_name and algo_name in _ALGORITHMS_AVAILABLE:
4043
+ # Use hashlib-backed implementation
4044
+ impl = _HashlibCRCWrapper(algo_name, spec)
4045
+ else:
4046
+ # Fallback to your pure-Python implementation
4047
+ impl = CRCContext(spec)
4048
+
4049
+ self._impl = impl
4050
+
4051
+ def update(self, data):
4052
+ self._impl.update(data)
4053
+ return self
4054
+
4055
+ def digest_int(self):
4056
+ return self._impl.digest_int()
4057
+
4058
+ def hexdigest(self):
4059
+ return self._impl.hexdigest()
4060
+
4367
4061
  def crc_context_from_name(name_norm):
4368
4062
  spec = _CRC_SPECS.get(name_norm)
4369
4063
  if spec is None:
@@ -5231,7 +4925,7 @@ def ReadFileHeaderDataWithContentToArray(fp, listonly=False, contentasfile=True,
5231
4925
  cfcontents.close()
5232
4926
  fcontents.seek(0, 0)
5233
4927
  fccs = GetFileChecksum(
5234
- fcontents.read(), HeaderOut[-3].lower(), False, formatspecs)
4928
+ fcontents, HeaderOut[-3].lower(), False, formatspecs)
5235
4929
  fcontentend = fp.tell()
5236
4930
  if(re.findall("^\\+([0-9]+)", fseeknextfile)):
5237
4931
  fseeknextasnum = int(fseeknextfile.replace("+", ""))
@@ -7090,9 +6784,9 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
7090
6784
  if(verbose):
7091
6785
  VerbosePrintOut(fname)
7092
6786
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
7093
- fpremode = int(stat.S_IFDIR + 511)
6787
+ fpremode = int(stat.S_IFDIR | 0x1ff)
7094
6788
  else:
7095
- fpremode = int(stat.S_IFREG + 438)
6789
+ fpremode = int(stat.S_IFREG | 0x1b6)
7096
6790
  flinkcount = 0
7097
6791
  ftype = 0
7098
6792
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
@@ -7121,37 +6815,42 @@ def AppendFilesWithContentFromZipFile(infile, fp, extradata=[], jsondata={}, com
7121
6815
  fbtime = format(
7122
6816
  int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower()
7123
6817
  if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
7124
- fwinattributes = format(int(zipinfo.external_attr), 'x').lower()
6818
+ fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
7125
6819
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
7126
- fmode = format(int(stat.S_IFDIR + 511), 'x').lower()
7127
- fchmode = stat.S_IMODE(int(stat.S_IFDIR + 511))
7128
- ftypemod = stat.S_IFMT(int(stat.S_IFDIR + 511))
6820
+ fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
6821
+ fchmode = stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))
6822
+ ftypemod = stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))
7129
6823
  else:
7130
- fmode = format(int(stat.S_IFREG + 438), 'x').lower()
7131
- fchmode = stat.S_IMODE(int(stat.S_IFREG + 438))
7132
- ftypemod = stat.S_IFMT(int(stat.S_IFREG + 438))
6824
+ fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
6825
+ fchmode = stat.S_IMODE(int(stat.S_IFREG | 0x1b6))
6826
+ ftypemod = stat.S_IFMT(int(stat.S_IFREG | 0x1b6))
7133
6827
  elif(zipinfo.create_system == 3):
7134
- fwinattributes = format(int(0), 'x').lower()
7135
- try:
7136
- fmode = format(int(zipinfo.external_attr), 'x').lower()
7137
- prefmode = int(zipinfo.external_attr)
7138
- fchmode = stat.S_IMODE(prefmode)
7139
- ftypemod = stat.S_IFMT(prefmode)
7140
- except OverflowError:
7141
- fmode = format(int(zipinfo.external_attr >> 16), 'x').lower()
7142
- prefmode = int(zipinfo.external_attr >> 16)
7143
- fchmode = stat.S_IMODE(prefmode)
7144
- ftypemod = stat.S_IFMT(prefmode)
7145
- else:
7146
- fwinattributes = format(int(0), 'x').lower()
6828
+ fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
6829
+ fmode = format(int((zipinfo.external_attr >> 16) & 0xFFFF), 'x').lower()
6830
+ prefmode = int((zipinfo.external_attr >> 16) & 0xFFFF)
6831
+ if (prefmode == 0):
6832
+ if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
6833
+ fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
6834
+ prefmode = int(stat.S_IFDIR | 0x1ff)
6835
+ fchmode = stat.S_IMODE(prefmode)
6836
+ ftypemod = stat.S_IFMT(prefmode)
6837
+ else:
6838
+ fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
6839
+ prefmode = int(stat.S_IFREG | 0x1b6)
6840
+ fchmode = stat.S_IMODE(prefmode)
6841
+ ftypemod = stat.S_IFMT(prefmode)
6842
+ fchmode = stat.S_IMODE(prefmode)
6843
+ ftypemod = stat.S_IFMT(prefmode)
6844
+ else:
6845
+ fwinattributes = format(int(zipinfo.external_attr & 0xFFFF), 'x').lower()
7147
6846
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
7148
- fmode = format(int(stat.S_IFDIR + 511), 'x').lower()
7149
- prefmode = int(stat.S_IFDIR + 511)
6847
+ fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
6848
+ prefmode = int(stat.S_IFDIR | 0x1ff)
7150
6849
  fchmode = stat.S_IMODE(prefmode)
7151
6850
  ftypemod = stat.S_IFMT(prefmode)
7152
6851
  else:
7153
- fmode = format(int(stat.S_IFREG + 438), 'x').lower()
7154
- prefmode = int(stat.S_IFREG + 438)
6852
+ fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
6853
+ prefmode = int(stat.S_IFREG | 0x1b6)
7155
6854
  fchmode = stat.S_IMODE(prefmode)
7156
6855
  ftypemod = stat.S_IFMT(prefmode)
7157
6856
  fcompression = ""
@@ -7333,11 +7032,11 @@ if(rarfile_support):
7333
7032
  if(is_unix and member.external_attr != 0):
7334
7033
  fpremode = int(member.external_attr)
7335
7034
  elif(member.is_file()):
7336
- fpremode = int(stat.S_IFREG + 438)
7035
+ fpremode = int(stat.S_IFREG | 0x1b6)
7337
7036
  elif(member.is_symlink()):
7338
- fpremode = int(stat.S_IFLNK + 438)
7037
+ fpremode = int(stat.S_IFLNK | 0x1b6)
7339
7038
  elif(member.is_dir()):
7340
- fpremode = int(stat.S_IFDIR + 511)
7039
+ fpremode = int(stat.S_IFDIR | 0x1ff)
7341
7040
  if(is_windows and member.external_attr != 0):
7342
7041
  fwinattributes = format(int(member.external_attr), 'x').lower()
7343
7042
  else:
@@ -7390,23 +7089,23 @@ if(rarfile_support):
7390
7089
  ftypemod = format(
7391
7090
  int(stat.S_IFMT(member.external_attr)), 'x').lower()
7392
7091
  elif(member.is_file()):
7393
- fmode = format(int(stat.S_IFREG + 438), 'x').lower()
7092
+ fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
7394
7093
  fchmode = format(
7395
- int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower()
7094
+ int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7396
7095
  ftypemod = format(
7397
- int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower()
7096
+ int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7398
7097
  elif(member.is_symlink()):
7399
- fmode = format(int(stat.S_IFLNK + 438), 'x').lower()
7098
+ fmode = format(int(stat.S_IFLNK | 0x1b6), 'x').lower()
7400
7099
  fchmode = format(
7401
- int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower()
7100
+ int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7402
7101
  ftypemod = format(
7403
- int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower()
7102
+ int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7404
7103
  elif(member.is_dir()):
7405
- fmode = format(int(stat.S_IFDIR + 511), 'x').lower()
7104
+ fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
7406
7105
  fchmode = format(
7407
- int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower()
7106
+ int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
7408
7107
  ftypemod = format(
7409
- int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower()
7108
+ int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
7410
7109
  try:
7411
7110
  fuid = format(int(os.getuid()), 'x').lower()
7412
7111
  except AttributeError:
@@ -7510,11 +7209,11 @@ if(rarfile_support):
7510
7209
  return fp
7511
7210
 
7512
7211
  if(not py7zr_support):
7513
- def AppendFilesWithContentFromSevenZip(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
7212
+ def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
7514
7213
  return False
7515
7214
 
7516
7215
  if(py7zr_support):
7517
- def AppendFilesWithContentFromSevenZip(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
7216
+ def AppendFilesWithContentFromSevenZipFile(infile, fp, extradata=[], jsondata={}, compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_dict__, verbose=False):
7518
7217
  if(not hasattr(fp, "write")):
7519
7218
  return False
7520
7219
  if(verbose):
@@ -7557,9 +7256,9 @@ if(py7zr_support):
7557
7256
  if(verbose):
7558
7257
  VerbosePrintOut(fname)
7559
7258
  if(not member.is_directory):
7560
- fpremode = int(stat.S_IFREG + 438)
7259
+ fpremode = int(stat.S_IFREG | 0x1b6)
7561
7260
  elif(member.is_directory):
7562
- fpremode = int(stat.S_IFDIR + 511)
7261
+ fpremode = int(stat.S_IFDIR | 0x1ff)
7563
7262
  fwinattributes = format(int(0), 'x').lower()
7564
7263
  fcompression = ""
7565
7264
  fcsize = format(int(0), 'x').lower()
@@ -7583,17 +7282,17 @@ if(py7zr_support):
7583
7282
  fctime = format(int(member.creationtime.timestamp()), 'x').lower()
7584
7283
  fbtime = format(int(member.creationtime.timestamp()), 'x').lower()
7585
7284
  if(member.is_directory):
7586
- fmode = format(int(stat.S_IFDIR + 511), 'x').lower()
7285
+ fmode = format(int(stat.S_IFDIR | 0x1ff), 'x').lower()
7587
7286
  fchmode = format(
7588
- int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower()
7287
+ int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
7589
7288
  ftypemod = format(
7590
- int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower()
7289
+ int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff))), 'x').lower()
7591
7290
  else:
7592
- fmode = format(int(stat.S_IFREG + 438), 'x').lower()
7291
+ fmode = format(int(stat.S_IFREG | 0x1b6), 'x').lower()
7593
7292
  fchmode = format(
7594
- int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower()
7293
+ int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7595
7294
  ftypemod = format(
7596
- int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower()
7295
+ int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6))), 'x').lower()
7597
7296
  try:
7598
7297
  fuid = format(int(os.getuid()), 'x').lower()
7599
7298
  except AttributeError:
@@ -8232,11 +7931,11 @@ def AppendFilesWithContentFromRarFileToStackedOutFile(infiles, outfile, fmttype=
8232
7931
  return returnout
8233
7932
 
8234
7933
  if(not py7zr_support):
8235
- def AppendFilesWithContentFromSevenZipToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
7934
+ def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
8236
7935
  return False
8237
7936
 
8238
7937
  if(py7zr_support):
8239
- def AppendFilesWithContentFromSevenZipToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
7938
+ def AppendFilesWithContentFromSevenZipFileToOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
8240
7939
  if(IsNestedDict(formatspecs) and fmttype=="auto" and
8241
7940
  (outfile != "-" and outfile is not None and not hasattr(outfile, "read") and not hasattr(outfile, "write"))):
8242
7941
  get_in_ext = os.path.splitext(outfile)
@@ -8278,7 +7977,7 @@ if(py7zr_support):
8278
7977
  fp = CompressOpenFile(outfile, compresswholefile, compressionlevel)
8279
7978
  except PermissionError:
8280
7979
  return False
8281
- AppendFilesWithContentFromSevenZip(infiles, fp, extradata, jsondata, compression,
7980
+ AppendFilesWithContentFromSevenZipFile(infiles, fp, extradata, jsondata, compression,
8282
7981
  compresswholefile, compressionlevel, compressionuselist, checksumtype, formatspecs, verbose)
8283
7982
  if(outfile == "-" or outfile is None or hasattr(outfile, "read") or hasattr(outfile, "write")):
8284
7983
  fp = CompressOpenFileAlt(
@@ -8316,12 +8015,12 @@ if(py7zr_support):
8316
8015
  fp.close()
8317
8016
  return True
8318
8017
 
8319
- def AppendFilesWithContentFromSevenZipToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
8018
+ def AppendFilesWithContentFromSevenZipFileToStackedOutFile(infiles, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, extradata=[], jsondata={}, checksumtype=["crc32", "crc32", "crc32", "crc32"], formatspecs=__file_format_multi_dict__, verbose=False, returnfp=False):
8320
8019
  if not isinstance(infiles, list):
8321
8020
  infiles = [infiles]
8322
8021
  returnout = False
8323
8022
  for infileslist in infiles:
8324
- returnout = AppendFilesWithContentFromSevenZipToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
8023
+ returnout = AppendFilesWithContentFromSevenZipFileToOutFile(infileslist, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, True)
8325
8024
  if(not returnout):
8326
8025
  break
8327
8026
  else:
@@ -9025,8 +8724,6 @@ def CheckCompressionSubType(infile, formatspecs=__file_format_multi_dict__, file
9025
8724
  fp = pyzstd.zstdfile.ZstdFile(infile, mode="rb")
9026
8725
  else:
9027
8726
  return Flase
9028
- elif((compresscheck == "lzo" or compresscheck == "lzop") and compresscheck in compressionsupport):
9029
- fp = LzopFile(infile, mode="rb")
9030
8727
  elif((compresscheck == "lzma" or compresscheck == "xz") and compresscheck in compressionsupport):
9031
8728
  fp = lzma.open(infile, "rb")
9032
8729
  elif(compresscheck == "zlib" and compresscheck in compressionsupport):
@@ -9149,8 +8846,6 @@ def UncompressFileAlt(fp, formatspecs=__file_format_multi_dict__, filestart=0,
9149
8846
  return False
9150
8847
  elif kind == "lz4" and "lz4" in compressionsupport:
9151
8848
  wrapped = lz4.frame.LZ4FrameFile(src, mode="rb")
9152
- elif kind in ("lzo","lzop") and (("lzo" in compressionsupport) or ("lzop" in compressionsupport)):
9153
- wrapped = LzopFile(fileobj=src, mode="rb")
9154
8849
  elif kind == "zlib" and "zlib" in compressionsupport:
9155
8850
  wrapped = ZlibFile(fileobj=src, mode="rb")
9156
8851
  else:
@@ -9222,8 +8917,6 @@ def UncompressFile(infile, formatspecs=__file_format_multi_dict__, mode="rb",
9222
8917
  return False
9223
8918
  elif (compresscheck == "lz4" and "lz4" in compressionsupport):
9224
8919
  fp = lz4.frame.open(infile, mode)
9225
- elif ((compresscheck == "lzo" or compresscheck == "lzop") and "lzop" in compressionsupport):
9226
- fp = LzopFile(infile, mode=mode)
9227
8920
  elif ((compresscheck == "lzma" or compresscheck == "xz") and "xz" in compressionsupport):
9228
8921
  fp = lzma.open(infile, mode)
9229
8922
  elif (compresscheck == "zlib" and "zlib" in compressionsupport):
@@ -9791,7 +9484,7 @@ def fast_copy(infp, outfp, bufsize=1 << 20):
9791
9484
  outfp.write(data)
9792
9485
 
9793
9486
 
9794
- def copy_file_to_mmap_dest(src_path, outfp, chunk_size=8 << 20):
9487
+ def copy_file_to_mmap_dest(src_path, outfp, chunk_size=__spoolfile_size__):
9795
9488
  """
9796
9489
  Copy a disk file into an mmap-backed destination (FileLikeAdapter).
9797
9490
  Falls back to buffered copy if the source cannot be mmapped.
@@ -10017,9 +9710,6 @@ def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None,
10017
9710
  elif (fextname == ".lz4" and "lz4" in compressionsupport):
10018
9711
  outfp = FileLikeAdapter(lz4.frame.open(outfile, mode, compression_level=compressionlevel), mode="wb")
10019
9712
 
10020
- elif (fextname == ".lzo" and "lzop" in compressionsupport):
10021
- outfp = FileLikeAdapter(LzopFile(outfile, mode=mode, level=compressionlevel), mode="wb")
10022
-
10023
9713
  elif (fextname == ".lzma" and "lzma" in compressionsupport):
10024
9714
  try:
10025
9715
  outfp = FileLikeAdapter(
@@ -10141,7 +9831,7 @@ if(not py7zr_support):
10141
9831
 
10142
9832
  if(py7zr_support):
10143
9833
  def PackFoxFileFromSevenZipFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
10144
- return AppendFilesWithContentFromSevenZipToOutFile(infile, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, returnfp)
9834
+ return AppendFilesWithContentFromSevenZipFileToOutFile(infile, outfile, fmttype, compression, compresswholefile, compressionlevel, compressionuselist, extradata, jsondata, checksumtype, formatspecs, verbose, returnfp)
10145
9835
 
10146
9836
 
10147
9837
  def PackFoxFileFromInFile(infile, outfile, fmttype="auto", compression="auto", compresswholefile=True, compressionlevel=None, compressionuselist=compressionlistalt, checksumtype=["crc32", "crc32", "crc32"], extradata=[], jsondata={}, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
@@ -10249,14 +9939,6 @@ def FoxFileValidate(infile, fmttype="auto", filestart=0,
10249
9939
  checkcompressfile = CheckCompressionSubType(fp, formatspecs, filestart, True)
10250
9940
  if(IsNestedDict(formatspecs) and checkcompressfile in formatspecs):
10251
9941
  formatspecs = formatspecs[checkcompressfile]
10252
- if(checkcompressfile == "tarfile" and TarFileCheck(infile)):
10253
- return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
10254
- elif(checkcompressfile == "zipfile" and zipfile.is_zipfile(infile)):
10255
- return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
10256
- elif(rarfile_support and checkcompressfile == "rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
10257
- return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
10258
- elif(py7zr_support and checkcompressfile == "7zipfile" and py7zr.is_7zfile(infile)):
10259
- return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, seektoend, returnfp)
10260
9942
  elif(IsSingleDict(formatspecs) and checkcompressfile != formatspecs['format_magic']):
10261
9943
  return False
10262
9944
  elif(IsNestedDict(formatspecs) and checkcompressfile not in formatspecs):
@@ -10618,9 +10300,10 @@ def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__fi
10618
10300
  while True:
10619
10301
  if outstartfile >= outfsize: # stop when function signals False
10620
10302
  break
10621
- is_valid_file = FoxFileValidate(infile, fmttype, filestart, formatspecs, seektoend, verbose, True)
10303
+ is_valid_file = ArchiveFileValidate(infile, fmttype, outstartfile, formatspecs, seektoend, verbose, True)
10622
10304
  if is_valid_file is False: # stop when function signals False
10623
10305
  outretval.append(is_valid_file)
10306
+ break
10624
10307
  else:
10625
10308
  outretval.append(True)
10626
10309
  infile = is_valid_file
@@ -11626,35 +11309,6 @@ def MultipleFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, s
11626
11309
  return outretval
11627
11310
 
11628
11311
 
11629
- def StackedFoxFileValidate(infile, fmttype="auto", filestart=0, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, returnfp=False):
11630
- outretval = []
11631
- outstartfile = filestart
11632
- outfsize = float('inf')
11633
- while True:
11634
- if outstartfile >= outfsize: # stop when function signals False
11635
- break
11636
- is_valid_file = FoxFileValidate(infile, fmttype, filestart, formatspecs, seektoend, verbose, True)
11637
- if is_valid_file is False: # stop when function signals False
11638
- outretval.append(is_valid_file)
11639
- else:
11640
- outretval.append(True)
11641
- infile = is_valid_file
11642
- outstartfile = infile.tell()
11643
- try:
11644
- infile.seek(0, 2)
11645
- except OSError:
11646
- SeekToEndOfFile(infile)
11647
- except ValueError:
11648
- SeekToEndOfFile(infile)
11649
- outfsize = infile.tell()
11650
- infile.seek(outstartfile, 0)
11651
- if(returnfp):
11652
- return infile
11653
- else:
11654
- infile.close()
11655
- return outretval
11656
-
11657
-
11658
11312
  def StackedFoxFileListFiles(infile, fmttype="auto", filestart=0, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_multi_dict__, seektoend=False, verbose=False, newstyle=False, returnfp=False):
11659
11313
  outretval = []
11660
11314
  outstartfile = filestart
@@ -11865,11 +11519,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
11865
11519
  if(zipinfo.create_system == 0 or zipinfo.create_system == 10):
11866
11520
  fwinattributes = int(zipinfo.external_attr)
11867
11521
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
11868
- fmode = int(stat.S_IFDIR + 511)
11869
- fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)))
11870
- ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)))
11522
+ fmode = int(stat.S_IFDIR | 0x1ff)
11523
+ fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
11524
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
11871
11525
  else:
11872
- fmode = int(stat.S_IFREG + 438)
11526
+ fmode = int(stat.S_IFREG | 0x1b6)
11873
11527
  fchmode = int(stat.S_IMODE(fmode))
11874
11528
  ftypemod = int(stat.S_IFMT(fmode))
11875
11529
  elif(zipinfo.create_system == 3):
@@ -11885,11 +11539,11 @@ def ZipFileListFiles(infile, verbose=False, returnfp=False):
11885
11539
  else:
11886
11540
  fwinattributes = int(0)
11887
11541
  if ((hasattr(member, "is_dir") and member.is_dir()) or member.filename.endswith('/')):
11888
- fmode = int(stat.S_IFDIR + 511)
11889
- fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)))
11890
- ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)))
11542
+ fmode = int(stat.S_IFDIR | 0x1ff)
11543
+ fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
11544
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
11891
11545
  else:
11892
- fmode = int(stat.S_IFREG + 438)
11546
+ fmode = int(stat.S_IFREG | 0x1b6)
11893
11547
  fchmode = int(stat.S_IMODE(fmode))
11894
11548
  ftypemod = int(stat.S_IFMT(fmode))
11895
11549
  returnval.update({lcfi: member.filename})
@@ -12000,11 +11654,11 @@ if(rarfile_support):
12000
11654
  if(is_unix and member.external_attr != 0):
12001
11655
  fpremode = int(member.external_attr)
12002
11656
  elif(member.is_file()):
12003
- fpremode = int(stat.S_IFREG + 438)
11657
+ fpremode = int(stat.S_IFREG | 0x1b6)
12004
11658
  elif(member.is_symlink()):
12005
- fpremode = int(stat.S_IFLNK + 438)
11659
+ fpremode = int(stat.S_IFLNK | 0x1b6)
12006
11660
  elif(member.is_dir()):
12007
- fpremode = int(stat.S_IFDIR + 511)
11661
+ fpremode = int(stat.S_IFDIR | 0x1ff)
12008
11662
  if(is_windows and member.external_attr != 0):
12009
11663
  fwinattributes = int(member.external_attr)
12010
11664
  else:
@@ -12014,17 +11668,17 @@ if(rarfile_support):
12014
11668
  fchmode = int(stat.S_IMODE(member.external_attr))
12015
11669
  ftypemod = int(stat.S_IFMT(member.external_attr))
12016
11670
  elif(member.is_file()):
12017
- fmode = int(stat.S_IFREG + 438)
12018
- fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)))
12019
- ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)))
11671
+ fmode = int(stat.S_IFREG | 0x1b6)
11672
+ fchmode = int(stat.S_IMODE(int(stat.S_IFREG | 0x1b6)))
11673
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFREG | 0x1b6)))
12020
11674
  elif(member.is_symlink()):
12021
- fmode = int(stat.S_IFLNK + 438)
12022
- fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)))
12023
- ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)))
11675
+ fmode = int(stat.S_IFLNK | 0x1b6)
11676
+ fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
11677
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
12024
11678
  elif(member.is_dir()):
12025
- fmode = int(stat.S_IFDIR + 511)
12026
- fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)))
12027
- ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)))
11679
+ fmode = int(stat.S_IFDIR | 0x1ff)
11680
+ fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
11681
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
12028
11682
  returnval.update({lcfi: member.filename})
12029
11683
  if(not verbose):
12030
11684
  VerbosePrintOut(member.filename)
@@ -12120,18 +11774,18 @@ if(py7zr_support):
12120
11774
  else:
12121
11775
  fname = "./"+member.filename
12122
11776
  if(not member.is_directory):
12123
- fpremode = int(stat.S_IFREG + 438)
11777
+ fpremode = int(stat.S_IFREG | 0x1b6)
12124
11778
  elif(member.is_directory):
12125
- fpremode = int(stat.S_IFDIR + 511)
11779
+ fpremode = int(stat.S_IFDIR | 0x1ff)
12126
11780
  fwinattributes = int(0)
12127
11781
  if(member.is_directory):
12128
- fmode = int(stat.S_IFDIR + 511)
12129
- fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)))
12130
- ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)))
11782
+ fmode = int(stat.S_IFDIR | 0x1ff)
11783
+ fchmode = int(stat.S_IMODE(int(stat.S_IFDIR | 0x1ff)))
11784
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR | 0x1ff)))
12131
11785
  else:
12132
- fmode = int(stat.S_IFLNK + 438)
12133
- fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)))
12134
- ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)))
11786
+ fmode = int(stat.S_IFLNK | 0x1b6)
11787
+ fchmode = int(stat.S_IMODE(int(stat.S_IFLNK | 0x1b6)))
11788
+ ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK | 0x1b6)))
12135
11789
  returnval.update({lcfi: member.filename})
12136
11790
  if(not verbose):
12137
11791
  VerbosePrintOut(member.filename)