exonware-xwsystem 0.0.1.409__py3-none-any.whl → 0.0.1.411__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exonware/__init__.py +2 -2
- exonware/conf.py +10 -20
- exonware/xwsystem/__init__.py +6 -16
- exonware/xwsystem/caching/__init__.py +1 -1
- exonware/xwsystem/caching/base.py +16 -16
- exonware/xwsystem/caching/bloom_cache.py +5 -5
- exonware/xwsystem/caching/cache_manager.py +2 -2
- exonware/xwsystem/caching/conditional.py +4 -4
- exonware/xwsystem/caching/contracts.py +12 -12
- exonware/xwsystem/caching/decorators.py +2 -2
- exonware/xwsystem/caching/defs.py +1 -1
- exonware/xwsystem/caching/disk_cache.py +4 -4
- exonware/xwsystem/caching/distributed.py +1 -1
- exonware/xwsystem/caching/errors.py +1 -1
- exonware/xwsystem/caching/events.py +8 -8
- exonware/xwsystem/caching/eviction_strategies.py +9 -9
- exonware/xwsystem/caching/fluent.py +1 -1
- exonware/xwsystem/caching/integrity.py +1 -1
- exonware/xwsystem/caching/lfu_cache.py +24 -9
- exonware/xwsystem/caching/lfu_optimized.py +21 -21
- exonware/xwsystem/caching/lru_cache.py +14 -7
- exonware/xwsystem/caching/memory_bounded.py +8 -8
- exonware/xwsystem/caching/metrics_exporter.py +6 -6
- exonware/xwsystem/caching/observable_cache.py +1 -1
- exonware/xwsystem/caching/pluggable_cache.py +9 -9
- exonware/xwsystem/caching/rate_limiter.py +1 -1
- exonware/xwsystem/caching/read_through.py +6 -6
- exonware/xwsystem/caching/secure_cache.py +1 -1
- exonware/xwsystem/caching/serializable.py +3 -3
- exonware/xwsystem/caching/stats.py +7 -7
- exonware/xwsystem/caching/tagging.py +11 -11
- exonware/xwsystem/caching/ttl_cache.py +21 -6
- exonware/xwsystem/caching/two_tier_cache.py +5 -5
- exonware/xwsystem/caching/utils.py +3 -3
- exonware/xwsystem/caching/validation.py +1 -1
- exonware/xwsystem/caching/warming.py +9 -9
- exonware/xwsystem/caching/write_behind.py +5 -5
- exonware/xwsystem/cli/__init__.py +1 -1
- exonware/xwsystem/cli/args.py +10 -10
- exonware/xwsystem/cli/base.py +15 -15
- exonware/xwsystem/cli/colors.py +1 -1
- exonware/xwsystem/cli/console.py +1 -1
- exonware/xwsystem/cli/contracts.py +5 -5
- exonware/xwsystem/cli/defs.py +1 -1
- exonware/xwsystem/cli/errors.py +1 -1
- exonware/xwsystem/cli/progress.py +1 -1
- exonware/xwsystem/cli/prompts.py +1 -1
- exonware/xwsystem/cli/tables.py +7 -7
- exonware/xwsystem/config/__init__.py +1 -1
- exonware/xwsystem/config/base.py +14 -14
- exonware/xwsystem/config/contracts.py +22 -22
- exonware/xwsystem/config/defaults.py +2 -2
- exonware/xwsystem/config/defs.py +1 -1
- exonware/xwsystem/config/errors.py +2 -2
- exonware/xwsystem/config/logging.py +1 -1
- exonware/xwsystem/config/logging_setup.py +2 -2
- exonware/xwsystem/config/performance.py +7 -7
- exonware/xwsystem/config/performance_modes.py +20 -20
- exonware/xwsystem/config/version_manager.py +4 -4
- exonware/xwsystem/{http → http_client}/__init__.py +1 -1
- exonware/xwsystem/{http → http_client}/advanced_client.py +20 -20
- exonware/xwsystem/{http → http_client}/base.py +13 -13
- exonware/xwsystem/{http → http_client}/client.py +43 -43
- exonware/xwsystem/{http → http_client}/contracts.py +5 -5
- exonware/xwsystem/{http → http_client}/defs.py +2 -2
- exonware/xwsystem/{http → http_client}/errors.py +2 -2
- exonware/xwsystem/io/__init__.py +1 -1
- exonware/xwsystem/io/archive/__init__.py +1 -1
- exonware/xwsystem/io/archive/archive.py +5 -5
- exonware/xwsystem/io/archive/archive_files.py +8 -8
- exonware/xwsystem/io/archive/archivers.py +3 -3
- exonware/xwsystem/io/archive/base.py +17 -17
- exonware/xwsystem/io/archive/codec_integration.py +1 -1
- exonware/xwsystem/io/archive/compression.py +1 -1
- exonware/xwsystem/io/archive/formats/__init__.py +1 -1
- exonware/xwsystem/io/archive/formats/brotli_format.py +12 -9
- exonware/xwsystem/io/archive/formats/lz4_format.py +12 -9
- exonware/xwsystem/io/archive/formats/rar.py +12 -9
- exonware/xwsystem/io/archive/formats/sevenzip.py +12 -9
- exonware/xwsystem/io/archive/formats/squashfs_format.py +7 -7
- exonware/xwsystem/io/archive/formats/tar.py +8 -8
- exonware/xwsystem/io/archive/formats/wim_format.py +12 -9
- exonware/xwsystem/io/archive/formats/zip.py +8 -8
- exonware/xwsystem/io/archive/formats/zpaq_format.py +7 -7
- exonware/xwsystem/io/archive/formats/zstandard.py +12 -9
- exonware/xwsystem/io/base.py +17 -17
- exonware/xwsystem/io/codec/__init__.py +1 -1
- exonware/xwsystem/io/codec/base.py +261 -14
- exonware/xwsystem/io/codec/contracts.py +3 -6
- exonware/xwsystem/io/codec/registry.py +29 -29
- exonware/xwsystem/io/common/__init__.py +1 -1
- exonware/xwsystem/io/common/atomic.py +2 -2
- exonware/xwsystem/io/common/base.py +1 -1
- exonware/xwsystem/io/common/lock.py +1 -1
- exonware/xwsystem/io/common/watcher.py +4 -4
- exonware/xwsystem/io/contracts.py +34 -39
- exonware/xwsystem/io/data_operations.py +480 -0
- exonware/xwsystem/io/defs.py +2 -2
- exonware/xwsystem/io/errors.py +32 -3
- exonware/xwsystem/io/facade.py +4 -4
- exonware/xwsystem/io/file/__init__.py +1 -1
- exonware/xwsystem/io/file/base.py +2 -2
- exonware/xwsystem/io/file/conversion.py +1 -1
- exonware/xwsystem/io/file/file.py +10 -8
- exonware/xwsystem/io/file/paged_source.py +8 -1
- exonware/xwsystem/io/file/paging/__init__.py +1 -1
- exonware/xwsystem/io/file/paging/byte_paging.py +1 -1
- exonware/xwsystem/io/file/paging/line_paging.py +1 -1
- exonware/xwsystem/io/file/paging/record_paging.py +1 -1
- exonware/xwsystem/io/file/paging/registry.py +5 -5
- exonware/xwsystem/io/file/source.py +22 -11
- exonware/xwsystem/io/filesystem/__init__.py +1 -1
- exonware/xwsystem/io/filesystem/base.py +1 -1
- exonware/xwsystem/io/filesystem/local.py +9 -1
- exonware/xwsystem/io/folder/__init__.py +1 -1
- exonware/xwsystem/io/folder/base.py +2 -2
- exonware/xwsystem/io/folder/folder.py +6 -6
- exonware/xwsystem/io/serialization/__init__.py +1 -1
- exonware/xwsystem/io/serialization/auto_serializer.py +53 -40
- exonware/xwsystem/io/serialization/base.py +248 -35
- exonware/xwsystem/io/serialization/contracts.py +93 -4
- exonware/xwsystem/io/serialization/defs.py +1 -1
- exonware/xwsystem/io/serialization/errors.py +1 -1
- exonware/xwsystem/io/serialization/flyweight.py +22 -22
- exonware/xwsystem/io/serialization/format_detector.py +18 -15
- exonware/xwsystem/io/serialization/formats/__init__.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/bson.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/cbor.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/marshal.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/msgpack.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/pickle.py +1 -1
- exonware/xwsystem/io/serialization/formats/binary/plistlib.py +1 -1
- exonware/xwsystem/io/serialization/formats/database/dbm.py +53 -1
- exonware/xwsystem/io/serialization/formats/database/shelve.py +48 -1
- exonware/xwsystem/io/serialization/formats/database/sqlite3.py +85 -1
- exonware/xwsystem/io/serialization/formats/text/configparser.py +2 -2
- exonware/xwsystem/io/serialization/formats/text/csv.py +2 -2
- exonware/xwsystem/io/serialization/formats/text/formdata.py +2 -2
- exonware/xwsystem/io/serialization/formats/text/json.py +23 -5
- exonware/xwsystem/io/serialization/formats/text/json5.py +98 -13
- exonware/xwsystem/io/serialization/formats/text/jsonlines.py +230 -20
- exonware/xwsystem/io/serialization/formats/text/multipart.py +2 -2
- exonware/xwsystem/io/serialization/formats/text/toml.py +65 -4
- exonware/xwsystem/io/serialization/formats/text/xml.py +451 -69
- exonware/xwsystem/io/serialization/formats/text/yaml.py +52 -2
- exonware/xwsystem/io/serialization/registry.py +5 -5
- exonware/xwsystem/io/serialization/serializer.py +184 -12
- exonware/xwsystem/io/serialization/utils/__init__.py +1 -1
- exonware/xwsystem/io/serialization/utils/path_ops.py +3 -3
- exonware/xwsystem/io/stream/__init__.py +1 -1
- exonware/xwsystem/io/stream/async_operations.py +3 -3
- exonware/xwsystem/io/stream/base.py +3 -7
- exonware/xwsystem/io/stream/codec_io.py +4 -7
- exonware/xwsystem/ipc/async_fabric.py +7 -8
- exonware/xwsystem/ipc/base.py +9 -9
- exonware/xwsystem/ipc/contracts.py +5 -5
- exonware/xwsystem/ipc/defs.py +1 -1
- exonware/xwsystem/ipc/errors.py +2 -2
- exonware/xwsystem/ipc/message_queue.py +4 -6
- exonware/xwsystem/ipc/pipes.py +2 -2
- exonware/xwsystem/ipc/process_manager.py +7 -7
- exonware/xwsystem/ipc/process_pool.py +8 -8
- exonware/xwsystem/ipc/shared_memory.py +7 -7
- exonware/xwsystem/monitoring/base.py +33 -33
- exonware/xwsystem/monitoring/contracts.py +27 -27
- exonware/xwsystem/monitoring/defs.py +1 -1
- exonware/xwsystem/monitoring/error_recovery.py +16 -16
- exonware/xwsystem/monitoring/errors.py +2 -2
- exonware/xwsystem/monitoring/memory_monitor.py +12 -12
- exonware/xwsystem/monitoring/metrics.py +8 -8
- exonware/xwsystem/monitoring/performance_manager_generic.py +20 -20
- exonware/xwsystem/monitoring/performance_monitor.py +11 -11
- exonware/xwsystem/monitoring/performance_validator.py +21 -21
- exonware/xwsystem/monitoring/system_monitor.py +17 -17
- exonware/xwsystem/monitoring/tracing.py +20 -20
- exonware/xwsystem/monitoring/tracker.py +7 -7
- exonware/xwsystem/operations/__init__.py +5 -5
- exonware/xwsystem/operations/base.py +3 -3
- exonware/xwsystem/operations/contracts.py +3 -3
- exonware/xwsystem/operations/defs.py +5 -5
- exonware/xwsystem/operations/diff.py +5 -5
- exonware/xwsystem/operations/merge.py +2 -2
- exonware/xwsystem/operations/patch.py +5 -5
- exonware/xwsystem/patterns/base.py +4 -4
- exonware/xwsystem/patterns/context_manager.py +7 -7
- exonware/xwsystem/patterns/contracts.py +29 -31
- exonware/xwsystem/patterns/defs.py +1 -1
- exonware/xwsystem/patterns/dynamic_facade.py +9 -9
- exonware/xwsystem/patterns/errors.py +10 -10
- exonware/xwsystem/patterns/handler_factory.py +15 -14
- exonware/xwsystem/patterns/import_registry.py +22 -22
- exonware/xwsystem/patterns/object_pool.py +14 -13
- exonware/xwsystem/patterns/registry.py +45 -32
- exonware/xwsystem/plugins/__init__.py +1 -1
- exonware/xwsystem/plugins/base.py +25 -25
- exonware/xwsystem/plugins/contracts.py +28 -28
- exonware/xwsystem/plugins/defs.py +1 -1
- exonware/xwsystem/plugins/errors.py +9 -9
- exonware/xwsystem/runtime/__init__.py +1 -1
- exonware/xwsystem/runtime/base.py +42 -42
- exonware/xwsystem/runtime/contracts.py +9 -9
- exonware/xwsystem/runtime/defs.py +1 -1
- exonware/xwsystem/runtime/env.py +9 -9
- exonware/xwsystem/runtime/errors.py +1 -1
- exonware/xwsystem/runtime/reflection.py +15 -15
- exonware/xwsystem/security/auth.py +47 -15
- exonware/xwsystem/security/base.py +17 -17
- exonware/xwsystem/security/contracts.py +30 -30
- exonware/xwsystem/security/crypto.py +8 -8
- exonware/xwsystem/security/defs.py +1 -1
- exonware/xwsystem/security/errors.py +2 -2
- exonware/xwsystem/security/hazmat.py +7 -7
- exonware/xwsystem/security/path_validator.py +1 -1
- exonware/xwsystem/shared/__init__.py +1 -1
- exonware/xwsystem/shared/base.py +14 -14
- exonware/xwsystem/shared/contracts.py +6 -6
- exonware/xwsystem/shared/defs.py +1 -1
- exonware/xwsystem/shared/errors.py +1 -1
- exonware/xwsystem/structures/__init__.py +1 -1
- exonware/xwsystem/structures/base.py +29 -29
- exonware/xwsystem/structures/circular_detector.py +15 -15
- exonware/xwsystem/structures/contracts.py +9 -9
- exonware/xwsystem/structures/defs.py +1 -1
- exonware/xwsystem/structures/errors.py +2 -2
- exonware/xwsystem/structures/tree_walker.py +8 -8
- exonware/xwsystem/threading/async_primitives.py +7 -7
- exonware/xwsystem/threading/base.py +19 -19
- exonware/xwsystem/threading/contracts.py +13 -13
- exonware/xwsystem/threading/defs.py +1 -1
- exonware/xwsystem/threading/errors.py +2 -2
- exonware/xwsystem/threading/safe_factory.py +13 -12
- exonware/xwsystem/utils/base.py +34 -34
- exonware/xwsystem/utils/contracts.py +9 -9
- exonware/xwsystem/utils/dt/__init__.py +1 -1
- exonware/xwsystem/utils/dt/base.py +6 -6
- exonware/xwsystem/utils/dt/contracts.py +2 -2
- exonware/xwsystem/utils/dt/defs.py +1 -1
- exonware/xwsystem/utils/dt/errors.py +2 -2
- exonware/xwsystem/utils/dt/formatting.py +3 -3
- exonware/xwsystem/utils/dt/humanize.py +2 -2
- exonware/xwsystem/utils/dt/parsing.py +2 -2
- exonware/xwsystem/utils/dt/timezone_utils.py +5 -5
- exonware/xwsystem/utils/errors.py +2 -2
- exonware/xwsystem/utils/test_runner.py +6 -6
- exonware/xwsystem/utils/utils_contracts.py +1 -1
- exonware/xwsystem/validation/__init__.py +1 -1
- exonware/xwsystem/validation/base.py +48 -48
- exonware/xwsystem/validation/contracts.py +8 -8
- exonware/xwsystem/validation/data_validator.py +10 -0
- exonware/xwsystem/validation/declarative.py +15 -15
- exonware/xwsystem/validation/defs.py +1 -1
- exonware/xwsystem/validation/errors.py +2 -2
- exonware/xwsystem/validation/fluent_validator.py +10 -10
- exonware/xwsystem/version.py +2 -2
- {exonware_xwsystem-0.0.1.409.dist-info → exonware_xwsystem-0.0.1.411.dist-info}/METADATA +9 -11
- exonware_xwsystem-0.0.1.411.dist-info/RECORD +274 -0
- {exonware_xwsystem-0.0.1.409.dist-info → exonware_xwsystem-0.0.1.411.dist-info}/WHEEL +1 -1
- exonware/xwsystem/lazy_bootstrap.py +0 -79
- exonware_xwsystem-0.0.1.409.dist-info/RECORD +0 -274
- {exonware_xwsystem-0.0.1.409.dist-info → exonware_xwsystem-0.0.1.411.dist-info}/licenses/LICENSE +0 -0
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
Company: eXonware.com
|
|
5
5
|
Author: Eng. Muhammad AlShehri
|
|
6
6
|
Email: connect@exonware.com
|
|
7
|
-
Version: 0.0.1.
|
|
7
|
+
Version: 0.0.1.411
|
|
8
8
|
Generation Date: 30-Oct-2025
|
|
9
9
|
|
|
10
10
|
ZIP archive format implementation.
|
|
@@ -18,7 +18,7 @@ Priority 5 (Extensibility): Registered via registry
|
|
|
18
18
|
|
|
19
19
|
import zipfile
|
|
20
20
|
from pathlib import Path
|
|
21
|
-
from typing import
|
|
21
|
+
from typing import Optional
|
|
22
22
|
|
|
23
23
|
from ...contracts import IArchiveFormat
|
|
24
24
|
|
|
@@ -32,16 +32,16 @@ class ZipArchiver(IArchiveFormat):
|
|
|
32
32
|
return "zip"
|
|
33
33
|
|
|
34
34
|
@property
|
|
35
|
-
def file_extensions(self) ->
|
|
35
|
+
def file_extensions(self) -> list[str]:
|
|
36
36
|
"""Supported extensions."""
|
|
37
37
|
return [".zip", ".jar", ".war", ".ear"]
|
|
38
38
|
|
|
39
39
|
@property
|
|
40
|
-
def mime_types(self) ->
|
|
40
|
+
def mime_types(self) -> list[str]:
|
|
41
41
|
"""MIME types."""
|
|
42
42
|
return ["application/zip", "application/java-archive"]
|
|
43
43
|
|
|
44
|
-
def create(self, files:
|
|
44
|
+
def create(self, files: list[Path], output: Path, **opts) -> None:
|
|
45
45
|
"""Create ZIP archive."""
|
|
46
46
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
47
47
|
|
|
@@ -60,10 +60,10 @@ class ZipArchiver(IArchiveFormat):
|
|
|
60
60
|
arcname = str(item.relative_to(file_path.parent))
|
|
61
61
|
zf.write(item, arcname=arcname)
|
|
62
62
|
|
|
63
|
-
def extract(self, archive: Path, output_dir: Path, members: Optional[
|
|
63
|
+
def extract(self, archive: Path, output_dir: Path, members: Optional[list[str]] = None, **opts) -> list[Path]:
|
|
64
64
|
"""Extract ZIP archive."""
|
|
65
65
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
66
|
-
extracted:
|
|
66
|
+
extracted: list[Path] = []
|
|
67
67
|
|
|
68
68
|
with zipfile.ZipFile(archive, 'r') as zf:
|
|
69
69
|
if members:
|
|
@@ -76,7 +76,7 @@ class ZipArchiver(IArchiveFormat):
|
|
|
76
76
|
|
|
77
77
|
return extracted
|
|
78
78
|
|
|
79
|
-
def list_contents(self, archive: Path) ->
|
|
79
|
+
def list_contents(self, archive: Path) -> list[str]:
|
|
80
80
|
"""List ZIP contents."""
|
|
81
81
|
with zipfile.ZipFile(archive, 'r') as zf:
|
|
82
82
|
return zf.namelist()
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
Company: eXonware.com
|
|
5
5
|
Author: Eng. Muhammad AlShehri
|
|
6
6
|
Email: connect@exonware.com
|
|
7
|
-
Version: 0.0.1.
|
|
7
|
+
Version: 0.0.1.411
|
|
8
8
|
Generation Date: November 1, 2025
|
|
9
9
|
|
|
10
10
|
ZPAQ journaled compression format - RANK #8 EXTREME COMPRESSION.
|
|
@@ -19,7 +19,7 @@ Priority 5 (Extensibility): Lazy installation of zpaq
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
from pathlib import Path
|
|
22
|
-
from typing import
|
|
22
|
+
from typing import Optional
|
|
23
23
|
import subprocess
|
|
24
24
|
import shutil
|
|
25
25
|
|
|
@@ -56,12 +56,12 @@ class ZpaqArchiver(IArchiveFormat):
|
|
|
56
56
|
return "zpaq"
|
|
57
57
|
|
|
58
58
|
@property
|
|
59
|
-
def file_extensions(self) ->
|
|
59
|
+
def file_extensions(self) -> list[str]:
|
|
60
60
|
"""Supported extensions."""
|
|
61
61
|
return [".zpaq"]
|
|
62
62
|
|
|
63
63
|
@property
|
|
64
|
-
def mime_types(self) ->
|
|
64
|
+
def mime_types(self) -> list[str]:
|
|
65
65
|
"""MIME types."""
|
|
66
66
|
return ["application/x-zpaq"]
|
|
67
67
|
|
|
@@ -74,7 +74,7 @@ class ZpaqArchiver(IArchiveFormat):
|
|
|
74
74
|
)
|
|
75
75
|
return Path(zpaq_path)
|
|
76
76
|
|
|
77
|
-
def create(self, files:
|
|
77
|
+
def create(self, files: list[Path], output: Path, **opts) -> None:
|
|
78
78
|
"""
|
|
79
79
|
Create ZPAQ archive.
|
|
80
80
|
|
|
@@ -102,7 +102,7 @@ class ZpaqArchiver(IArchiveFormat):
|
|
|
102
102
|
except Exception as e:
|
|
103
103
|
raise ArchiveError(f"Failed to create zpaq archive: {e}")
|
|
104
104
|
|
|
105
|
-
def extract(self, archive: Path, output_dir: Path, members: Optional[
|
|
105
|
+
def extract(self, archive: Path, output_dir: Path, members: Optional[list[str]] = None, **opts) -> list[Path]:
|
|
106
106
|
"""Extract ZPAQ archive."""
|
|
107
107
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
108
108
|
zpaq = self._check_zpaq()
|
|
@@ -123,7 +123,7 @@ class ZpaqArchiver(IArchiveFormat):
|
|
|
123
123
|
except Exception as e:
|
|
124
124
|
raise ArchiveError(f"Failed to extract zpaq archive: {e}")
|
|
125
125
|
|
|
126
|
-
def list_contents(self, archive: Path) ->
|
|
126
|
+
def list_contents(self, archive: Path) -> list[str]:
|
|
127
127
|
"""List ZPAQ contents."""
|
|
128
128
|
zpaq = self._check_zpaq()
|
|
129
129
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
Company: eXonware.com
|
|
5
5
|
Author: Eng. Muhammad AlShehri
|
|
6
6
|
Email: connect@exonware.com
|
|
7
|
-
Version: 0.0.1.
|
|
7
|
+
Version: 0.0.1.411
|
|
8
8
|
Generation Date: November 1, 2025
|
|
9
9
|
|
|
10
10
|
Zstandard (.zst) compression format - RANK #2 MODERN STANDARD.
|
|
@@ -20,13 +20,16 @@ Priority 5 (Extensibility): Lazy installation of zstandard
|
|
|
20
20
|
|
|
21
21
|
import tarfile
|
|
22
22
|
from pathlib import Path
|
|
23
|
-
from typing import
|
|
23
|
+
from typing import Optional
|
|
24
24
|
|
|
25
25
|
from ...contracts import IArchiveFormat
|
|
26
26
|
from ...errors import ArchiveError
|
|
27
27
|
|
|
28
|
-
# Lazy import for zstandard -
|
|
29
|
-
|
|
28
|
+
# Lazy import for zstandard - optional dependency
|
|
29
|
+
try:
|
|
30
|
+
import zstandard
|
|
31
|
+
except ImportError:
|
|
32
|
+
zstandard = None # type: ignore
|
|
30
33
|
|
|
31
34
|
|
|
32
35
|
class ZstandardArchiver(IArchiveFormat):
|
|
@@ -59,16 +62,16 @@ class ZstandardArchiver(IArchiveFormat):
|
|
|
59
62
|
return "zst"
|
|
60
63
|
|
|
61
64
|
@property
|
|
62
|
-
def file_extensions(self) ->
|
|
65
|
+
def file_extensions(self) -> list[str]:
|
|
63
66
|
"""Supported extensions."""
|
|
64
67
|
return [".zst", ".tar.zst", ".tzst"]
|
|
65
68
|
|
|
66
69
|
@property
|
|
67
|
-
def mime_types(self) ->
|
|
70
|
+
def mime_types(self) -> list[str]:
|
|
68
71
|
"""MIME types."""
|
|
69
72
|
return ["application/zstd", "application/x-zstd"]
|
|
70
73
|
|
|
71
|
-
def create(self, files:
|
|
74
|
+
def create(self, files: list[Path], output: Path, **opts) -> None:
|
|
72
75
|
"""
|
|
73
76
|
Create Zstandard-compressed tar archive.
|
|
74
77
|
|
|
@@ -98,7 +101,7 @@ class ZstandardArchiver(IArchiveFormat):
|
|
|
98
101
|
except Exception as e:
|
|
99
102
|
raise ArchiveError(f"Failed to create zst archive: {e}")
|
|
100
103
|
|
|
101
|
-
def extract(self, archive: Path, output_dir: Path, members: Optional[
|
|
104
|
+
def extract(self, archive: Path, output_dir: Path, members: Optional[list[str]] = None, **opts) -> list[Path]:
|
|
102
105
|
"""Extract Zstandard archive."""
|
|
103
106
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
104
107
|
|
|
@@ -122,7 +125,7 @@ class ZstandardArchiver(IArchiveFormat):
|
|
|
122
125
|
except Exception as e:
|
|
123
126
|
raise ArchiveError(f"Failed to extract zst archive: {e}")
|
|
124
127
|
|
|
125
|
-
def list_contents(self, archive: Path) ->
|
|
128
|
+
def list_contents(self, archive: Path) -> list[str]:
|
|
126
129
|
"""List Zstandard archive contents."""
|
|
127
130
|
try:
|
|
128
131
|
dctx = zstandard.ZstdDecompressor()
|
exonware/xwsystem/io/base.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Company: eXonware.com
|
|
3
3
|
Author: Eng. Muhammad AlShehri
|
|
4
4
|
Email: connect@exonware.com
|
|
5
|
-
Version: 0.0.1.
|
|
5
|
+
Version: 0.0.1.411
|
|
6
6
|
Generation Date: September 04, 2025
|
|
7
7
|
|
|
8
8
|
IO module base classes - abstract classes for input/output functionality.
|
|
@@ -11,7 +11,7 @@ IO module base classes - abstract classes for input/output functionality.
|
|
|
11
11
|
import os
|
|
12
12
|
import time
|
|
13
13
|
from abc import ABC, abstractmethod
|
|
14
|
-
from typing import Any,
|
|
14
|
+
from typing import Any, Optional, Union, BinaryIO, TextIO
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
from .contracts import FileMode, FileType, PathType, OperationResult, LockType, IFile, IFolder, IPath, IStream, IAsyncIO, IAtomicOperations, IBackupOperations, ITemporaryOperations, IUnifiedIO, IFileManager
|
|
17
17
|
|
|
@@ -354,15 +354,15 @@ class AFolder(IFolder, ABC):
|
|
|
354
354
|
"""Delete directory."""
|
|
355
355
|
pass
|
|
356
356
|
|
|
357
|
-
def list_files(self, pattern: Optional[str] = None, recursive: bool = False) ->
|
|
357
|
+
def list_files(self, pattern: Optional[str] = None, recursive: bool = False) -> list[Path]:
|
|
358
358
|
"""List files in directory."""
|
|
359
359
|
return AFolder.list_files_static(self.dir_path, pattern, recursive)
|
|
360
360
|
|
|
361
|
-
def list_directories(self, recursive: bool = False) ->
|
|
361
|
+
def list_directories(self, recursive: bool = False) -> list[Path]:
|
|
362
362
|
"""List subdirectories."""
|
|
363
363
|
return AFolder.list_directories_static(self.dir_path, recursive)
|
|
364
364
|
|
|
365
|
-
def walk(self) ->
|
|
365
|
+
def walk(self) -> list[tuple[Path, list[str], list[str]]]:
|
|
366
366
|
"""Walk directory tree."""
|
|
367
367
|
return AFolder.walk_static(self.dir_path)
|
|
368
368
|
|
|
@@ -414,7 +414,7 @@ class AFolder(IFolder, ABC):
|
|
|
414
414
|
return False
|
|
415
415
|
|
|
416
416
|
@staticmethod
|
|
417
|
-
def list_files_static(path: Union[str, Path], pattern: Optional[str] = None, recursive: bool = False) ->
|
|
417
|
+
def list_files_static(path: Union[str, Path], pattern: Optional[str] = None, recursive: bool = False) -> list[Path]:
|
|
418
418
|
"""List files in directory."""
|
|
419
419
|
if not AFolder.exists(path):
|
|
420
420
|
return []
|
|
@@ -431,7 +431,7 @@ class AFolder(IFolder, ABC):
|
|
|
431
431
|
return [p for p in Path(path).iterdir() if p.is_file()]
|
|
432
432
|
|
|
433
433
|
@staticmethod
|
|
434
|
-
def list_directories_static(path: Union[str, Path], recursive: bool = False) ->
|
|
434
|
+
def list_directories_static(path: Union[str, Path], recursive: bool = False) -> list[Path]:
|
|
435
435
|
"""List subdirectories."""
|
|
436
436
|
if not AFolder.exists(path):
|
|
437
437
|
return []
|
|
@@ -442,7 +442,7 @@ class AFolder(IFolder, ABC):
|
|
|
442
442
|
return [p for p in Path(path).iterdir() if p.is_dir()]
|
|
443
443
|
|
|
444
444
|
@staticmethod
|
|
445
|
-
def walk_static(path: Union[str, Path]) ->
|
|
445
|
+
def walk_static(path: Union[str, Path]) -> list[tuple[Path, list[str], list[str]]]:
|
|
446
446
|
"""Walk directory tree."""
|
|
447
447
|
if not AFolder.exists(path):
|
|
448
448
|
return []
|
|
@@ -935,7 +935,7 @@ class ABackupOperations(IBackupOperations, ABC):
|
|
|
935
935
|
"""Restore from backup."""
|
|
936
936
|
pass
|
|
937
937
|
|
|
938
|
-
def list_backups(self, backup_dir: Union[str, Path]) ->
|
|
938
|
+
def list_backups(self, backup_dir: Union[str, Path]) -> list[Path]:
|
|
939
939
|
"""List available backups."""
|
|
940
940
|
return ABackupOperations.list_backups_static(backup_dir)
|
|
941
941
|
|
|
@@ -1000,7 +1000,7 @@ class ABackupOperations(IBackupOperations, ABC):
|
|
|
1000
1000
|
return OperationResult.FAILED
|
|
1001
1001
|
|
|
1002
1002
|
@staticmethod
|
|
1003
|
-
def list_backups_static(backup_dir: Union[str, Path]) ->
|
|
1003
|
+
def list_backups_static(backup_dir: Union[str, Path]) -> list[Path]:
|
|
1004
1004
|
"""List available backups."""
|
|
1005
1005
|
try:
|
|
1006
1006
|
backup_path = Path(backup_dir)
|
|
@@ -1051,8 +1051,8 @@ class ATemporaryOperations(ITemporaryOperations, ABC):
|
|
|
1051
1051
|
|
|
1052
1052
|
def __init__(self):
|
|
1053
1053
|
"""Initialize temporary operations base."""
|
|
1054
|
-
self._temp_files:
|
|
1055
|
-
self._temp_dirs:
|
|
1054
|
+
self._temp_files: list[Path] = []
|
|
1055
|
+
self._temp_dirs: list[Path] = []
|
|
1056
1056
|
self._temp_base_dir: Optional[Path] = None
|
|
1057
1057
|
|
|
1058
1058
|
# ============================================================================
|
|
@@ -1189,8 +1189,8 @@ class AUnifiedIO(AFile, AFolder, APath, AStream, AAsyncIO, AAtomicOperations, AB
|
|
|
1189
1189
|
self.cleanup_temp_on_exit = config.get('cleanup_temp_on_exit', True)
|
|
1190
1190
|
|
|
1191
1191
|
# Track temporary files for cleanup
|
|
1192
|
-
self._temp_files:
|
|
1193
|
-
self._temp_dirs:
|
|
1192
|
+
self._temp_files: list[Path] = []
|
|
1193
|
+
self._temp_dirs: list[Path] = []
|
|
1194
1194
|
|
|
1195
1195
|
def __enter__(self):
|
|
1196
1196
|
"""Enter context manager for resource management."""
|
|
@@ -1256,8 +1256,8 @@ class AFileManager(AFile, AFolder, APath, AAtomicOperations, ABackupOperations,
|
|
|
1256
1256
|
self.max_file_size = config.get('max_file_size', 100 * 1024 * 1024) # 100MB
|
|
1257
1257
|
|
|
1258
1258
|
# Track temporary files for cleanup
|
|
1259
|
-
self._temp_files:
|
|
1260
|
-
self._temp_dirs:
|
|
1259
|
+
self._temp_files: list[Path] = []
|
|
1260
|
+
self._temp_dirs: list[Path] = []
|
|
1261
1261
|
|
|
1262
1262
|
def __enter__(self):
|
|
1263
1263
|
"""Enter context manager for resource management."""
|
|
@@ -1315,7 +1315,7 @@ class AFileManager(AFile, AFolder, APath, AAtomicOperations, ABackupOperations,
|
|
|
1315
1315
|
|
|
1316
1316
|
return type_mappings.get(ext, 'unknown')
|
|
1317
1317
|
|
|
1318
|
-
def get_file_info(self, file_path: Union[str, Path]) ->
|
|
1318
|
+
def get_file_info(self, file_path: Union[str, Path]) -> dict[str, Any]:
|
|
1319
1319
|
"""
|
|
1320
1320
|
Get comprehensive file information.
|
|
1321
1321
|
|
|
@@ -4,23 +4,34 @@
|
|
|
4
4
|
Company: eXonware.com
|
|
5
5
|
Author: Eng. Muhammad AlShehri
|
|
6
6
|
Email: connect@exonware.com
|
|
7
|
-
Version: 0.0.1.
|
|
7
|
+
Version: 0.0.1.411
|
|
8
8
|
Generation Date: October 30, 2025
|
|
9
9
|
|
|
10
10
|
Base classes, registry, adapters, and helper functions for codec system.
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
|
-
from typing import
|
|
14
|
+
from typing import Optional, Any, IO, Union
|
|
15
|
+
# Root cause: Migrating to Python 3.12 built-in generic syntax for consistency
|
|
16
|
+
# Priority #3: Maintainability - Modern type annotations improve code clarity
|
|
15
17
|
from pathlib import Path
|
|
16
18
|
from dataclasses import dataclass
|
|
17
19
|
from abc import ABC, abstractmethod
|
|
18
20
|
import mimetypes
|
|
21
|
+
import sys
|
|
19
22
|
|
|
20
23
|
from .contracts import ICodec, ICodecMetadata
|
|
21
24
|
from ..contracts import Serializer, Formatter, EncodeOptions, DecodeOptions
|
|
22
25
|
from ..defs import CodecCapability
|
|
23
|
-
from ..errors import EncodeError, DecodeError, CodecNotFoundError, CodecRegistrationError
|
|
26
|
+
from ..errors import EncodeError, DecodeError, CodecNotFoundError, CodecRegistrationError, SerializationError
|
|
27
|
+
|
|
28
|
+
# Default safety limits to prevent infinite recursion and excessive memory usage
|
|
29
|
+
# Root cause: Some codecs (like JSON5) hang on very large/deep nested structures
|
|
30
|
+
# Solution: Add configurable limits at base class level
|
|
31
|
+
# Priority #1: Security - Prevent DoS via excessive nesting
|
|
32
|
+
# Priority #4: Performance - Prevent hangs on large data
|
|
33
|
+
DEFAULT_MAX_DEPTH = 100 # Maximum nesting depth
|
|
34
|
+
DEFAULT_MAX_SIZE_MB = 100 # Maximum estimated size in MB
|
|
24
35
|
|
|
25
36
|
__all__ = [
|
|
26
37
|
'ACodec',
|
|
@@ -31,9 +42,6 @@ __all__ = [
|
|
|
31
42
|
'SerializerToFormatter',
|
|
32
43
|
]
|
|
33
44
|
|
|
34
|
-
T = TypeVar("T")
|
|
35
|
-
R = TypeVar("R")
|
|
36
|
-
|
|
37
45
|
|
|
38
46
|
# ============================================================================
|
|
39
47
|
# MEDIA KEY
|
|
@@ -112,12 +120,12 @@ class CodecRegistry:
|
|
|
112
120
|
"""
|
|
113
121
|
|
|
114
122
|
def __init__(self) -> None:
|
|
115
|
-
self._by_media_type:
|
|
116
|
-
self._by_extension:
|
|
117
|
-
self._by_id:
|
|
118
|
-
self._instances:
|
|
123
|
+
self._by_media_type: dict[MediaKey, type[ICodec]] = {}
|
|
124
|
+
self._by_extension: dict[str, type[ICodec]] = {}
|
|
125
|
+
self._by_id: dict[str, type[ICodec]] = {}
|
|
126
|
+
self._instances: dict[str, ICodec] = {} # Cached instances
|
|
119
127
|
|
|
120
|
-
def register(self, codec_class:
|
|
128
|
+
def register(self, codec_class: type[ICodec]) -> None:
|
|
121
129
|
"""
|
|
122
130
|
Register a codec class.
|
|
123
131
|
|
|
@@ -296,7 +304,7 @@ def get_global_registry() -> CodecRegistry:
|
|
|
296
304
|
# BASE CODEC CLASS WITH CONVENIENCE METHODS
|
|
297
305
|
# ============================================================================
|
|
298
306
|
|
|
299
|
-
class ACodec
|
|
307
|
+
class ACodec[T, R](ICodec[T, R], ICodecMetadata, ABC):
|
|
300
308
|
"""
|
|
301
309
|
Base codec class with all convenience methods.
|
|
302
310
|
|
|
@@ -305,6 +313,7 @@ class ACodec(Generic[T, R], ICodec[T, R], ICodecMetadata, ABC):
|
|
|
305
313
|
- All convenience aliases (dumps/loads/serialize/etc.)
|
|
306
314
|
- File I/O helpers (save/load/export/import)
|
|
307
315
|
- Stream operations (write/read)
|
|
316
|
+
- Safety validation (depth and size limits with caching)
|
|
308
317
|
|
|
309
318
|
Subclasses only need to implement:
|
|
310
319
|
- encode()
|
|
@@ -328,6 +337,244 @@ class ACodec(Generic[T, R], ICodec[T, R], ICodecMetadata, ABC):
|
|
|
328
337
|
... return CodecCapability.BIDIRECTIONAL | CodecCapability.TEXT
|
|
329
338
|
"""
|
|
330
339
|
|
|
340
|
+
def __init__(self, max_depth: Optional[int] = None, max_size_mb: Optional[float] = None):
|
|
341
|
+
"""
|
|
342
|
+
Initialize codec base.
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
max_depth: Maximum nesting depth allowed (default: DEFAULT_MAX_DEPTH)
|
|
346
|
+
max_size_mb: Maximum estimated data size in MB (default: DEFAULT_MAX_SIZE_MB)
|
|
347
|
+
|
|
348
|
+
Root cause: Codecs can hang on very large/deep nested structures.
|
|
349
|
+
Solution: Add configurable limits to prevent infinite recursion and excessive memory.
|
|
350
|
+
Priority #1: Security - Prevent DoS via excessive nesting
|
|
351
|
+
Priority #4: Performance - Prevent hangs on large data
|
|
352
|
+
"""
|
|
353
|
+
self._max_depth = max_depth if max_depth is not None else DEFAULT_MAX_DEPTH
|
|
354
|
+
self._max_size_mb = max_size_mb if max_size_mb is not None else DEFAULT_MAX_SIZE_MB
|
|
355
|
+
# Cache for depth/size calculations to avoid reprocessing same objects
|
|
356
|
+
self._depth_cache: dict[int, int] = {} # obj_id -> depth
|
|
357
|
+
self._size_cache: dict[int, float] = {} # obj_id -> size_mb
|
|
358
|
+
|
|
359
|
+
# ========================================================================
|
|
360
|
+
# SAFETY VALIDATION METHODS (Protect against infinite recursion)
|
|
361
|
+
# ========================================================================
|
|
362
|
+
|
|
363
|
+
def _get_data_depth(self, data: Any, cache: Optional[dict[int, int]] = None, visited: Optional[set] = None, current_depth: int = 0) -> int:
|
|
364
|
+
"""
|
|
365
|
+
Calculate maximum nesting depth of data structure using caching.
|
|
366
|
+
|
|
367
|
+
Root cause: Deeply nested structures can cause infinite recursion in parsers.
|
|
368
|
+
Solution: Recursively calculate depth with cycle detection and caching.
|
|
369
|
+
Priority #1: Security - Prevent DoS via excessive nesting
|
|
370
|
+
Priority #4: Performance - Detect problematic structures early, cache results
|
|
371
|
+
|
|
372
|
+
Args:
|
|
373
|
+
data: Data structure to analyze
|
|
374
|
+
cache: Optional cache dictionary (uses instance cache if None)
|
|
375
|
+
visited: Set of object IDs currently being processed (for cycle detection)
|
|
376
|
+
current_depth: Current recursion depth
|
|
377
|
+
|
|
378
|
+
Returns:
|
|
379
|
+
Maximum nesting depth found
|
|
380
|
+
"""
|
|
381
|
+
if cache is None:
|
|
382
|
+
cache = self._depth_cache
|
|
383
|
+
if visited is None:
|
|
384
|
+
visited = set()
|
|
385
|
+
|
|
386
|
+
# Safety check: prevent infinite recursion
|
|
387
|
+
if current_depth > self._max_depth * 2: # Allow some overhead for cycle detection
|
|
388
|
+
return current_depth
|
|
389
|
+
|
|
390
|
+
# Handle primitive types (no nesting)
|
|
391
|
+
if data is None or isinstance(data, (str, int, float, bool, bytes)):
|
|
392
|
+
return current_depth
|
|
393
|
+
|
|
394
|
+
obj_id = id(data)
|
|
395
|
+
|
|
396
|
+
# Handle cycles (reference to currently-being-processed object)
|
|
397
|
+
if obj_id in visited:
|
|
398
|
+
return current_depth # Cycle detected, don't count as additional depth
|
|
399
|
+
|
|
400
|
+
# Check cache first (avoid reprocessing same object)
|
|
401
|
+
if obj_id in cache:
|
|
402
|
+
# Use cached depth (maximum depth from this object), add current_depth
|
|
403
|
+
return cache[obj_id] + current_depth
|
|
404
|
+
|
|
405
|
+
# Mark as being processed
|
|
406
|
+
visited.add(obj_id)
|
|
407
|
+
|
|
408
|
+
try:
|
|
409
|
+
# Calculate maximum depth from this object (relative depth)
|
|
410
|
+
max_relative_depth = 0
|
|
411
|
+
|
|
412
|
+
if isinstance(data, dict):
|
|
413
|
+
if data: # Non-empty dict
|
|
414
|
+
child_depths = [
|
|
415
|
+
self._get_data_depth(v, cache, visited, current_depth + 1) - current_depth - 1
|
|
416
|
+
for v in data.values()
|
|
417
|
+
]
|
|
418
|
+
max_relative_depth = max(child_depths) if child_depths else 1
|
|
419
|
+
else:
|
|
420
|
+
max_relative_depth = 1 # Empty dict still counts as one level
|
|
421
|
+
|
|
422
|
+
elif isinstance(data, (list, tuple)):
|
|
423
|
+
if data: # Non-empty list/tuple
|
|
424
|
+
child_depths = [
|
|
425
|
+
self._get_data_depth(item, cache, visited, current_depth + 1) - current_depth - 1
|
|
426
|
+
for item in data
|
|
427
|
+
]
|
|
428
|
+
max_relative_depth = max(child_depths) if child_depths else 1
|
|
429
|
+
else:
|
|
430
|
+
max_relative_depth = 1 # Empty list still counts as one level
|
|
431
|
+
|
|
432
|
+
elif hasattr(data, '__dict__'):
|
|
433
|
+
# Custom object with attributes
|
|
434
|
+
child_depths = [
|
|
435
|
+
self._get_data_depth(v, cache, visited, current_depth + 1) - current_depth - 1
|
|
436
|
+
for v in vars(data).values()
|
|
437
|
+
]
|
|
438
|
+
max_relative_depth = max(child_depths) if child_depths else 0
|
|
439
|
+
|
|
440
|
+
# Cache the result (maximum relative depth from this object)
|
|
441
|
+
cache[obj_id] = max_relative_depth
|
|
442
|
+
|
|
443
|
+
return max_relative_depth + current_depth
|
|
444
|
+
|
|
445
|
+
except RecursionError:
|
|
446
|
+
# Fallback if recursion limit hit
|
|
447
|
+
return current_depth
|
|
448
|
+
finally:
|
|
449
|
+
# Remove from visited when done processing
|
|
450
|
+
visited.discard(obj_id)
|
|
451
|
+
|
|
452
|
+
def _estimate_data_size_mb(self, data: Any, cache: Optional[dict[int, float]] = None) -> float:
|
|
453
|
+
"""
|
|
454
|
+
Estimate data size in megabytes using caching.
|
|
455
|
+
|
|
456
|
+
Root cause: Very large data structures can cause memory issues and hangs.
|
|
457
|
+
Solution: Recursively estimate size with cycle detection and caching.
|
|
458
|
+
Priority #4: Performance - Detect large structures early, cache results
|
|
459
|
+
|
|
460
|
+
Args:
|
|
461
|
+
data: Data structure to analyze
|
|
462
|
+
cache: Optional cache dictionary (uses instance cache if None)
|
|
463
|
+
|
|
464
|
+
Returns:
|
|
465
|
+
Estimated size in megabytes
|
|
466
|
+
"""
|
|
467
|
+
if cache is None:
|
|
468
|
+
cache = self._size_cache
|
|
469
|
+
|
|
470
|
+
# Check cache first (avoid reprocessing same object)
|
|
471
|
+
obj_id = id(data)
|
|
472
|
+
if obj_id in cache:
|
|
473
|
+
return cache[obj_id]
|
|
474
|
+
|
|
475
|
+
# Calculate size for this object
|
|
476
|
+
size_bytes = 0.0
|
|
477
|
+
|
|
478
|
+
if isinstance(data, (str, bytes)):
|
|
479
|
+
size_bytes = len(data)
|
|
480
|
+
elif isinstance(data, (int, float)):
|
|
481
|
+
size_bytes = 8 # Approximate
|
|
482
|
+
elif isinstance(data, bool):
|
|
483
|
+
size_bytes = 1
|
|
484
|
+
elif isinstance(data, dict):
|
|
485
|
+
size_bytes = sys.getsizeof(data)
|
|
486
|
+
for k, v in data.items():
|
|
487
|
+
size_bytes += self._estimate_data_size_mb(k, cache) * 1024 * 1024
|
|
488
|
+
size_bytes += self._estimate_data_size_mb(v, cache) * 1024 * 1024
|
|
489
|
+
elif isinstance(data, (list, tuple)):
|
|
490
|
+
size_bytes = sys.getsizeof(data)
|
|
491
|
+
for item in data:
|
|
492
|
+
size_bytes += self._estimate_data_size_mb(item, cache) * 1024 * 1024
|
|
493
|
+
else:
|
|
494
|
+
size_bytes = sys.getsizeof(data)
|
|
495
|
+
if hasattr(data, '__dict__'):
|
|
496
|
+
for v in vars(data).values():
|
|
497
|
+
size_bytes += self._estimate_data_size_mb(v, cache) * 1024 * 1024
|
|
498
|
+
|
|
499
|
+
size_mb = size_bytes / (1024 * 1024) # Convert to MB
|
|
500
|
+
|
|
501
|
+
# Cache the result
|
|
502
|
+
cache[obj_id] = size_mb
|
|
503
|
+
|
|
504
|
+
return size_mb
|
|
505
|
+
|
|
506
|
+
def _validate_data_limits(
|
|
507
|
+
self,
|
|
508
|
+
data: Any,
|
|
509
|
+
operation: str = "encode",
|
|
510
|
+
file_path: Optional[Union[str, Path]] = None,
|
|
511
|
+
skip_size_check: bool = False
|
|
512
|
+
) -> None:
|
|
513
|
+
"""
|
|
514
|
+
Validate data structure against safety limits.
|
|
515
|
+
|
|
516
|
+
Root cause: Some codecs hang on very large/deep nested structures.
|
|
517
|
+
Solution: Check depth (always) and size (only for in-memory data, not large files).
|
|
518
|
+
Priority #1: Security - Prevent DoS via excessive nesting
|
|
519
|
+
Priority #4: Performance - Prevent hangs on large data
|
|
520
|
+
|
|
521
|
+
Args:
|
|
522
|
+
data: Data structure to validate
|
|
523
|
+
operation: Operation name for error messages (encode/decode)
|
|
524
|
+
file_path: Optional file path - if provided and file is large, skip size check
|
|
525
|
+
skip_size_check: If True, skip size validation (for large files that use lazy loading)
|
|
526
|
+
|
|
527
|
+
Raises:
|
|
528
|
+
SerializationError: If data exceeds safety limits
|
|
529
|
+
|
|
530
|
+
Note:
|
|
531
|
+
- Depth validation is ALWAYS performed (prevents infinite recursion)
|
|
532
|
+
- Size validation is SKIPPED for large files (10GB+ files are expected)
|
|
533
|
+
- Size validation is performed for in-memory data to catch problematic structures
|
|
534
|
+
"""
|
|
535
|
+
# Clear caches for fresh calculation
|
|
536
|
+
self._depth_cache.clear()
|
|
537
|
+
self._size_cache.clear()
|
|
538
|
+
|
|
539
|
+
# ALWAYS check depth - this prevents infinite recursion which is the real security issue
|
|
540
|
+
depth = self._get_data_depth(data)
|
|
541
|
+
if depth > self._max_depth:
|
|
542
|
+
raise SerializationError(
|
|
543
|
+
f"Data structure exceeds maximum nesting depth of {self._max_depth} "
|
|
544
|
+
f"(found {depth} levels). This may cause infinite recursion in {self.codec_id} {operation}. "
|
|
545
|
+
f"Consider flattening the data structure or using a different format.",
|
|
546
|
+
format_name=getattr(self, 'format_name', self.codec_id)
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
# Size check: Skip for large files (they use lazy loading/streaming)
|
|
550
|
+
# Only validate size for in-memory data structures
|
|
551
|
+
if skip_size_check:
|
|
552
|
+
return # Skip size check (e.g., for atomic path operations on large files)
|
|
553
|
+
|
|
554
|
+
# Check if file exists and is large - if so, skip size validation
|
|
555
|
+
if file_path:
|
|
556
|
+
try:
|
|
557
|
+
path_obj = Path(file_path)
|
|
558
|
+
if path_obj.exists():
|
|
559
|
+
file_size_mb = path_obj.stat().st_size / (1024 * 1024)
|
|
560
|
+
# If file is > 1GB, assume it's meant to be large and skip size validation
|
|
561
|
+
# Large files should use lazy loading/streaming features
|
|
562
|
+
if file_size_mb > 1024: # 1GB threshold
|
|
563
|
+
return # Skip size check for large files
|
|
564
|
+
except (OSError, ValueError):
|
|
565
|
+
pass # If we can't check file size, proceed with validation
|
|
566
|
+
|
|
567
|
+
# Check size for in-memory data (not from large files)
|
|
568
|
+
size_mb = self._estimate_data_size_mb(data)
|
|
569
|
+
if size_mb > self._max_size_mb:
|
|
570
|
+
raise SerializationError(
|
|
571
|
+
f"Data structure exceeds maximum size of {self._max_size_mb}MB "
|
|
572
|
+
f"(estimated {size_mb:.1f}MB). This may cause memory issues or hangs. "
|
|
573
|
+
f"For large files (10GB+), use lazy loading or streaming features. "
|
|
574
|
+
f"Consider splitting the data or using a streaming format.",
|
|
575
|
+
format_name=getattr(self, 'format_name', self.codec_id)
|
|
576
|
+
)
|
|
577
|
+
|
|
331
578
|
# ========================================================================
|
|
332
579
|
# CORE METHODS (Must implement in subclasses)
|
|
333
580
|
# ========================================================================
|
|
@@ -543,7 +790,7 @@ class ACodec(Generic[T, R], ICodec[T, R], ICodecMetadata, ABC):
|
|
|
543
790
|
# ADAPTERS (Bytes ↔ String)
|
|
544
791
|
# ============================================================================
|
|
545
792
|
|
|
546
|
-
class FormatterToSerializer
|
|
793
|
+
class FormatterToSerializer[T]:
|
|
547
794
|
"""
|
|
548
795
|
Adapter: Formatter[T, str] → Serializer[T, bytes].
|
|
549
796
|
|
|
@@ -589,7 +836,7 @@ class FormatterToSerializer(Generic[T]):
|
|
|
589
836
|
return self._formatter.decode(text, options=options)
|
|
590
837
|
|
|
591
838
|
|
|
592
|
-
class SerializerToFormatter
|
|
839
|
+
class SerializerToFormatter[T]:
|
|
593
840
|
"""
|
|
594
841
|
Adapter: Serializer[T, bytes] → Formatter[T, str].
|
|
595
842
|
|