arkparser 0.5.0__tar.gz → 0.5.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arkparser-0.5.0 → arkparser-0.5.4}/PKG-INFO +1 -1
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/__init__.py +98 -98
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/normalization.py +6 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/data_models.py +25 -2
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/export.py +149 -37
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/game_object.py +40 -20
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/base.py +2 -2
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/byte_property.py +1 -1
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/compound.py +14 -13
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/primitives.py +14 -14
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/PKG-INFO +1 -1
- {arkparser-0.5.0 → arkparser-0.5.4}/pyproject.toml +86 -86
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_review_fixes.py +18 -3
- {arkparser-0.5.0 → arkparser-0.5.4}/LICENSE +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/README.md +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/__init__.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/binary_reader.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/exceptions.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/map_config.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/types.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/version_detection.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/__init__.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/base.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/cloud_inventory.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/profile.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/tribe.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/world_save.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/__init__.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/container.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/location.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/__init__.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/registry.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/__init__.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/base.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/colors.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/misc.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/property_list.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/registry.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/vectors.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/SOURCES.txt +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/dependency_links.txt +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/requires.txt +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/top_level.txt +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/setup.cfg +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_asa_header_position.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_asa_name_table.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_ase_cluster_drift.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_binary_reader.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_binary_reader_layouts.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cloud_export.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cloud_inventory.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cryopod_export.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_current_stats.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_data_models.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_export.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_game_objects.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_profile.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_tribe.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_v13_property_layouts.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_version_detection.py +0 -0
- {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_world_save.py +0 -0
|
@@ -1,98 +1,98 @@
|
|
|
1
|
-
"""
|
|
2
|
-
ARK Save Parser - Parse ARK: Survival Evolved/Ascended save files.
|
|
3
|
-
|
|
4
|
-
This package provides tools to parse various ARK save file formats:
|
|
5
|
-
|
|
6
|
-
- Profile: Player profile data (.arkprofile)
|
|
7
|
-
- Tribe: Tribe data (.arktribe)
|
|
8
|
-
- CloudInventory: Obelisk/cloud inventory data (no extension)
|
|
9
|
-
- WorldSave: World save data (.ark), auto-detects ASE binary and ASA SQLite
|
|
10
|
-
|
|
11
|
-
Supports both ASE (ARK: Survival Evolved) and ASA (ARK: Survival Ascended)
|
|
12
|
-
formats with automatic detection.
|
|
13
|
-
|
|
14
|
-
Example usage:
|
|
15
|
-
>>> from arkparser import Profile, Tribe, CloudInventory, WorldSave
|
|
16
|
-
>>> profile = Profile.load("path/to/profile.arkprofile")
|
|
17
|
-
>>> tribe = Tribe.load("path/to/tribe.arktribe")
|
|
18
|
-
>>> inv = CloudInventory.load("path/to/obelisk_file")
|
|
19
|
-
>>> save = WorldSave.load("path/to/TheIsland.ark") # ASE
|
|
20
|
-
>>> save = WorldSave.load("path/to/Extinction_WP.ark") # ASA
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
from arkparser.common.exceptions import ArkParseError
|
|
24
|
-
from arkparser.common.map_config import MapConfig, get_map_config, get_map_config_by_name
|
|
25
|
-
from arkparser.common.version_detection import (
|
|
26
|
-
ArkFileFormat,
|
|
27
|
-
ArkFileType,
|
|
28
|
-
detect_file_type,
|
|
29
|
-
detect_format,
|
|
30
|
-
)
|
|
31
|
-
from arkparser.data_models import (
|
|
32
|
-
CryopodCreature,
|
|
33
|
-
DinoStats,
|
|
34
|
-
UploadedCreature,
|
|
35
|
-
UploadedItem,
|
|
36
|
-
)
|
|
37
|
-
from arkparser.export import (
|
|
38
|
-
export_all,
|
|
39
|
-
export_cloud_inventory,
|
|
40
|
-
export_cluster_items,
|
|
41
|
-
export_cluster_uploads,
|
|
42
|
-
export_map_structures,
|
|
43
|
-
export_players,
|
|
44
|
-
export_structures,
|
|
45
|
-
export_tamed,
|
|
46
|
-
export_to_files,
|
|
47
|
-
export_tribe_logs,
|
|
48
|
-
export_tribes,
|
|
49
|
-
export_wild,
|
|
50
|
-
)
|
|
51
|
-
from arkparser.files import CloudInventory, Profile, Tribe, WorldSave
|
|
52
|
-
from arkparser.game_objects import GameObject, GameObjectContainer, LocationData
|
|
53
|
-
|
|
54
|
-
# Convenience alias - users may know this as "Obelisk" from the game
|
|
55
|
-
Obelisk = CloudInventory
|
|
56
|
-
|
|
57
|
-
__all__ = [
|
|
58
|
-
# File parsers
|
|
59
|
-
"Profile",
|
|
60
|
-
"Tribe",
|
|
61
|
-
"CloudInventory",
|
|
62
|
-
"Obelisk",
|
|
63
|
-
"WorldSave",
|
|
64
|
-
# Cloud-inventory data models
|
|
65
|
-
"UploadedCreature",
|
|
66
|
-
"UploadedItem",
|
|
67
|
-
"CryopodCreature",
|
|
68
|
-
"DinoStats",
|
|
69
|
-
# Game objects
|
|
70
|
-
"GameObject",
|
|
71
|
-
"GameObjectContainer",
|
|
72
|
-
"LocationData",
|
|
73
|
-
# Map config
|
|
74
|
-
"MapConfig",
|
|
75
|
-
"get_map_config",
|
|
76
|
-
"get_map_config_by_name",
|
|
77
|
-
# Export
|
|
78
|
-
"export_all",
|
|
79
|
-
"export_tamed",
|
|
80
|
-
"export_wild",
|
|
81
|
-
"export_players",
|
|
82
|
-
"export_tribes",
|
|
83
|
-
"export_structures",
|
|
84
|
-
"export_tribe_logs",
|
|
85
|
-
"export_map_structures",
|
|
86
|
-
"export_cluster_uploads",
|
|
87
|
-
"export_cluster_items",
|
|
88
|
-
"export_cloud_inventory",
|
|
89
|
-
"export_to_files",
|
|
90
|
-
# Utilities
|
|
91
|
-
"detect_format",
|
|
92
|
-
"detect_file_type",
|
|
93
|
-
"ArkFileFormat",
|
|
94
|
-
"ArkFileType",
|
|
95
|
-
"ArkParseError",
|
|
96
|
-
]
|
|
97
|
-
|
|
98
|
-
__version__ = "0.5.
|
|
1
|
+
"""
|
|
2
|
+
ARK Save Parser - Parse ARK: Survival Evolved/Ascended save files.
|
|
3
|
+
|
|
4
|
+
This package provides tools to parse various ARK save file formats:
|
|
5
|
+
|
|
6
|
+
- Profile: Player profile data (.arkprofile)
|
|
7
|
+
- Tribe: Tribe data (.arktribe)
|
|
8
|
+
- CloudInventory: Obelisk/cloud inventory data (no extension)
|
|
9
|
+
- WorldSave: World save data (.ark), auto-detects ASE binary and ASA SQLite
|
|
10
|
+
|
|
11
|
+
Supports both ASE (ARK: Survival Evolved) and ASA (ARK: Survival Ascended)
|
|
12
|
+
formats with automatic detection.
|
|
13
|
+
|
|
14
|
+
Example usage:
|
|
15
|
+
>>> from arkparser import Profile, Tribe, CloudInventory, WorldSave
|
|
16
|
+
>>> profile = Profile.load("path/to/profile.arkprofile")
|
|
17
|
+
>>> tribe = Tribe.load("path/to/tribe.arktribe")
|
|
18
|
+
>>> inv = CloudInventory.load("path/to/obelisk_file")
|
|
19
|
+
>>> save = WorldSave.load("path/to/TheIsland.ark") # ASE
|
|
20
|
+
>>> save = WorldSave.load("path/to/Extinction_WP.ark") # ASA
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from arkparser.common.exceptions import ArkParseError
|
|
24
|
+
from arkparser.common.map_config import MapConfig, get_map_config, get_map_config_by_name
|
|
25
|
+
from arkparser.common.version_detection import (
|
|
26
|
+
ArkFileFormat,
|
|
27
|
+
ArkFileType,
|
|
28
|
+
detect_file_type,
|
|
29
|
+
detect_format,
|
|
30
|
+
)
|
|
31
|
+
from arkparser.data_models import (
|
|
32
|
+
CryopodCreature,
|
|
33
|
+
DinoStats,
|
|
34
|
+
UploadedCreature,
|
|
35
|
+
UploadedItem,
|
|
36
|
+
)
|
|
37
|
+
from arkparser.export import (
|
|
38
|
+
export_all,
|
|
39
|
+
export_cloud_inventory,
|
|
40
|
+
export_cluster_items,
|
|
41
|
+
export_cluster_uploads,
|
|
42
|
+
export_map_structures,
|
|
43
|
+
export_players,
|
|
44
|
+
export_structures,
|
|
45
|
+
export_tamed,
|
|
46
|
+
export_to_files,
|
|
47
|
+
export_tribe_logs,
|
|
48
|
+
export_tribes,
|
|
49
|
+
export_wild,
|
|
50
|
+
)
|
|
51
|
+
from arkparser.files import CloudInventory, Profile, Tribe, WorldSave
|
|
52
|
+
from arkparser.game_objects import GameObject, GameObjectContainer, LocationData
|
|
53
|
+
|
|
54
|
+
# Convenience alias - users may know this as "Obelisk" from the game
|
|
55
|
+
Obelisk = CloudInventory
|
|
56
|
+
|
|
57
|
+
__all__ = [
|
|
58
|
+
# File parsers
|
|
59
|
+
"Profile",
|
|
60
|
+
"Tribe",
|
|
61
|
+
"CloudInventory",
|
|
62
|
+
"Obelisk",
|
|
63
|
+
"WorldSave",
|
|
64
|
+
# Cloud-inventory data models
|
|
65
|
+
"UploadedCreature",
|
|
66
|
+
"UploadedItem",
|
|
67
|
+
"CryopodCreature",
|
|
68
|
+
"DinoStats",
|
|
69
|
+
# Game objects
|
|
70
|
+
"GameObject",
|
|
71
|
+
"GameObjectContainer",
|
|
72
|
+
"LocationData",
|
|
73
|
+
# Map config
|
|
74
|
+
"MapConfig",
|
|
75
|
+
"get_map_config",
|
|
76
|
+
"get_map_config_by_name",
|
|
77
|
+
# Export
|
|
78
|
+
"export_all",
|
|
79
|
+
"export_tamed",
|
|
80
|
+
"export_wild",
|
|
81
|
+
"export_players",
|
|
82
|
+
"export_tribes",
|
|
83
|
+
"export_structures",
|
|
84
|
+
"export_tribe_logs",
|
|
85
|
+
"export_map_structures",
|
|
86
|
+
"export_cluster_uploads",
|
|
87
|
+
"export_cluster_items",
|
|
88
|
+
"export_cloud_inventory",
|
|
89
|
+
"export_to_files",
|
|
90
|
+
# Utilities
|
|
91
|
+
"detect_format",
|
|
92
|
+
"detect_file_type",
|
|
93
|
+
"ArkFileFormat",
|
|
94
|
+
"ArkFileType",
|
|
95
|
+
"ArkParseError",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
__version__ = "0.5.4"
|
|
@@ -54,4 +54,10 @@ def normalize_indexed_list(value: t.Any) -> list[t.Any]:
|
|
|
54
54
|
return []
|
|
55
55
|
if isinstance(normalized, list):
|
|
56
56
|
return normalized
|
|
57
|
+
# A raw ByteProperty array is stored as `bytes` for memory efficiency
|
|
58
|
+
# (8x lighter than list[int]); expose it element-wise as a list of ints so
|
|
59
|
+
# consumers that iterate it (e.g. tribe MembersRankGroups -> int(rank))
|
|
60
|
+
# see the same shape they did before byte arrays were stored as bytes.
|
|
61
|
+
if isinstance(normalized, (bytes, bytearray)):
|
|
62
|
+
return list(normalized)
|
|
57
63
|
return [normalized]
|
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
11
|
import math
|
|
12
|
+
import re
|
|
12
13
|
import typing as t
|
|
13
14
|
from dataclasses import dataclass, field
|
|
14
15
|
|
|
@@ -18,6 +19,26 @@ from .properties.registry import read_properties
|
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(__name__)
|
|
20
21
|
|
|
22
|
+
# Trailing UE actor spawn-instance suffix on cryopod class names, e.g.
|
|
23
|
+
# "Raptor_Character_BP_C_2145673735". ARK stores the full instance name in the
|
|
24
|
+
# cryopod CustomDataStrings/SoftClasses blob; the worldsave name-table path
|
|
25
|
+
# discards this instance int32, so non-cryo tames never carry it. Strip it to
|
|
26
|
+
# match the canonical class name ("Raptor_Character_BP_C") and legacy parity.
|
|
27
|
+
_INSTANCE_SUFFIX_RE = re.compile(r"_\d+$")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _strip_instance_suffix(class_name: str) -> str:
|
|
31
|
+
"""Drop a trailing ``_<digits>`` actor-instance suffix from a class name.
|
|
32
|
+
|
|
33
|
+
Pre: ``class_name`` is a str. Post: returns the same string with at most one
|
|
34
|
+
trailing ``_<digits>`` group removed. Legit variant suffixes (``_Aberrant_C``,
|
|
35
|
+
``_Fire_C``) end in a letter, so they are never touched.
|
|
36
|
+
"""
|
|
37
|
+
assert isinstance(class_name, str), "class_name must be str"
|
|
38
|
+
stripped = _INSTANCE_SUFFIX_RE.sub("", class_name)
|
|
39
|
+
assert len(stripped) <= len(class_name), "strip must not grow the string"
|
|
40
|
+
return stripped
|
|
41
|
+
|
|
21
42
|
|
|
22
43
|
def _finite(value: t.Any, default: float) -> float:
|
|
23
44
|
"""Coerce to a finite float; NaN / inf / non-numeric collapse to ``default``.
|
|
@@ -514,7 +535,7 @@ class CryopodCreature:
|
|
|
514
535
|
# Parse strings - need at least 3 for basic info
|
|
515
536
|
strings = normalize_indexed_list(custom_data.get("CustomDataStrings"))
|
|
516
537
|
if len(strings) >= 3:
|
|
517
|
-
cryo.class_name = strings[0] #
|
|
538
|
+
cryo.class_name = _strip_instance_suffix(strings[0]) # raw: "Raptor_Character_BP_C_2145673735"
|
|
518
539
|
display_name = strings[1] # e.g., "bluey - Lvl 228 (Raptor)"
|
|
519
540
|
colors_str = strings[2] # e.g., "2,2,2,2,2,2,"
|
|
520
541
|
|
|
@@ -602,7 +623,9 @@ class CryopodCreature:
|
|
|
602
623
|
if soft_classes:
|
|
603
624
|
first_class = soft_classes[0]
|
|
604
625
|
if isinstance(first_class, dict):
|
|
605
|
-
cryo.class_name =
|
|
626
|
+
cryo.class_name = _strip_instance_suffix(
|
|
627
|
+
first_class.get("name", cryo.class_name)
|
|
628
|
+
)
|
|
606
629
|
|
|
607
630
|
# Populate raw creature_props / status_props so the export
|
|
608
631
|
# pipeline's _SyntheticGameObject adapter (which calls
|
|
@@ -32,6 +32,7 @@ Performance:
|
|
|
32
32
|
from __future__ import annotations
|
|
33
33
|
|
|
34
34
|
import datetime as dt
|
|
35
|
+
import itertools
|
|
35
36
|
import json
|
|
36
37
|
import logging
|
|
37
38
|
import math
|
|
@@ -1366,13 +1367,21 @@ def export_tamed(save: t.Any, map_config: MapConfig | None = None) -> list[dict[
|
|
|
1366
1367
|
(b) every cryopod-embedded tame, with the latter carrying ``cryo=True``
|
|
1367
1368
|
and inheriting the cryopod item's world location for GPS fields.
|
|
1368
1369
|
"""
|
|
1370
|
+
return list(_iter_tamed(save, map_config))
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _iter_tamed(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
|
|
1374
|
+
"""Yield ASV_Tamed records one at a time (see :func:`export_tamed`).
|
|
1375
|
+
|
|
1376
|
+
Generator form so :func:`export_to_files` can stream each record to disk
|
|
1377
|
+
and release it instead of materializing the whole list (the list is the
|
|
1378
|
+
dominant export-time allocation on large PvE saves).
|
|
1379
|
+
"""
|
|
1369
1380
|
objects = _world_objects(save, "get_tamed_creatures", "tamed_objects")
|
|
1370
1381
|
lookup = _save_lookup(save)
|
|
1371
|
-
|
|
1372
|
-
_tamed_dict(obj, _status_for(obj, lookup), lookup, map_config, save)
|
|
1373
|
-
|
|
1374
|
-
results.extend(_export_world_cryopods(save, map_config))
|
|
1375
|
-
return results
|
|
1382
|
+
for obj in objects:
|
|
1383
|
+
yield _tamed_dict(obj, _status_for(obj, lookup), lookup, map_config, save)
|
|
1384
|
+
yield from _export_world_cryopods(save, map_config)
|
|
1376
1385
|
|
|
1377
1386
|
|
|
1378
1387
|
def _build_item_owner_lookup(save: t.Any, lookup: dict[t.Any, t.Any]) -> dict[t.Any, dict[str, t.Any]]:
|
|
@@ -1820,10 +1829,16 @@ def _wild_dict(
|
|
|
1820
1829
|
|
|
1821
1830
|
|
|
1822
1831
|
def export_wild(save: t.Any, map_config: MapConfig | None = None) -> list[dict[str, t.Any]]:
|
|
1832
|
+
return list(_iter_wild(save, map_config))
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
def _iter_wild(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
|
|
1836
|
+
"""Yield ASV_Wild records one at a time (streaming form of export_wild)."""
|
|
1823
1837
|
objects = _world_objects(save, "get_wild_creatures", "wild_objects")
|
|
1824
1838
|
is_asa = bool(getattr(save, "is_asa", False))
|
|
1825
1839
|
lookup = _save_lookup(save)
|
|
1826
|
-
|
|
1840
|
+
for obj in objects:
|
|
1841
|
+
yield _wild_dict(obj, _status_for(obj, lookup), map_config, is_asa)
|
|
1827
1842
|
|
|
1828
1843
|
|
|
1829
1844
|
def _player_from_profile(
|
|
@@ -2073,6 +2088,15 @@ def export_players(
|
|
|
2073
2088
|
map_config: MapConfig | None = None,
|
|
2074
2089
|
cluster_inventories: t.Iterable[CloudInventory] | None = None,
|
|
2075
2090
|
) -> list[dict[str, t.Any]]:
|
|
2091
|
+
return list(_iter_players(save, map_config, cluster_inventories))
|
|
2092
|
+
|
|
2093
|
+
|
|
2094
|
+
def _iter_players(
|
|
2095
|
+
save: t.Any,
|
|
2096
|
+
map_config: MapConfig | None,
|
|
2097
|
+
cluster_inventories: t.Iterable[CloudInventory] | None,
|
|
2098
|
+
) -> t.Iterator[dict[str, t.Any]]:
|
|
2099
|
+
"""Yield ASV_Players records one at a time (streaming form of export_players)."""
|
|
2076
2100
|
profiles = _collection(save, "profiles", Profile)
|
|
2077
2101
|
lookup = _save_lookup(save)
|
|
2078
2102
|
pawn_status_by_id = _player_status_by_data_id(save, lookup)
|
|
@@ -2083,7 +2107,6 @@ def export_players(
|
|
|
2083
2107
|
allocation = _assemble_tribes(save)
|
|
2084
2108
|
profile_tribeid = allocation["profile_tribeid"]
|
|
2085
2109
|
tribe_names = allocation["names"]
|
|
2086
|
-
results: list[dict[str, t.Any]] = []
|
|
2087
2110
|
for entry in profiles:
|
|
2088
2111
|
record, join_keys = _player_record_for(entry, save, pawn_status_by_id, lookup, map_config)
|
|
2089
2112
|
if record is None:
|
|
@@ -2100,11 +2123,10 @@ def export_players(
|
|
|
2100
2123
|
inv_list = []
|
|
2101
2124
|
record["inventory"] = inv_list
|
|
2102
2125
|
inv_list.extend(spliced)
|
|
2103
|
-
|
|
2126
|
+
yield record
|
|
2104
2127
|
# Tribe members with no .arkprofile surface as stub players (legacy +N).
|
|
2105
2128
|
for tid, pid, name in allocation["member_stubs"]:
|
|
2106
|
-
|
|
2107
|
-
return results
|
|
2129
|
+
yield _member_stub_player(tid, pid, name, tribe_names.get(tid, ""))
|
|
2108
2130
|
|
|
2109
2131
|
|
|
2110
2132
|
def _strip_rich_color(msg: str) -> str:
|
|
@@ -2761,20 +2783,23 @@ def _structure_dict(
|
|
|
2761
2783
|
|
|
2762
2784
|
|
|
2763
2785
|
def export_structures(save: t.Any, map_config: MapConfig | None = None) -> list[dict[str, t.Any]]:
|
|
2786
|
+
return list(_iter_structures(save, map_config))
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def _iter_structures(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
|
|
2790
|
+
"""Yield ASV_Structures records one at a time (streaming form of export_structures)."""
|
|
2764
2791
|
objects = _world_objects(save, "get_structures", "structure_objects")
|
|
2765
2792
|
lookup = _save_lookup(save)
|
|
2766
2793
|
# Reuse the assembled tribe-name map so a structure's ``tribe`` resolves to
|
|
2767
2794
|
# the same name the tribes export uses (file TribeName / stub OwnerName).
|
|
2768
2795
|
tribe_names = _assemble_tribes(save)["names"]
|
|
2769
|
-
results: list[dict[str, t.Any]] = []
|
|
2770
2796
|
for obj in objects:
|
|
2771
2797
|
team = _int(_prop(obj, "TargetingTeam"))
|
|
2772
2798
|
if team < _PLAYER_TEAM_THRESHOLD and _is_excluded_abandoned(getattr(obj, "class_name", "") or ""):
|
|
2773
2799
|
# Unowned map element / crate / debug actor: legacy drops these
|
|
2774
2800
|
# from ASV_Structures (surfaced via ASV_MapStructures instead).
|
|
2775
2801
|
continue
|
|
2776
|
-
|
|
2777
|
-
return results
|
|
2802
|
+
yield _structure_dict(obj, save, lookup, map_config, tribe_names)
|
|
2778
2803
|
|
|
2779
2804
|
|
|
2780
2805
|
# Mirrors C# ContentContainer.cs:846. Order matters: first match wins.
|
|
@@ -2810,10 +2835,14 @@ def export_map_structures(
|
|
|
2810
2835
|
save: t.Any,
|
|
2811
2836
|
map_config: MapConfig | None = None,
|
|
2812
2837
|
) -> list[dict[str, t.Any]]:
|
|
2838
|
+
return list(_iter_map_structures(save, map_config))
|
|
2839
|
+
|
|
2840
|
+
|
|
2841
|
+
def _iter_map_structures(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
|
|
2842
|
+
"""Yield ASV_MapStructures records one at a time (streaming form of export_map_structures)."""
|
|
2813
2843
|
objects = getattr(save, "objects", None) or []
|
|
2814
|
-
all_objs =
|
|
2844
|
+
all_objs = objects.values() if isinstance(objects, dict) else objects
|
|
2815
2845
|
lookup = _save_lookup(save)
|
|
2816
|
-
results: list[dict[str, t.Any]] = []
|
|
2817
2846
|
for obj in all_objs:
|
|
2818
2847
|
cn = getattr(obj, "class_name", "") or ""
|
|
2819
2848
|
label = _asv_map_struct_label(cn)
|
|
@@ -2829,8 +2858,7 @@ def export_map_structures(
|
|
|
2829
2858
|
"inventory": _inventory_items(obj, lookup),
|
|
2830
2859
|
}
|
|
2831
2860
|
data.update(_gps_payload(obj, map_config))
|
|
2832
|
-
|
|
2833
|
-
return results
|
|
2861
|
+
yield data
|
|
2834
2862
|
|
|
2835
2863
|
|
|
2836
2864
|
# Legacy ASVExport.exe filenames. Single canonical schema.
|
|
@@ -2845,12 +2873,8 @@ _EXPORT_NAMES: dict[str, str] = {
|
|
|
2845
2873
|
}
|
|
2846
2874
|
|
|
2847
2875
|
|
|
2848
|
-
def
|
|
2849
|
-
|
|
2850
|
-
save: t.Any,
|
|
2851
|
-
map_config: MapConfig | None = None,
|
|
2852
|
-
) -> dict[str, t.Any]:
|
|
2853
|
-
"""Wrap a payload in legacy ``{map, day, time, data}`` envelope."""
|
|
2876
|
+
def _meta_head(save: t.Any, map_config: MapConfig | None = None) -> dict[str, t.Any]:
|
|
2877
|
+
"""Return the legacy envelope head ``{map, day, time}`` (no ``data`` key)."""
|
|
2854
2878
|
map_name = getattr(map_config, "name", "") if map_config is not None else ""
|
|
2855
2879
|
day = 0
|
|
2856
2880
|
time_str = "00:00"
|
|
@@ -2859,7 +2883,74 @@ def _wrap_with_meta(
|
|
|
2859
2883
|
day = int(game_time // 86400)
|
|
2860
2884
|
rem = int(game_time % 86400)
|
|
2861
2885
|
time_str = f"{rem // 3600:02d}:{(rem % 3600) // 60:02d}"
|
|
2862
|
-
return {"map": map_name, "day": day, "time": time_str
|
|
2886
|
+
return {"map": map_name, "day": day, "time": time_str}
|
|
2887
|
+
|
|
2888
|
+
|
|
2889
|
+
def _wrap_with_meta(
|
|
2890
|
+
data: list[dict[str, t.Any]],
|
|
2891
|
+
save: t.Any,
|
|
2892
|
+
map_config: MapConfig | None = None,
|
|
2893
|
+
) -> dict[str, t.Any]:
|
|
2894
|
+
"""Wrap a payload in the legacy ``{map, day, time, data}`` envelope.
|
|
2895
|
+
|
|
2896
|
+
Retained for callers that already hold a materialized list (e.g. the
|
|
2897
|
+
validation harness diffing against legacy). :func:`export_to_files`
|
|
2898
|
+
streams via :func:`_stream_dump` instead and does not use this.
|
|
2899
|
+
"""
|
|
2900
|
+
return {**_meta_head(save, map_config), "data": data}
|
|
2901
|
+
|
|
2902
|
+
|
|
2903
|
+
def _stream_dump(
|
|
2904
|
+
fh: t.TextIO,
|
|
2905
|
+
head: dict[str, t.Any] | None,
|
|
2906
|
+
records: t.Iterable[dict[str, t.Any]],
|
|
2907
|
+
dump_kwargs: dict[str, t.Any],
|
|
2908
|
+
) -> None:
|
|
2909
|
+
"""Stream a JSON array to ``fh``, one record at a time.
|
|
2910
|
+
|
|
2911
|
+
Writes a bare ``[records...]`` array when ``head`` is ``None``, otherwise
|
|
2912
|
+
splices the records into ``head`` under a ``"data"`` key
|
|
2913
|
+
(``{..head.., "data": [records...]}``). Each record is encoded and written
|
|
2914
|
+
individually, then released — the full record list never co-resides in
|
|
2915
|
+
memory. The result round-trips identically (at the value level) to
|
|
2916
|
+
``json.dump(head | {"data": list(records)}, fh, **dump_kwargs)``; only
|
|
2917
|
+
insignificant whitespace differs.
|
|
2918
|
+
|
|
2919
|
+
Pre-conditions: ``records`` is a finite iterable of JSON-serializable
|
|
2920
|
+
dicts; ``dump_kwargs`` carries the same ``indent`` / ``separators`` /
|
|
2921
|
+
``default`` used elsewhere. Post-conditions: ``fh`` holds one complete,
|
|
2922
|
+
valid JSON document.
|
|
2923
|
+
"""
|
|
2924
|
+
indent = dump_kwargs.get("indent")
|
|
2925
|
+
rec_kwargs: dict[str, t.Any] = {"default": dump_kwargs.get("default", str)}
|
|
2926
|
+
if indent:
|
|
2927
|
+
rec_kwargs["indent"] = indent
|
|
2928
|
+
else:
|
|
2929
|
+
rec_kwargs["separators"] = (",", ":")
|
|
2930
|
+
nl = "\n" if indent else ""
|
|
2931
|
+
if head is not None:
|
|
2932
|
+
head_json = json.dumps(head, **rec_kwargs)
|
|
2933
|
+
assert head_json.endswith("}"), "envelope head must serialize to a JSON object"
|
|
2934
|
+
# Splice the data array in just before the head object's closing brace
|
|
2935
|
+
# (indented dumps leave a trailing "\n}", compact a bare "}").
|
|
2936
|
+
fh.write(head_json[: -2 if indent else -1])
|
|
2937
|
+
fh.write(f',{nl}{" " * indent}"data": [' if indent else ',"data":[')
|
|
2938
|
+
rec_pad = " " * (indent * 2) if indent else ""
|
|
2939
|
+
close_pad = " " * indent if indent else ""
|
|
2940
|
+
else:
|
|
2941
|
+
fh.write("[")
|
|
2942
|
+
rec_pad = " " * indent if indent else ""
|
|
2943
|
+
close_pad = ""
|
|
2944
|
+
first = True
|
|
2945
|
+
for rec in records:
|
|
2946
|
+
chunk = json.dumps(rec, **rec_kwargs)
|
|
2947
|
+
if indent:
|
|
2948
|
+
chunk = rec_pad + chunk.replace("\n", "\n" + rec_pad)
|
|
2949
|
+
fh.write(("" if first else ",") + nl + chunk)
|
|
2950
|
+
first = False
|
|
2951
|
+
fh.write("]" if first else nl + close_pad + "]")
|
|
2952
|
+
if head is not None:
|
|
2953
|
+
fh.write(nl + "}")
|
|
2863
2954
|
|
|
2864
2955
|
|
|
2865
2956
|
def _load_cluster_inventories(
|
|
@@ -2909,18 +3000,33 @@ def export_all(
|
|
|
2909
3000
|
Note: returns flat lists per type (not the legacy ``{map, day, time, data}``
|
|
2910
3001
|
envelope). ``export_to_files`` adds the envelope when writing.
|
|
2911
3002
|
"""
|
|
3003
|
+
return {name: list(records) for name, records in _iter_exports(save, map_config, cluster)}
|
|
3004
|
+
|
|
3005
|
+
|
|
3006
|
+
def _iter_exports(
|
|
3007
|
+
save: t.Any,
|
|
3008
|
+
map_config: MapConfig | None,
|
|
3009
|
+
cluster: str | Path | t.Iterable[CloudInventory] | None,
|
|
3010
|
+
) -> t.Iterator[tuple[str, t.Iterable[dict[str, t.Any]]]]:
|
|
3011
|
+
"""Yield ``(ASV_name, record_iterable)`` one export type at a time.
|
|
3012
|
+
|
|
3013
|
+
The heavy types (tamed / wild / players / structures / map_structures)
|
|
3014
|
+
yield **lazy generators** so :func:`export_to_files` can stream each
|
|
3015
|
+
record to disk and release it — the full per-type list never
|
|
3016
|
+
materializes (it is the dominant export-time allocation on large PvE
|
|
3017
|
+
saves). The small types (tribes / tribe_logs) stay eager lists.
|
|
3018
|
+
:func:`export_all` wraps each iterable in ``list()`` for callers that
|
|
3019
|
+
want every record in memory.
|
|
3020
|
+
"""
|
|
2912
3021
|
cluster_invs = _load_cluster_inventories(cluster)
|
|
2913
3022
|
cluster_tamed = export_cluster_uploads(cluster_invs, map_config) if cluster_invs else []
|
|
2914
|
-
tamed
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
_EXPORT_NAMES["tribe_logs"]: export_tribe_logs(save),
|
|
2922
|
-
_EXPORT_NAMES["map_structures"]: export_map_structures(save, map_config),
|
|
2923
|
-
}
|
|
3023
|
+
yield _EXPORT_NAMES["tamed"], itertools.chain(_iter_tamed(save, map_config), cluster_tamed)
|
|
3024
|
+
yield _EXPORT_NAMES["wild"], _iter_wild(save, map_config)
|
|
3025
|
+
yield _EXPORT_NAMES["players"], _iter_players(save, map_config, cluster_invs or None)
|
|
3026
|
+
yield _EXPORT_NAMES["tribes"], export_tribes(save)
|
|
3027
|
+
yield _EXPORT_NAMES["structures"], _iter_structures(save, map_config)
|
|
3028
|
+
yield _EXPORT_NAMES["tribe_logs"], export_tribe_logs(save)
|
|
3029
|
+
yield _EXPORT_NAMES["map_structures"], _iter_map_structures(save, map_config)
|
|
2924
3030
|
|
|
2925
3031
|
|
|
2926
3032
|
def export_to_files(
|
|
@@ -2952,9 +3058,15 @@ def export_to_files(
|
|
|
2952
3058
|
dump_kwargs: dict[str, t.Any] = {"separators": (",", ":"), "default": str}
|
|
2953
3059
|
else:
|
|
2954
3060
|
dump_kwargs = {"indent": 2, "default": str}
|
|
2955
|
-
|
|
2956
|
-
|
|
3061
|
+
# Stream each export type record-by-record straight to its file: a record
|
|
3062
|
+
# is built, encoded, written, then released. The full per-type list never
|
|
3063
|
+
# materializes and no whole-file JSON string is built — both were the
|
|
3064
|
+
# measured peak-RAM drivers on large PvE saves (the structures list alone
|
|
3065
|
+
# was hundreds of MB of nested inventory dicts).
|
|
3066
|
+
head = _meta_head(save, map_config) if wrap else None
|
|
3067
|
+
for name, records in _iter_exports(save, map_config, cluster):
|
|
2957
3068
|
path = out / f"{name}.json"
|
|
2958
|
-
path.
|
|
3069
|
+
with path.open("w", encoding="utf-8") as fh:
|
|
3070
|
+
_stream_dump(fh, head, records, dump_kwargs)
|
|
2959
3071
|
created.append(path)
|
|
2960
3072
|
return created
|
|
@@ -64,23 +64,34 @@ class GameObject:
|
|
|
64
64
|
parent: GameObject | None = field(default=None, repr=False)
|
|
65
65
|
components: dict[str, GameObject] = field(default_factory=dict, repr=False)
|
|
66
66
|
|
|
67
|
-
# Lazy property index: name ->
|
|
68
|
-
#
|
|
69
|
-
|
|
67
|
+
# Lazy property index: name -> Property (single-index, the common case) OR
|
|
68
|
+
# name -> {index: Property} (only when a name carries >1 distinct index).
|
|
69
|
+
# Storing the bare Property for single-index names avoids allocating one
|
|
70
|
+
# inner dict per (object, name): on a 300k-object save that was ~5.4M dicts
|
|
71
|
+
# / ~600 MB of pure index overhead built during export. Built on first
|
|
72
|
+
# lookup, invalidated by setting to None when properties is mutated.
|
|
73
|
+
_prop_index: dict[str, "Property | dict[int, Property]"] | None = field(
|
|
74
|
+
default=None, repr=False, compare=False
|
|
75
|
+
)
|
|
70
76
|
|
|
71
|
-
def _build_prop_index(self) -> dict[str, dict[int,
|
|
72
|
-
|
|
77
|
+
def _build_prop_index(self) -> dict[str, "Property | dict[int, Property]"]:
|
|
78
|
+
# First-writer wins, matching legacy C# GetPropertyValue
|
|
79
|
+
# (IPropertyContainer.cs:45) which returns the first (name, index)
|
|
80
|
+
# match. _serialize_properties keeps all occurrences in the additive
|
|
81
|
+
# `properties` dict; lookups intentionally mirror legacy here. Do not
|
|
82
|
+
# switch to last-wins.
|
|
83
|
+
idx: dict[str, "Property | dict[int, Property]"] = {}
|
|
73
84
|
for prop in self.properties:
|
|
74
|
-
|
|
75
|
-
if
|
|
76
|
-
idx[prop.name] =
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
# index
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
|
|
85
|
+
existing = idx.get(prop.name)
|
|
86
|
+
if existing is None:
|
|
87
|
+
idx[prop.name] = prop # common case: store the bare Property
|
|
88
|
+
elif isinstance(existing, dict):
|
|
89
|
+
existing.setdefault(prop.index, prop)
|
|
90
|
+
elif existing.index != prop.index:
|
|
91
|
+
# Second distinct index for this name: promote to a dict.
|
|
92
|
+
# `existing` first preserves first-writer iteration order.
|
|
93
|
+
idx[prop.name] = {existing.index: existing, prop.index: prop}
|
|
94
|
+
# else: duplicate (name, index) -> keep first-writer `existing`.
|
|
84
95
|
self._prop_index = idx
|
|
85
96
|
return idx
|
|
86
97
|
|
|
@@ -108,11 +119,16 @@ class GameObject:
|
|
|
108
119
|
"""Get a property by name and optional index (None = first by insertion)."""
|
|
109
120
|
idx = self._prop_index if self._prop_index is not None else self._build_prop_index()
|
|
110
121
|
bucket = idx.get(name)
|
|
111
|
-
if
|
|
122
|
+
if bucket is None:
|
|
112
123
|
return None
|
|
113
|
-
if
|
|
114
|
-
|
|
115
|
-
|
|
124
|
+
if isinstance(bucket, dict):
|
|
125
|
+
if index is None:
|
|
126
|
+
return next(iter(bucket.values()))
|
|
127
|
+
return bucket.get(index)
|
|
128
|
+
# Single-index bucket: the bare Property.
|
|
129
|
+
if index is None or bucket.index == index:
|
|
130
|
+
return bucket
|
|
131
|
+
return None
|
|
116
132
|
|
|
117
133
|
def get_property_value(self, name: str, default: t.Any = None, index: int | None = None) -> t.Any:
|
|
118
134
|
"""Get a property value by name (returns default if missing)."""
|
|
@@ -123,7 +139,11 @@ class GameObject:
|
|
|
123
139
|
"""Get all properties with the given name (any index)."""
|
|
124
140
|
idx = self._prop_index if self._prop_index is not None else self._build_prop_index()
|
|
125
141
|
bucket = idx.get(name)
|
|
126
|
-
|
|
142
|
+
if bucket is None:
|
|
143
|
+
return []
|
|
144
|
+
if isinstance(bucket, dict):
|
|
145
|
+
return list(bucket.values())
|
|
146
|
+
return [bucket]
|
|
127
147
|
|
|
128
148
|
def has_property(self, name: str) -> bool:
|
|
129
149
|
"""Check if this object has a property with the given name."""
|