arkparser 0.5.0__tar.gz → 0.5.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {arkparser-0.5.0 → arkparser-0.5.4}/PKG-INFO +1 -1
  2. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/__init__.py +98 -98
  3. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/normalization.py +6 -0
  4. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/data_models.py +25 -2
  5. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/export.py +149 -37
  6. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/game_object.py +40 -20
  7. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/base.py +2 -2
  8. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/byte_property.py +1 -1
  9. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/compound.py +14 -13
  10. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/primitives.py +14 -14
  11. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/PKG-INFO +1 -1
  12. {arkparser-0.5.0 → arkparser-0.5.4}/pyproject.toml +86 -86
  13. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_review_fixes.py +18 -3
  14. {arkparser-0.5.0 → arkparser-0.5.4}/LICENSE +0 -0
  15. {arkparser-0.5.0 → arkparser-0.5.4}/README.md +0 -0
  16. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/__init__.py +0 -0
  17. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/binary_reader.py +0 -0
  18. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/exceptions.py +0 -0
  19. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/map_config.py +0 -0
  20. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/types.py +0 -0
  21. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/common/version_detection.py +0 -0
  22. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/__init__.py +0 -0
  23. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/base.py +0 -0
  24. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/cloud_inventory.py +0 -0
  25. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/profile.py +0 -0
  26. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/tribe.py +0 -0
  27. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/files/world_save.py +0 -0
  28. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/__init__.py +0 -0
  29. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/container.py +0 -0
  30. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/game_objects/location.py +0 -0
  31. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/__init__.py +0 -0
  32. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/properties/registry.py +0 -0
  33. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/__init__.py +0 -0
  34. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/base.py +0 -0
  35. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/colors.py +0 -0
  36. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/misc.py +0 -0
  37. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/property_list.py +0 -0
  38. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/registry.py +0 -0
  39. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser/structs/vectors.py +0 -0
  40. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/SOURCES.txt +0 -0
  41. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/dependency_links.txt +0 -0
  42. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/requires.txt +0 -0
  43. {arkparser-0.5.0 → arkparser-0.5.4}/arkparser.egg-info/top_level.txt +0 -0
  44. {arkparser-0.5.0 → arkparser-0.5.4}/setup.cfg +0 -0
  45. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_asa_header_position.py +0 -0
  46. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_asa_name_table.py +0 -0
  47. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_ase_cluster_drift.py +0 -0
  48. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_binary_reader.py +0 -0
  49. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_binary_reader_layouts.py +0 -0
  50. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cloud_export.py +0 -0
  51. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cloud_inventory.py +0 -0
  52. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_cryopod_export.py +0 -0
  53. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_current_stats.py +0 -0
  54. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_data_models.py +0 -0
  55. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_export.py +0 -0
  56. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_game_objects.py +0 -0
  57. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_profile.py +0 -0
  58. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_tribe.py +0 -0
  59. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_v13_property_layouts.py +0 -0
  60. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_version_detection.py +0 -0
  61. {arkparser-0.5.0 → arkparser-0.5.4}/tests/test_world_save.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arkparser
3
- Version: 0.5.0
3
+ Version: 0.5.4
4
4
  Summary: Pure Python parser for ARK: Survival Evolved and ARK: Survival Ascended save files
5
5
  Author: Vertyco
6
6
  License-Expression: MIT
@@ -1,98 +1,98 @@
1
- """
2
- ARK Save Parser - Parse ARK: Survival Evolved/Ascended save files.
3
-
4
- This package provides tools to parse various ARK save file formats:
5
-
6
- - Profile: Player profile data (.arkprofile)
7
- - Tribe: Tribe data (.arktribe)
8
- - CloudInventory: Obelisk/cloud inventory data (no extension)
9
- - WorldSave: World save data (.ark), auto-detects ASE binary and ASA SQLite
10
-
11
- Supports both ASE (ARK: Survival Evolved) and ASA (ARK: Survival Ascended)
12
- formats with automatic detection.
13
-
14
- Example usage:
15
- >>> from arkparser import Profile, Tribe, CloudInventory, WorldSave
16
- >>> profile = Profile.load("path/to/profile.arkprofile")
17
- >>> tribe = Tribe.load("path/to/tribe.arktribe")
18
- >>> inv = CloudInventory.load("path/to/obelisk_file")
19
- >>> save = WorldSave.load("path/to/TheIsland.ark") # ASE
20
- >>> save = WorldSave.load("path/to/Extinction_WP.ark") # ASA
21
- """
22
-
23
- from arkparser.common.exceptions import ArkParseError
24
- from arkparser.common.map_config import MapConfig, get_map_config, get_map_config_by_name
25
- from arkparser.common.version_detection import (
26
- ArkFileFormat,
27
- ArkFileType,
28
- detect_file_type,
29
- detect_format,
30
- )
31
- from arkparser.data_models import (
32
- CryopodCreature,
33
- DinoStats,
34
- UploadedCreature,
35
- UploadedItem,
36
- )
37
- from arkparser.export import (
38
- export_all,
39
- export_cloud_inventory,
40
- export_cluster_items,
41
- export_cluster_uploads,
42
- export_map_structures,
43
- export_players,
44
- export_structures,
45
- export_tamed,
46
- export_to_files,
47
- export_tribe_logs,
48
- export_tribes,
49
- export_wild,
50
- )
51
- from arkparser.files import CloudInventory, Profile, Tribe, WorldSave
52
- from arkparser.game_objects import GameObject, GameObjectContainer, LocationData
53
-
54
- # Convenience alias - users may know this as "Obelisk" from the game
55
- Obelisk = CloudInventory
56
-
57
- __all__ = [
58
- # File parsers
59
- "Profile",
60
- "Tribe",
61
- "CloudInventory",
62
- "Obelisk",
63
- "WorldSave",
64
- # Cloud-inventory data models
65
- "UploadedCreature",
66
- "UploadedItem",
67
- "CryopodCreature",
68
- "DinoStats",
69
- # Game objects
70
- "GameObject",
71
- "GameObjectContainer",
72
- "LocationData",
73
- # Map config
74
- "MapConfig",
75
- "get_map_config",
76
- "get_map_config_by_name",
77
- # Export
78
- "export_all",
79
- "export_tamed",
80
- "export_wild",
81
- "export_players",
82
- "export_tribes",
83
- "export_structures",
84
- "export_tribe_logs",
85
- "export_map_structures",
86
- "export_cluster_uploads",
87
- "export_cluster_items",
88
- "export_cloud_inventory",
89
- "export_to_files",
90
- # Utilities
91
- "detect_format",
92
- "detect_file_type",
93
- "ArkFileFormat",
94
- "ArkFileType",
95
- "ArkParseError",
96
- ]
97
-
98
- __version__ = "0.5.0"
1
+ """
2
+ ARK Save Parser - Parse ARK: Survival Evolved/Ascended save files.
3
+
4
+ This package provides tools to parse various ARK save file formats:
5
+
6
+ - Profile: Player profile data (.arkprofile)
7
+ - Tribe: Tribe data (.arktribe)
8
+ - CloudInventory: Obelisk/cloud inventory data (no extension)
9
+ - WorldSave: World save data (.ark), auto-detects ASE binary and ASA SQLite
10
+
11
+ Supports both ASE (ARK: Survival Evolved) and ASA (ARK: Survival Ascended)
12
+ formats with automatic detection.
13
+
14
+ Example usage:
15
+ >>> from arkparser import Profile, Tribe, CloudInventory, WorldSave
16
+ >>> profile = Profile.load("path/to/profile.arkprofile")
17
+ >>> tribe = Tribe.load("path/to/tribe.arktribe")
18
+ >>> inv = CloudInventory.load("path/to/obelisk_file")
19
+ >>> save = WorldSave.load("path/to/TheIsland.ark") # ASE
20
+ >>> save = WorldSave.load("path/to/Extinction_WP.ark") # ASA
21
+ """
22
+
23
+ from arkparser.common.exceptions import ArkParseError
24
+ from arkparser.common.map_config import MapConfig, get_map_config, get_map_config_by_name
25
+ from arkparser.common.version_detection import (
26
+ ArkFileFormat,
27
+ ArkFileType,
28
+ detect_file_type,
29
+ detect_format,
30
+ )
31
+ from arkparser.data_models import (
32
+ CryopodCreature,
33
+ DinoStats,
34
+ UploadedCreature,
35
+ UploadedItem,
36
+ )
37
+ from arkparser.export import (
38
+ export_all,
39
+ export_cloud_inventory,
40
+ export_cluster_items,
41
+ export_cluster_uploads,
42
+ export_map_structures,
43
+ export_players,
44
+ export_structures,
45
+ export_tamed,
46
+ export_to_files,
47
+ export_tribe_logs,
48
+ export_tribes,
49
+ export_wild,
50
+ )
51
+ from arkparser.files import CloudInventory, Profile, Tribe, WorldSave
52
+ from arkparser.game_objects import GameObject, GameObjectContainer, LocationData
53
+
54
+ # Convenience alias - users may know this as "Obelisk" from the game
55
+ Obelisk = CloudInventory
56
+
57
+ __all__ = [
58
+ # File parsers
59
+ "Profile",
60
+ "Tribe",
61
+ "CloudInventory",
62
+ "Obelisk",
63
+ "WorldSave",
64
+ # Cloud-inventory data models
65
+ "UploadedCreature",
66
+ "UploadedItem",
67
+ "CryopodCreature",
68
+ "DinoStats",
69
+ # Game objects
70
+ "GameObject",
71
+ "GameObjectContainer",
72
+ "LocationData",
73
+ # Map config
74
+ "MapConfig",
75
+ "get_map_config",
76
+ "get_map_config_by_name",
77
+ # Export
78
+ "export_all",
79
+ "export_tamed",
80
+ "export_wild",
81
+ "export_players",
82
+ "export_tribes",
83
+ "export_structures",
84
+ "export_tribe_logs",
85
+ "export_map_structures",
86
+ "export_cluster_uploads",
87
+ "export_cluster_items",
88
+ "export_cloud_inventory",
89
+ "export_to_files",
90
+ # Utilities
91
+ "detect_format",
92
+ "detect_file_type",
93
+ "ArkFileFormat",
94
+ "ArkFileType",
95
+ "ArkParseError",
96
+ ]
97
+
98
+ __version__ = "0.5.4"
@@ -54,4 +54,10 @@ def normalize_indexed_list(value: t.Any) -> list[t.Any]:
54
54
  return []
55
55
  if isinstance(normalized, list):
56
56
  return normalized
57
+ # A raw ByteProperty array is stored as `bytes` for memory efficiency
58
+ # (8x lighter than list[int]); expose it element-wise as a list of ints so
59
+ # consumers that iterate it (e.g. tribe MembersRankGroups -> int(rank))
60
+ # see the same shape they did before byte arrays were stored as bytes.
61
+ if isinstance(normalized, (bytes, bytearray)):
62
+ return list(normalized)
57
63
  return [normalized]
@@ -9,6 +9,7 @@ from __future__ import annotations
9
9
 
10
10
  import logging
11
11
  import math
12
+ import re
12
13
  import typing as t
13
14
  from dataclasses import dataclass, field
14
15
 
@@ -18,6 +19,26 @@ from .properties.registry import read_properties
18
19
 
19
20
  logger = logging.getLogger(__name__)
20
21
 
22
+ # Trailing UE actor spawn-instance suffix on cryopod class names, e.g.
23
+ # "Raptor_Character_BP_C_2145673735". ARK stores the full instance name in the
24
+ # cryopod CustomDataStrings/SoftClasses blob; the worldsave name-table path
25
+ # discards this instance int32, so non-cryo tames never carry it. Strip it to
26
+ # match the canonical class name ("Raptor_Character_BP_C") and legacy parity.
27
+ _INSTANCE_SUFFIX_RE = re.compile(r"_\d+$")
28
+
29
+
30
+ def _strip_instance_suffix(class_name: str) -> str:
31
+ """Drop a trailing ``_<digits>`` actor-instance suffix from a class name.
32
+
33
+ Pre: ``class_name`` is a str. Post: returns the same string with at most one
34
+ trailing ``_<digits>`` group removed. Legit variant suffixes (``_Aberrant_C``,
35
+ ``_Fire_C``) end in a letter, so they are never touched.
36
+ """
37
+ assert isinstance(class_name, str), "class_name must be str"
38
+ stripped = _INSTANCE_SUFFIX_RE.sub("", class_name)
39
+ assert len(stripped) <= len(class_name), "strip must not grow the string"
40
+ return stripped
41
+
21
42
 
22
43
  def _finite(value: t.Any, default: float) -> float:
23
44
  """Coerce to a finite float; NaN / inf / non-numeric collapse to ``default``.
@@ -514,7 +535,7 @@ class CryopodCreature:
514
535
  # Parse strings - need at least 3 for basic info
515
536
  strings = normalize_indexed_list(custom_data.get("CustomDataStrings"))
516
537
  if len(strings) >= 3:
517
- cryo.class_name = strings[0] # e.g., "Raptor_Character_BP_C_2145673735"
538
+ cryo.class_name = _strip_instance_suffix(strings[0]) # raw: "Raptor_Character_BP_C_2145673735"
518
539
  display_name = strings[1] # e.g., "bluey - Lvl 228 (Raptor)"
519
540
  colors_str = strings[2] # e.g., "2,2,2,2,2,2,"
520
541
 
@@ -602,7 +623,9 @@ class CryopodCreature:
602
623
  if soft_classes:
603
624
  first_class = soft_classes[0]
604
625
  if isinstance(first_class, dict):
605
- cryo.class_name = first_class.get("name", cryo.class_name)
626
+ cryo.class_name = _strip_instance_suffix(
627
+ first_class.get("name", cryo.class_name)
628
+ )
606
629
 
607
630
  # Populate raw creature_props / status_props so the export
608
631
  # pipeline's _SyntheticGameObject adapter (which calls
@@ -32,6 +32,7 @@ Performance:
32
32
  from __future__ import annotations
33
33
 
34
34
  import datetime as dt
35
+ import itertools
35
36
  import json
36
37
  import logging
37
38
  import math
@@ -1366,13 +1367,21 @@ def export_tamed(save: t.Any, map_config: MapConfig | None = None) -> list[dict[
1366
1367
  (b) every cryopod-embedded tame, with the latter carrying ``cryo=True``
1367
1368
  and inheriting the cryopod item's world location for GPS fields.
1368
1369
  """
1370
+ return list(_iter_tamed(save, map_config))
1371
+
1372
+
1373
+ def _iter_tamed(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
1374
+ """Yield ASV_Tamed records one at a time (see :func:`export_tamed`).
1375
+
1376
+ Generator form so :func:`export_to_files` can stream each record to disk
1377
+ and release it instead of materializing the whole list (the list is the
1378
+ dominant export-time allocation on large PvE saves).
1379
+ """
1369
1380
  objects = _world_objects(save, "get_tamed_creatures", "tamed_objects")
1370
1381
  lookup = _save_lookup(save)
1371
- results: list[dict[str, t.Any]] = [
1372
- _tamed_dict(obj, _status_for(obj, lookup), lookup, map_config, save) for obj in objects
1373
- ]
1374
- results.extend(_export_world_cryopods(save, map_config))
1375
- return results
1382
+ for obj in objects:
1383
+ yield _tamed_dict(obj, _status_for(obj, lookup), lookup, map_config, save)
1384
+ yield from _export_world_cryopods(save, map_config)
1376
1385
 
1377
1386
 
1378
1387
  def _build_item_owner_lookup(save: t.Any, lookup: dict[t.Any, t.Any]) -> dict[t.Any, dict[str, t.Any]]:
@@ -1820,10 +1829,16 @@ def _wild_dict(
1820
1829
 
1821
1830
 
1822
1831
  def export_wild(save: t.Any, map_config: MapConfig | None = None) -> list[dict[str, t.Any]]:
1832
+ return list(_iter_wild(save, map_config))
1833
+
1834
+
1835
+ def _iter_wild(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
1836
+ """Yield ASV_Wild records one at a time (streaming form of export_wild)."""
1823
1837
  objects = _world_objects(save, "get_wild_creatures", "wild_objects")
1824
1838
  is_asa = bool(getattr(save, "is_asa", False))
1825
1839
  lookup = _save_lookup(save)
1826
- return [_wild_dict(obj, _status_for(obj, lookup), map_config, is_asa) for obj in objects]
1840
+ for obj in objects:
1841
+ yield _wild_dict(obj, _status_for(obj, lookup), map_config, is_asa)
1827
1842
 
1828
1843
 
1829
1844
  def _player_from_profile(
@@ -2073,6 +2088,15 @@ def export_players(
2073
2088
  map_config: MapConfig | None = None,
2074
2089
  cluster_inventories: t.Iterable[CloudInventory] | None = None,
2075
2090
  ) -> list[dict[str, t.Any]]:
2091
+ return list(_iter_players(save, map_config, cluster_inventories))
2092
+
2093
+
2094
+ def _iter_players(
2095
+ save: t.Any,
2096
+ map_config: MapConfig | None,
2097
+ cluster_inventories: t.Iterable[CloudInventory] | None,
2098
+ ) -> t.Iterator[dict[str, t.Any]]:
2099
+ """Yield ASV_Players records one at a time (streaming form of export_players)."""
2076
2100
  profiles = _collection(save, "profiles", Profile)
2077
2101
  lookup = _save_lookup(save)
2078
2102
  pawn_status_by_id = _player_status_by_data_id(save, lookup)
@@ -2083,7 +2107,6 @@ def export_players(
2083
2107
  allocation = _assemble_tribes(save)
2084
2108
  profile_tribeid = allocation["profile_tribeid"]
2085
2109
  tribe_names = allocation["names"]
2086
- results: list[dict[str, t.Any]] = []
2087
2110
  for entry in profiles:
2088
2111
  record, join_keys = _player_record_for(entry, save, pawn_status_by_id, lookup, map_config)
2089
2112
  if record is None:
@@ -2100,11 +2123,10 @@ def export_players(
2100
2123
  inv_list = []
2101
2124
  record["inventory"] = inv_list
2102
2125
  inv_list.extend(spliced)
2103
- results.append(record)
2126
+ yield record
2104
2127
  # Tribe members with no .arkprofile surface as stub players (legacy +N).
2105
2128
  for tid, pid, name in allocation["member_stubs"]:
2106
- results.append(_member_stub_player(tid, pid, name, tribe_names.get(tid, "")))
2107
- return results
2129
+ yield _member_stub_player(tid, pid, name, tribe_names.get(tid, ""))
2108
2130
 
2109
2131
 
2110
2132
  def _strip_rich_color(msg: str) -> str:
@@ -2761,20 +2783,23 @@ def _structure_dict(
2761
2783
 
2762
2784
 
2763
2785
  def export_structures(save: t.Any, map_config: MapConfig | None = None) -> list[dict[str, t.Any]]:
2786
+ return list(_iter_structures(save, map_config))
2787
+
2788
+
2789
+ def _iter_structures(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
2790
+ """Yield ASV_Structures records one at a time (streaming form of export_structures)."""
2764
2791
  objects = _world_objects(save, "get_structures", "structure_objects")
2765
2792
  lookup = _save_lookup(save)
2766
2793
  # Reuse the assembled tribe-name map so a structure's ``tribe`` resolves to
2767
2794
  # the same name the tribes export uses (file TribeName / stub OwnerName).
2768
2795
  tribe_names = _assemble_tribes(save)["names"]
2769
- results: list[dict[str, t.Any]] = []
2770
2796
  for obj in objects:
2771
2797
  team = _int(_prop(obj, "TargetingTeam"))
2772
2798
  if team < _PLAYER_TEAM_THRESHOLD and _is_excluded_abandoned(getattr(obj, "class_name", "") or ""):
2773
2799
  # Unowned map element / crate / debug actor: legacy drops these
2774
2800
  # from ASV_Structures (surfaced via ASV_MapStructures instead).
2775
2801
  continue
2776
- results.append(_structure_dict(obj, save, lookup, map_config, tribe_names))
2777
- return results
2802
+ yield _structure_dict(obj, save, lookup, map_config, tribe_names)
2778
2803
 
2779
2804
 
2780
2805
  # Mirrors C# ContentContainer.cs:846. Order matters: first match wins.
@@ -2810,10 +2835,14 @@ def export_map_structures(
2810
2835
  save: t.Any,
2811
2836
  map_config: MapConfig | None = None,
2812
2837
  ) -> list[dict[str, t.Any]]:
2838
+ return list(_iter_map_structures(save, map_config))
2839
+
2840
+
2841
+ def _iter_map_structures(save: t.Any, map_config: MapConfig | None) -> t.Iterator[dict[str, t.Any]]:
2842
+ """Yield ASV_MapStructures records one at a time (streaming form of export_map_structures)."""
2813
2843
  objects = getattr(save, "objects", None) or []
2814
- all_objs = list(objects.values()) if isinstance(objects, dict) else list(objects)
2844
+ all_objs = objects.values() if isinstance(objects, dict) else objects
2815
2845
  lookup = _save_lookup(save)
2816
- results: list[dict[str, t.Any]] = []
2817
2846
  for obj in all_objs:
2818
2847
  cn = getattr(obj, "class_name", "") or ""
2819
2848
  label = _asv_map_struct_label(cn)
@@ -2829,8 +2858,7 @@ def export_map_structures(
2829
2858
  "inventory": _inventory_items(obj, lookup),
2830
2859
  }
2831
2860
  data.update(_gps_payload(obj, map_config))
2832
- results.append(data)
2833
- return results
2861
+ yield data
2834
2862
 
2835
2863
 
2836
2864
  # Legacy ASVExport.exe filenames. Single canonical schema.
@@ -2845,12 +2873,8 @@ _EXPORT_NAMES: dict[str, str] = {
2845
2873
  }
2846
2874
 
2847
2875
 
2848
- def _wrap_with_meta(
2849
- data: list[dict[str, t.Any]],
2850
- save: t.Any,
2851
- map_config: MapConfig | None = None,
2852
- ) -> dict[str, t.Any]:
2853
- """Wrap a payload in legacy ``{map, day, time, data}`` envelope."""
2876
+ def _meta_head(save: t.Any, map_config: MapConfig | None = None) -> dict[str, t.Any]:
2877
+ """Return the legacy envelope head ``{map, day, time}`` (no ``data`` key)."""
2854
2878
  map_name = getattr(map_config, "name", "") if map_config is not None else ""
2855
2879
  day = 0
2856
2880
  time_str = "00:00"
@@ -2859,7 +2883,74 @@ def _wrap_with_meta(
2859
2883
  day = int(game_time // 86400)
2860
2884
  rem = int(game_time % 86400)
2861
2885
  time_str = f"{rem // 3600:02d}:{(rem % 3600) // 60:02d}"
2862
- return {"map": map_name, "day": day, "time": time_str, "data": data}
2886
+ return {"map": map_name, "day": day, "time": time_str}
2887
+
2888
+
2889
+ def _wrap_with_meta(
2890
+ data: list[dict[str, t.Any]],
2891
+ save: t.Any,
2892
+ map_config: MapConfig | None = None,
2893
+ ) -> dict[str, t.Any]:
2894
+ """Wrap a payload in the legacy ``{map, day, time, data}`` envelope.
2895
+
2896
+ Retained for callers that already hold a materialized list (e.g. the
2897
+ validation harness diffing against legacy). :func:`export_to_files`
2898
+ streams via :func:`_stream_dump` instead and does not use this.
2899
+ """
2900
+ return {**_meta_head(save, map_config), "data": data}
2901
+
2902
+
2903
+ def _stream_dump(
2904
+ fh: t.TextIO,
2905
+ head: dict[str, t.Any] | None,
2906
+ records: t.Iterable[dict[str, t.Any]],
2907
+ dump_kwargs: dict[str, t.Any],
2908
+ ) -> None:
2909
+ """Stream a JSON array to ``fh``, one record at a time.
2910
+
2911
+ Writes a bare ``[records...]`` array when ``head`` is ``None``, otherwise
2912
+ splices the records into ``head`` under a ``"data"`` key
2913
+ (``{..head.., "data": [records...]}``). Each record is encoded and written
2914
+ individually, then released — the full record list never co-resides in
2915
+ memory. The result round-trips identically (at the value level) to
2916
+ ``json.dump(head | {"data": list(records)}, fh, **dump_kwargs)``; only
2917
+ insignificant whitespace differs.
2918
+
2919
+ Pre-conditions: ``records`` is a finite iterable of JSON-serializable
2920
+ dicts; ``dump_kwargs`` carries the same ``indent`` / ``separators`` /
2921
+ ``default`` used elsewhere. Post-conditions: ``fh`` holds one complete,
2922
+ valid JSON document.
2923
+ """
2924
+ indent = dump_kwargs.get("indent")
2925
+ rec_kwargs: dict[str, t.Any] = {"default": dump_kwargs.get("default", str)}
2926
+ if indent:
2927
+ rec_kwargs["indent"] = indent
2928
+ else:
2929
+ rec_kwargs["separators"] = (",", ":")
2930
+ nl = "\n" if indent else ""
2931
+ if head is not None:
2932
+ head_json = json.dumps(head, **rec_kwargs)
2933
+ assert head_json.endswith("}"), "envelope head must serialize to a JSON object"
2934
+ # Splice the data array in just before the head object's closing brace
2935
+ # (indented dumps leave a trailing "\n}", compact a bare "}").
2936
+ fh.write(head_json[: -2 if indent else -1])
2937
+ fh.write(f',{nl}{" " * indent}"data": [' if indent else ',"data":[')
2938
+ rec_pad = " " * (indent * 2) if indent else ""
2939
+ close_pad = " " * indent if indent else ""
2940
+ else:
2941
+ fh.write("[")
2942
+ rec_pad = " " * indent if indent else ""
2943
+ close_pad = ""
2944
+ first = True
2945
+ for rec in records:
2946
+ chunk = json.dumps(rec, **rec_kwargs)
2947
+ if indent:
2948
+ chunk = rec_pad + chunk.replace("\n", "\n" + rec_pad)
2949
+ fh.write(("" if first else ",") + nl + chunk)
2950
+ first = False
2951
+ fh.write("]" if first else nl + close_pad + "]")
2952
+ if head is not None:
2953
+ fh.write(nl + "}")
2863
2954
 
2864
2955
 
2865
2956
  def _load_cluster_inventories(
@@ -2909,18 +3000,33 @@ def export_all(
2909
3000
  Note: returns flat lists per type (not the legacy ``{map, day, time, data}``
2910
3001
  envelope). ``export_to_files`` adds the envelope when writing.
2911
3002
  """
3003
+ return {name: list(records) for name, records in _iter_exports(save, map_config, cluster)}
3004
+
3005
+
3006
+ def _iter_exports(
3007
+ save: t.Any,
3008
+ map_config: MapConfig | None,
3009
+ cluster: str | Path | t.Iterable[CloudInventory] | None,
3010
+ ) -> t.Iterator[tuple[str, t.Iterable[dict[str, t.Any]]]]:
3011
+ """Yield ``(ASV_name, record_iterable)`` one export type at a time.
3012
+
3013
+ The heavy types (tamed / wild / players / structures / map_structures)
3014
+ yield **lazy generators** so :func:`export_to_files` can stream each
3015
+ record to disk and release it — the full per-type list never
3016
+ materializes (it is the dominant export-time allocation on large PvE
3017
+ saves). The small types (tribes / tribe_logs) stay eager lists.
3018
+ :func:`export_all` wraps each iterable in ``list()`` for callers that
3019
+ want every record in memory.
3020
+ """
2912
3021
  cluster_invs = _load_cluster_inventories(cluster)
2913
3022
  cluster_tamed = export_cluster_uploads(cluster_invs, map_config) if cluster_invs else []
2914
- tamed = export_tamed(save, map_config) + cluster_tamed
2915
- return {
2916
- _EXPORT_NAMES["tamed"]: tamed,
2917
- _EXPORT_NAMES["wild"]: export_wild(save, map_config),
2918
- _EXPORT_NAMES["players"]: export_players(save, map_config, cluster_invs or None),
2919
- _EXPORT_NAMES["tribes"]: export_tribes(save),
2920
- _EXPORT_NAMES["structures"]: export_structures(save, map_config),
2921
- _EXPORT_NAMES["tribe_logs"]: export_tribe_logs(save),
2922
- _EXPORT_NAMES["map_structures"]: export_map_structures(save, map_config),
2923
- }
3023
+ yield _EXPORT_NAMES["tamed"], itertools.chain(_iter_tamed(save, map_config), cluster_tamed)
3024
+ yield _EXPORT_NAMES["wild"], _iter_wild(save, map_config)
3025
+ yield _EXPORT_NAMES["players"], _iter_players(save, map_config, cluster_invs or None)
3026
+ yield _EXPORT_NAMES["tribes"], export_tribes(save)
3027
+ yield _EXPORT_NAMES["structures"], _iter_structures(save, map_config)
3028
+ yield _EXPORT_NAMES["tribe_logs"], export_tribe_logs(save)
3029
+ yield _EXPORT_NAMES["map_structures"], _iter_map_structures(save, map_config)
2924
3030
 
2925
3031
 
2926
3032
  def export_to_files(
@@ -2952,9 +3058,15 @@ def export_to_files(
2952
3058
  dump_kwargs: dict[str, t.Any] = {"separators": (",", ":"), "default": str}
2953
3059
  else:
2954
3060
  dump_kwargs = {"indent": 2, "default": str}
2955
- for name, data in export_all(save, map_config, cluster=cluster).items():
2956
- payload: t.Any = _wrap_with_meta(data, save, map_config) if wrap else data
3061
+ # Stream each export type record-by-record straight to its file: a record
3062
+ # is built, encoded, written, then released. The full per-type list never
3063
+ # materializes and no whole-file JSON string is built — both were the
3064
+ # measured peak-RAM drivers on large PvE saves (the structures list alone
3065
+ # was hundreds of MB of nested inventory dicts).
3066
+ head = _meta_head(save, map_config) if wrap else None
3067
+ for name, records in _iter_exports(save, map_config, cluster):
2957
3068
  path = out / f"{name}.json"
2958
- path.write_text(json.dumps(payload, **dump_kwargs), encoding="utf-8")
3069
+ with path.open("w", encoding="utf-8") as fh:
3070
+ _stream_dump(fh, head, records, dump_kwargs)
2959
3071
  created.append(path)
2960
3072
  return created
@@ -64,23 +64,34 @@ class GameObject:
64
64
  parent: GameObject | None = field(default=None, repr=False)
65
65
  components: dict[str, GameObject] = field(default_factory=dict, repr=False)
66
66
 
67
- # Lazy property index: name -> {index: Property}. Built on first lookup,
68
- # invalidated by setting to None when properties is mutated.
69
- _prop_index: dict[str, dict[int, "Property"]] | None = field(default=None, repr=False, compare=False)
67
+ # Lazy property index: name -> Property (single-index, the common case) OR
68
+ # name -> {index: Property} (only when a name carries >1 distinct index).
69
+ # Storing the bare Property for single-index names avoids allocating one
70
+ # inner dict per (object, name): on a 300k-object save that was ~5.4M dicts
71
+ # / ~600 MB of pure index overhead built during export. Built on first
72
+ # lookup, invalidated by setting to None when properties is mutated.
73
+ _prop_index: dict[str, "Property | dict[int, Property]"] | None = field(
74
+ default=None, repr=False, compare=False
75
+ )
70
76
 
71
- def _build_prop_index(self) -> dict[str, dict[int, "Property"]]:
72
- idx: dict[str, dict[int, "Property"]] = {}
77
+ def _build_prop_index(self) -> dict[str, "Property | dict[int, Property]"]:
78
+ # First-writer wins, matching legacy C# GetPropertyValue
79
+ # (IPropertyContainer.cs:45) which returns the first (name, index)
80
+ # match. _serialize_properties keeps all occurrences in the additive
81
+ # `properties` dict; lookups intentionally mirror legacy here. Do not
82
+ # switch to last-wins.
83
+ idx: dict[str, "Property | dict[int, Property]"] = {}
73
84
  for prop in self.properties:
74
- bucket = idx.get(prop.name)
75
- if bucket is None:
76
- idx[prop.name] = {prop.index: prop}
77
- else:
78
- # First-writer wins, matching legacy C# GetPropertyValue
79
- # (IPropertyContainer.cs:45) which returns the first (name,
80
- # index) match. _serialize_properties keeps all occurrences in
81
- # the additive `properties` dict; lookups intentionally mirror
82
- # legacy here. Do not switch to last-wins.
83
- bucket.setdefault(prop.index, prop)
85
+ existing = idx.get(prop.name)
86
+ if existing is None:
87
+ idx[prop.name] = prop # common case: store the bare Property
88
+ elif isinstance(existing, dict):
89
+ existing.setdefault(prop.index, prop)
90
+ elif existing.index != prop.index:
91
+ # Second distinct index for this name: promote to a dict.
92
+ # `existing` first preserves first-writer iteration order.
93
+ idx[prop.name] = {existing.index: existing, prop.index: prop}
94
+ # else: duplicate (name, index) -> keep first-writer `existing`.
84
95
  self._prop_index = idx
85
96
  return idx
86
97
 
@@ -108,11 +119,16 @@ class GameObject:
108
119
  """Get a property by name and optional index (None = first by insertion)."""
109
120
  idx = self._prop_index if self._prop_index is not None else self._build_prop_index()
110
121
  bucket = idx.get(name)
111
- if not bucket:
122
+ if bucket is None:
112
123
  return None
113
- if index is None:
114
- return next(iter(bucket.values()))
115
- return bucket.get(index)
124
+ if isinstance(bucket, dict):
125
+ if index is None:
126
+ return next(iter(bucket.values()))
127
+ return bucket.get(index)
128
+ # Single-index bucket: the bare Property.
129
+ if index is None or bucket.index == index:
130
+ return bucket
131
+ return None
116
132
 
117
133
  def get_property_value(self, name: str, default: t.Any = None, index: int | None = None) -> t.Any:
118
134
  """Get a property value by name (returns default if missing)."""
@@ -123,7 +139,11 @@ class GameObject:
123
139
  """Get all properties with the given name (any index)."""
124
140
  idx = self._prop_index if self._prop_index is not None else self._build_prop_index()
125
141
  bucket = idx.get(name)
126
- return list(bucket.values()) if bucket else []
142
+ if bucket is None:
143
+ return []
144
+ if isinstance(bucket, dict):
145
+ return list(bucket.values())
146
+ return [bucket]
127
147
 
128
148
  def has_property(self, name: str) -> bool:
129
149
  """Check if this object has a property with the given name."""