arkparser 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. {arkparser-0.8.0 → arkparser-0.9.0}/PKG-INFO +23 -1
  2. {arkparser-0.8.0 → arkparser-0.9.0}/README.md +22 -0
  3. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/__init__.py +3 -2
  4. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/__init__.py +2 -0
  5. arkparser-0.9.0/arkparser/files/embedded_store.py +261 -0
  6. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/world_save.py +32 -0
  7. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser.egg-info/PKG-INFO +23 -1
  8. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser.egg-info/SOURCES.txt +2 -0
  9. {arkparser-0.8.0 → arkparser-0.9.0}/pyproject.toml +1 -1
  10. arkparser-0.9.0/tests/test_embedded_store.py +197 -0
  11. {arkparser-0.8.0 → arkparser-0.9.0}/LICENSE +0 -0
  12. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/__init__.py +0 -0
  13. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/binary_reader.py +0 -0
  14. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/exceptions.py +0 -0
  15. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/map_config.py +0 -0
  16. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/normalization.py +0 -0
  17. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/types.py +0 -0
  18. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/common/version_detection.py +0 -0
  19. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/data_models.py +0 -0
  20. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/export.py +0 -0
  21. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/base.py +0 -0
  22. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/cloud_inventory.py +0 -0
  23. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/profile.py +0 -0
  24. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/files/tribe.py +0 -0
  25. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/game_objects/__init__.py +0 -0
  26. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/game_objects/container.py +0 -0
  27. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/game_objects/game_object.py +0 -0
  28. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/game_objects/location.py +0 -0
  29. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/__init__.py +0 -0
  30. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/base.py +0 -0
  31. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/byte_property.py +0 -0
  32. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/compound.py +0 -0
  33. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/primitives.py +0 -0
  34. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/properties/registry.py +0 -0
  35. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/__init__.py +0 -0
  36. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/base.py +0 -0
  37. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/colors.py +0 -0
  38. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/misc.py +0 -0
  39. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/property_list.py +0 -0
  40. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/registry.py +0 -0
  41. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser/structs/vectors.py +0 -0
  42. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser.egg-info/dependency_links.txt +0 -0
  43. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser.egg-info/requires.txt +0 -0
  44. {arkparser-0.8.0 → arkparser-0.9.0}/arkparser.egg-info/top_level.txt +0 -0
  45. {arkparser-0.8.0 → arkparser-0.9.0}/setup.cfg +0 -0
  46. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_asa_header_position.py +0 -0
  47. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_asa_name_table.py +0 -0
  48. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_asa_profile_drop.py +0 -0
  49. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_ase_cluster_drift.py +0 -0
  50. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_binary_reader.py +0 -0
  51. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_binary_reader_layouts.py +0 -0
  52. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_cloud_export.py +0 -0
  53. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_cloud_inventory.py +0 -0
  54. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_cryopod_export.py +0 -0
  55. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_current_stats.py +0 -0
  56. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_data_models.py +0 -0
  57. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_export.py +0 -0
  58. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_game_objects.py +0 -0
  59. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_golden_exports.py +0 -0
  60. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_lazy_properties.py +0 -0
  61. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_profile.py +0 -0
  62. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_review_fixes.py +0 -0
  63. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_tribe.py +0 -0
  64. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_v13_property_layouts.py +0 -0
  65. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_version_detection.py +0 -0
  66. {arkparser-0.8.0 → arkparser-0.9.0}/tests/test_world_save.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arkparser
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Pure Python parser for ARK: Survival Evolved and ARK: Survival Ascended save files
5
5
  Author: Vertyco
6
6
  License-Expression: MIT
@@ -133,6 +133,27 @@ print(f"Parse errors: {save.parse_error_count}")
133
133
  print(f"Is ASA: {save.is_asa}")
134
134
  ```
135
135
 
136
+ ### Player/tribe data packed in the save (ASA)
137
+
138
+ Some ASA servers (Nitrado observed) write no `.arkprofile` / `.arktribe` files.
139
+ The same files are packed inside the `.ark` instead. Read them and attach them
140
+ the same way you would attach sidecar files:
141
+
142
+ ```python
143
+ from arkparser import WorldSave, export_all
144
+
145
+ save = WorldSave.load("path/to/Ragnarok_WP.ark", lazy_properties=True)
146
+ store = save.read_embedded_store()
147
+ save.profiles, save.tribes = store.profiles, store.tribes
148
+ data = export_all(save)
149
+ ```
150
+
151
+ `EmbeddedStore` holds `profiles`, `tribes`, `superseded_profiles` /
152
+ `superseded_tribes` (old copies the game leaves behind, skipped), and
153
+ `errors` (one line per skipped entry; a bad entry never fails the load). Saves
154
+ from servers that write sidecar files return an empty store. Not run by
155
+ `load()`, so it costs nothing unless called.
156
+
136
157
  ### Low-memory parsing (large saves)
137
158
 
138
159
  By default every object's property block is parsed up front, which on a busy
@@ -577,6 +598,7 @@ Key methods:
577
598
  | `get_object_by_guid(guid)` | `GameObject \| None` | Lookup by GUID (ASA) |
578
599
  | `get_actor_location(guid)` | `LocationData \| None` | Actor location by GUID (ASA) |
579
600
  | `iter_cryopod_creatures()` | `Iterator[(GameObject, CryopodCreature)]` | Walk filled cryopods |
601
+ | `read_embedded_store()` | `EmbeddedStore` | Player and tribe files packed inside an ASA save (servers that write no `.arkprofile` / `.arktribe`); see "Player/tribe data packed in the save" |
580
602
 
581
603
  ### Game Objects
582
604
 
@@ -99,6 +99,27 @@ print(f"Parse errors: {save.parse_error_count}")
99
99
  print(f"Is ASA: {save.is_asa}")
100
100
  ```
101
101
 
102
+ ### Player/tribe data packed in the save (ASA)
103
+
104
+ Some ASA servers (Nitrado observed) write no `.arkprofile` / `.arktribe` files.
105
+ The same files are packed inside the `.ark` instead. Read them and attach them
106
+ the same way you would attach sidecar files:
107
+
108
+ ```python
109
+ from arkparser import WorldSave, export_all
110
+
111
+ save = WorldSave.load("path/to/Ragnarok_WP.ark", lazy_properties=True)
112
+ store = save.read_embedded_store()
113
+ save.profiles, save.tribes = store.profiles, store.tribes
114
+ data = export_all(save)
115
+ ```
116
+
117
+ `EmbeddedStore` holds `profiles`, `tribes`, `superseded_profiles` /
118
+ `superseded_tribes` (old copies the game leaves behind, skipped), and
119
+ `errors` (one line per skipped entry; a bad entry never fails the load). Saves
120
+ from servers that write sidecar files return an empty store. Not run by
121
+ `load()`, so it costs nothing unless called.
122
+
102
123
  ### Low-memory parsing (large saves)
103
124
 
104
125
  By default every object's property block is parsed up front, which on a busy
@@ -543,6 +564,7 @@ Key methods:
543
564
  | `get_object_by_guid(guid)` | `GameObject \| None` | Lookup by GUID (ASA) |
544
565
  | `get_actor_location(guid)` | `LocationData \| None` | Actor location by GUID (ASA) |
545
566
  | `iter_cryopod_creatures()` | `Iterator[(GameObject, CryopodCreature)]` | Walk filled cryopods |
567
+ | `read_embedded_store()` | `EmbeddedStore` | Player and tribe files packed inside an ASA save (servers that write no `.arkprofile` / `.arktribe`); see "Player/tribe data packed in the save" |
546
568
 
547
569
  ### Game Objects
548
570
 
@@ -48,7 +48,7 @@ from arkparser.export import (
48
48
  export_tribes,
49
49
  export_wild,
50
50
  )
51
- from arkparser.files import CloudInventory, Profile, Tribe, WorldSave
51
+ from arkparser.files import CloudInventory, EmbeddedStore, Profile, Tribe, WorldSave
52
52
  from arkparser.game_objects import GameObject, GameObjectContainer, LocationData
53
53
 
54
54
  # Convenience alias - users may know this as "Obelisk" from the game
@@ -61,6 +61,7 @@ __all__ = [
61
61
  "CloudInventory",
62
62
  "Obelisk",
63
63
  "WorldSave",
64
+ "EmbeddedStore",
64
65
  # Cloud-inventory data models
65
66
  "UploadedCreature",
66
67
  "UploadedItem",
@@ -95,4 +96,4 @@ __all__ = [
95
96
  "ArkParseError",
96
97
  ]
97
98
 
98
- __version__ = "0.8.0"
99
+ __version__ = "0.9.0"
@@ -12,6 +12,7 @@ All parsers support both ASE and ASA formats with automatic detection.
12
12
 
13
13
  from .base import ArkFile
14
14
  from .cloud_inventory import CloudInventory
15
+ from .embedded_store import EmbeddedStore
15
16
  from .profile import Profile
16
17
  from .tribe import Tribe
17
18
  from .world_save import WorldSave
@@ -22,4 +23,5 @@ __all__ = [
22
23
  "Tribe",
23
24
  "CloudInventory",
24
25
  "WorldSave",
26
+ "EmbeddedStore",
25
27
  ]
@@ -0,0 +1,261 @@
1
+ """
2
+ Player and tribe store packed inside an ASA world save.
3
+
4
+ Some ASA servers (Nitrado observed) never write ``.arkprofile`` / ``.arktribe``
5
+ files. The same files are packed into the save's ``custom`` table under the
6
+ ``GameModeCustomBytes`` key instead. Servers that do write the files leave that
7
+ row at a few dozen bytes.
8
+
9
+ Blob layout (int32 little-endian)::
10
+
11
+ Int32 format # 1
12
+ repeat (tribes, then profiles):
13
+ Int32 section_size # bytes of packed files that follow
14
+ Int32 count # size 0 + count 0 = terminator, ends the blob
15
+ Bytes files[section_size] # complete .arktribe / .arkprofile files
16
+ count x { Bytes(8) key, Int32 offset, Int32 size }
17
+
18
+ The store is append-style: each save writes a fresh copy of a changed tribe or
19
+ profile and zeroes the old copy's key. Only entries with a non-zero key are
20
+ current. Each packed file is byte-identical in format to a sidecar file, so it
21
+ loads with ``Tribe.load(bytes)`` / ``Profile.load(bytes)`` unchanged.
22
+
23
+ Pieces are read with ``substr()`` slices so a large store never sits in memory
24
+ whole: the index first, then one current file at a time.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import logging
30
+ import sqlite3
31
+ import struct
32
+ import typing as t
33
+ from dataclasses import dataclass, field
34
+
35
+ from ..common.exceptions import CorruptDataError
36
+ from ..game_objects.game_object import MAX_OBJECT_COUNT
37
+ from .profile import Profile
38
+ from .tribe import Tribe
39
+
40
+ logger = logging.getLogger(__name__)
41
+
42
+ STORE_KEY = "GameModeCustomBytes"
43
+ STORE_FORMAT = 1
44
+ NO_STORE_FORMAT = 0
45
+ # Tribes + profiles + one spare for a future section type.
46
+ MAX_SECTIONS = 3
47
+ INDEX_ENTRY = struct.Struct("<8sii")
48
+ ZERO_KEY = b"\x00" * 8
49
+ # Where the class-name string sits in a v7 packed file: version, two header
50
+ # ints, object count, then the first object's 16-byte GUID.
51
+ CLASS_NAME_OFFSET = 32
52
+ MAX_CLASS_NAME = 256
53
+
54
+ _SIZE_SQL = "SELECT length(value) FROM custom WHERE key = ?"
55
+ _SLICE_SQL = "SELECT substr(value, ?, ?) FROM custom WHERE key = ?"
56
+
57
+
58
+ @dataclass
59
+ class EmbeddedStore:
60
+ """Current profiles and tribes read out of a save's packed store.
61
+
62
+ Attributes:
63
+ profiles: One ``Profile`` per player (latest copy).
64
+ tribes: One ``Tribe`` per tribe (latest copy).
65
+ superseded_profiles: Old profile copies skipped (zero key).
66
+ superseded_tribes: Old tribe copies skipped (zero key).
67
+ errors: Problems found, one line each. A bad entry is skipped and the
68
+ rest still load; a bad layout empties the result.
69
+ """
70
+
71
+ profiles: list[Profile] = field(default_factory=list)
72
+ tribes: list[Tribe] = field(default_factory=list)
73
+ superseded_profiles: int = 0
74
+ superseded_tribes: int = 0
75
+ errors: list[str] = field(default_factory=list)
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class StoreSection:
80
+ """One section's location in the blob plus its raw index entries."""
81
+
82
+ data_start: int
83
+ data_size: int
84
+ entries: list[tuple[bytes, int, int]]
85
+
86
+
87
+ class BlobSlicer:
88
+ """Reads byte ranges of the store row without loading the whole value."""
89
+
90
+ def __init__(self, conn: sqlite3.Connection, size: int) -> None:
91
+ assert size >= 0, "negative blob size"
92
+ self.conn = conn
93
+ self.size = size
94
+
95
+ def read(self, offset: int, length: int) -> bytes:
96
+ """Return ``length`` bytes at 0-based ``offset``, or raise if out of range."""
97
+ if offset < 0 or length < 0 or offset + length > self.size:
98
+ raise CorruptDataError(
99
+ f"store read {offset}+{length} past blob end {self.size}"
100
+ )
101
+ if length == 0:
102
+ return b""
103
+ row = self.conn.execute(_SLICE_SQL, (offset + 1, length, STORE_KEY)).fetchone()
104
+ data = bytes(row[0]) if row is not None and row[0] is not None else b""
105
+ if len(data) != length:
106
+ raise CorruptDataError(f"store read {offset}+{length} returned {len(data)} bytes")
107
+ return data
108
+
109
+ def read_int32_pair(self, offset: int) -> tuple[int, int]:
110
+ """Read two little-endian int32 values at ``offset``."""
111
+ first, second = struct.unpack("<2i", self.read(offset, 8))
112
+ return first, second
113
+
114
+
115
+ def load_embedded_store(conn: sqlite3.Connection) -> EmbeddedStore:
116
+ """Read the packed profile/tribe store from an open ASA save connection.
117
+
118
+ Pre: ``conn`` is an open connection to an ASA save (has a ``custom``
119
+ table). Post: returns the current entries; an absent or near-empty store
120
+ returns an empty result with no errors, and a malformed layout returns an
121
+ empty result with one error line. Never raises for bad store data.
122
+ """
123
+ assert conn is not None, "connection required"
124
+ row = conn.execute(_SIZE_SQL, (STORE_KEY,)).fetchone()
125
+ size = int(row[0]) if row is not None and row[0] is not None else 0
126
+ store = EmbeddedStore()
127
+ if size <= 8:
128
+ return store
129
+ slicer = BlobSlicer(conn, size)
130
+ # Servers that write sidecar files leave this row all zeros (20-28 bytes
131
+ # observed), so format 0 means "no store", not a malformed one.
132
+ if struct.unpack("<i", slicer.read(0, 4))[0] == NO_STORE_FORMAT:
133
+ return store
134
+ try:
135
+ sections = read_sections(slicer)
136
+ except CorruptDataError as exc:
137
+ store.errors.append(f"store layout: {exc}")
138
+ return store
139
+ for number, section in enumerate(sections):
140
+ load_section(slicer, section, number, store)
141
+ assert len(store.profiles) + len(store.tribes) <= MAX_OBJECT_COUNT, "store overflow"
142
+ return store
143
+
144
+
145
+ def read_sections(slicer: BlobSlicer) -> list[StoreSection]:
146
+ """Walk the blob's section headers and index tables.
147
+
148
+ Pre: blob is longer than 8 bytes. Post: returns every section up to the
149
+ terminator, with bounds checked; raises ``CorruptDataError`` on any
150
+ impossible count, size, or a missing terminator.
151
+ """
152
+ assert slicer.size > 8, "store too small to hold a section"
153
+ fmt = struct.unpack("<i", slicer.read(0, 4))[0]
154
+ if fmt != STORE_FORMAT:
155
+ raise CorruptDataError(f"unknown store format {fmt}")
156
+ sections: list[StoreSection] = []
157
+ pos = 4
158
+ for _ in range(MAX_SECTIONS + 1):
159
+ data_size, count = slicer.read_int32_pair(pos)
160
+ pos += 8
161
+ if data_size == 0 and count == 0:
162
+ assert pos <= slicer.size, "terminator past blob end"
163
+ return sections
164
+ if data_size < 0 or not 0 <= count <= MAX_OBJECT_COUNT:
165
+ raise CorruptDataError(f"section size {data_size} / count {count} out of range")
166
+ if len(sections) == MAX_SECTIONS:
167
+ raise CorruptDataError(f"more than {MAX_SECTIONS} sections")
168
+ index_start = pos + data_size
169
+ raw_index = slicer.read(index_start, count * INDEX_ENTRY.size)
170
+ entries = [tuple(e) for e in INDEX_ENTRY.iter_unpack(raw_index)]
171
+ sections.append(StoreSection(pos, data_size, t.cast(list[tuple[bytes, int, int]], entries)))
172
+ pos = index_start + count * INDEX_ENTRY.size
173
+ raise CorruptDataError("no section terminator")
174
+
175
+
176
+ def entry_in_bounds(section: StoreSection, offset: int, size: int) -> bool:
177
+ """True when an index entry's file lies inside its section's data."""
178
+ assert section.data_size >= 0, "negative section size"
179
+ return 0 <= offset and 0 < size and offset + size <= section.data_size
180
+
181
+
182
+ def section_kind(slicer: BlobSlicer, section: StoreSection) -> type[Profile] | type[Tribe] | None:
183
+ """Classify a section by the class name in its first in-bounds file.
184
+
185
+ Pre: section bounds already validated. Post: ``Tribe``, ``Profile``, or
186
+ ``None`` when no entry names a known class.
187
+ """
188
+ assert section.entries is not None, "section has no index"
189
+ for _, offset, size in section.entries:
190
+ if not entry_in_bounds(section, offset, size) or size < CLASS_NAME_OFFSET + 4:
191
+ continue
192
+ head = slicer.read(section.data_start + offset, min(size, CLASS_NAME_OFFSET + 4 + MAX_CLASS_NAME))
193
+ length = struct.unpack_from("<i", head, CLASS_NAME_OFFSET)[0]
194
+ name = head[CLASS_NAME_OFFSET + 4 : CLASS_NAME_OFFSET + 4 + max(0, length)]
195
+ if Tribe.MAIN_CLASS_NAME.encode() in name:
196
+ return Tribe
197
+ if Profile.MAIN_CLASS_NAME.encode() in name:
198
+ return Profile
199
+ return None
200
+ return None
201
+
202
+
203
+ def load_section(slicer: BlobSlicer, section: StoreSection, number: int, store: EmbeddedStore) -> None:
204
+ """Load the current entries of one section into ``store``.
205
+
206
+ Pre: ``section`` came from :func:`read_sections`. Post: current files are
207
+ parsed and de-duplicated by id (latest in index order wins); superseded
208
+ copies are counted; per-entry failures are recorded, never raised.
209
+ """
210
+ assert number >= 0, "negative section number"
211
+ kind = section_kind(slicer, section)
212
+ if kind is None:
213
+ store.errors.append(f"section {number}: unknown content, skipped")
214
+ return
215
+ label = "tribe" if kind is Tribe else "profile"
216
+ latest: dict[int, Profile | Tribe] = {}
217
+ superseded = 0
218
+ for i, (key, offset, size) in enumerate(section.entries):
219
+ if key == ZERO_KEY:
220
+ superseded += 1
221
+ continue
222
+ loaded = load_entry(slicer, section, kind, offset, size)
223
+ if isinstance(loaded, str):
224
+ store.errors.append(f"{label} #{i}: {loaded}")
225
+ continue
226
+ ident = loaded.tribe_id if isinstance(loaded, Tribe) else loaded.player_id
227
+ latest[ident if isinstance(ident, int) else -1 - i] = loaded
228
+ if kind is Tribe:
229
+ store.tribes.extend(t.cast(list[Tribe], list(latest.values())))
230
+ store.superseded_tribes += superseded
231
+ else:
232
+ store.profiles.extend(t.cast(list[Profile], list(latest.values())))
233
+ store.superseded_profiles += superseded
234
+ assert superseded <= len(section.entries), "superseded count exceeds entries"
235
+
236
+
237
+ def load_entry(
238
+ slicer: BlobSlicer,
239
+ section: StoreSection,
240
+ kind: type[Profile] | type[Tribe],
241
+ offset: int,
242
+ size: int,
243
+ ) -> Profile | Tribe | str:
244
+ """Parse one packed file; return the object or an error string.
245
+
246
+ Pre: ``kind`` is ``Profile`` or ``Tribe``. Post: never raises for bad
247
+ entry data; out-of-bounds entries, unknown versions, and parse failures
248
+ come back as a one-line reason.
249
+ """
250
+ assert kind in (Profile, Tribe), "unsupported entry kind"
251
+ if not entry_in_bounds(section, offset, size) or size < 4:
252
+ return f"offset {offset} size {size} outside section of {section.data_size} bytes"
253
+ raw = slicer.read(section.data_start + offset, size)
254
+ version = struct.unpack_from("<i", raw, 0)[0]
255
+ if version not in kind.VALID_VERSIONS:
256
+ return f"unsupported file version {version}"
257
+ try:
258
+ return kind.load(raw)
259
+ except Exception as exc: # noqa: BLE001 - one bad entry must not sink the store
260
+ logger.debug("Embedded %s entry failed to parse", kind.__name__, exc_info=True)
261
+ return f"{type(exc).__name__}: {exc}"
@@ -40,6 +40,7 @@ from ..game_objects.container import GameObjectContainer
40
40
  from ..game_objects.game_object import MAX_OBJECT_COUNT, GameObject
41
41
  from ..game_objects.location import LocationData
42
42
  from ..properties.registry import read_properties, read_properties_partial
43
+ from .embedded_store import EmbeddedStore, load_embedded_store
43
44
 
44
45
  logger = logging.getLogger(__name__)
45
46
 
@@ -171,6 +172,9 @@ class WorldSave:
171
172
  # matching the legacy ContentContainer.GetApproxDateTimeOf formula:
172
173
  # real = file_mtime + (object_time - game_time).
173
174
  file_mtime: dt.datetime | None = None
175
+ # Path the save was loaded from (None when loaded from bytes). Lets
176
+ # read_embedded_store reopen an eagerly loaded ASA save's database.
177
+ source_path: Path | None = None
174
178
 
175
179
  # ASE-specific
176
180
  embedded_data: list[EmbeddedData] = field(default_factory=list)
@@ -324,6 +328,7 @@ class WorldSave:
324
328
  reader = BinaryReader.from_file(path)
325
329
  save = cls._parse_ase(reader, load_properties, lazy_properties)
326
330
  save.file_mtime = mtime
331
+ save.source_path = path
327
332
  return save
328
333
 
329
334
  # ------------------------------------------------------------------
@@ -499,6 +504,33 @@ class WorldSave:
499
504
  if cryo is not None:
500
505
  yield obj, cryo
501
506
 
507
+ def read_embedded_store(self) -> EmbeddedStore:
508
+ """Read the player/tribe files packed inside an ASA save.
509
+
510
+ Some ASA servers write no ``.arkprofile`` / ``.arktribe`` files and
511
+ pack them into the save instead (see ``embedded_store``). Not run by
512
+ :meth:`load`; callers that want the data ask for it, so load cost is
513
+ unchanged for everyone else.
514
+
515
+ Pre: the save was loaded from a path (ASA) or is ASE. Post: returns
516
+ the current profiles and tribes; ASE saves and saves without a store
517
+ return an empty :class:`EmbeddedStore`. Reuses the lazy connection
518
+ when present, otherwise opens and closes a read-only one.
519
+ """
520
+ if not self.is_asa:
521
+ return EmbeddedStore()
522
+ if self._lazy_conn is not None:
523
+ return load_embedded_store(self._lazy_conn)
524
+ assert self.source_path is not None, "ASA saves always load from a path"
525
+ uri = self.source_path.resolve().as_uri() + "?mode=ro"
526
+ conn = sqlite3.connect(uri, uri=True)
527
+ try:
528
+ store = load_embedded_store(conn)
529
+ finally:
530
+ conn.close()
531
+ assert isinstance(store, EmbeddedStore), "store loader returned wrong type"
532
+ return store
533
+
502
534
  # ------------------------------------------------------------------
503
535
  # Properties
504
536
  # ------------------------------------------------------------------
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arkparser
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Pure Python parser for ARK: Survival Evolved and ARK: Survival Ascended save files
5
5
  Author: Vertyco
6
6
  License-Expression: MIT
@@ -133,6 +133,27 @@ print(f"Parse errors: {save.parse_error_count}")
133
133
  print(f"Is ASA: {save.is_asa}")
134
134
  ```
135
135
 
136
+ ### Player/tribe data packed in the save (ASA)
137
+
138
+ Some ASA servers (Nitrado observed) write no `.arkprofile` / `.arktribe` files.
139
+ The same files are packed inside the `.ark` instead. Read them and attach them
140
+ the same way you would attach sidecar files:
141
+
142
+ ```python
143
+ from arkparser import WorldSave, export_all
144
+
145
+ save = WorldSave.load("path/to/Ragnarok_WP.ark", lazy_properties=True)
146
+ store = save.read_embedded_store()
147
+ save.profiles, save.tribes = store.profiles, store.tribes
148
+ data = export_all(save)
149
+ ```
150
+
151
+ `EmbeddedStore` holds `profiles`, `tribes`, `superseded_profiles` /
152
+ `superseded_tribes` (old copies the game leaves behind, skipped), and
153
+ `errors` (one line per skipped entry; a bad entry never fails the load). Saves
154
+ from servers that write sidecar files return an empty store. Not run by
155
+ `load()`, so it costs nothing unless called.
156
+
136
157
  ### Low-memory parsing (large saves)
137
158
 
138
159
  By default every object's property block is parsed up front, which on a busy
@@ -577,6 +598,7 @@ Key methods:
577
598
  | `get_object_by_guid(guid)` | `GameObject \| None` | Lookup by GUID (ASA) |
578
599
  | `get_actor_location(guid)` | `LocationData \| None` | Actor location by GUID (ASA) |
579
600
  | `iter_cryopod_creatures()` | `Iterator[(GameObject, CryopodCreature)]` | Walk filled cryopods |
601
+ | `read_embedded_store()` | `EmbeddedStore` | Player and tribe files packed inside an ASA save (servers that write no `.arkprofile` / `.arktribe`); see "Player/tribe data packed in the save" |
580
602
 
581
603
  ### Game Objects
582
604
 
@@ -19,6 +19,7 @@ arkparser/common/version_detection.py
19
19
  arkparser/files/__init__.py
20
20
  arkparser/files/base.py
21
21
  arkparser/files/cloud_inventory.py
22
+ arkparser/files/embedded_store.py
22
23
  arkparser/files/profile.py
23
24
  arkparser/files/tribe.py
24
25
  arkparser/files/world_save.py
@@ -50,6 +51,7 @@ tests/test_cloud_inventory.py
50
51
  tests/test_cryopod_export.py
51
52
  tests/test_current_stats.py
52
53
  tests/test_data_models.py
54
+ tests/test_embedded_store.py
53
55
  tests/test_export.py
54
56
  tests/test_game_objects.py
55
57
  tests/test_golden_exports.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "arkparser"
7
- version = "0.8.0"
7
+ version = "0.9.0"
8
8
  description = "Pure Python parser for ARK: Survival Evolved and ARK: Survival Ascended save files"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -0,0 +1,197 @@
1
+ """Tests for the player/tribe store packed inside ASA world saves.
2
+
3
+ Layout tests build stores in an in-memory SQLite database and need no files.
4
+ Content tests pack real sidecar files from the ScorchedEarth reference bundle
5
+ (skipped when absent). The real Nitrado sample is local-only and never
6
+ committed: it holds a third party's player data.
7
+ """
8
+
9
+ import shutil
10
+ import sqlite3
11
+ import struct
12
+ from pathlib import Path
13
+
14
+ import pytest
15
+
16
+ from arkparser import EmbeddedStore, Profile, Tribe, WorldSave, export_all
17
+ from arkparser.files.embedded_store import STORE_KEY, load_embedded_store
18
+
19
+ ASA_SAVES = Path(__file__).parent.parent / "references" / "local_saves" / "survival_ascended"
20
+ SCORCHED_DIR = ASA_SAVES / "scorchedearth_wp"
21
+ # Outside survival_ascended/ so the golden suite never discovers it.
22
+ NITRADO_SAMPLE = ASA_SAVES.parent / "third_party" / "nitrado_store_ragnarok" / "Ragnarok_WP.ark"
23
+
24
+ Entry = tuple[bytes, bytes] # (8-byte key, packed file bytes)
25
+
26
+
27
+ def pack_section(entries: list[Entry]) -> bytes:
28
+ """One section: size, count, files back to back, then the index."""
29
+ files = b"".join(data for _, data in entries)
30
+ index = b""
31
+ offset = 0
32
+ for key, data in entries:
33
+ index += struct.pack("<8sii", key, offset, len(data))
34
+ offset += len(data)
35
+ return struct.pack("<2i", len(files), len(entries)) + files + index
36
+
37
+
38
+ def pack_store(*sections: bytes, fmt: int = 1, terminator: bool = True) -> bytes:
39
+ return struct.pack("<i", fmt) + b"".join(sections) + (b"\x00" * 8 if terminator else b"")
40
+
41
+
42
+ def store_db(blob: bytes | None) -> sqlite3.Connection:
43
+ conn = sqlite3.connect(":memory:")
44
+ conn.execute("CREATE TABLE custom (key VARCHAR(64) primary key not null, value blob not null)")
45
+ if blob is not None:
46
+ conn.execute("INSERT INTO custom VALUES (?, ?)", (STORE_KEY, blob))
47
+ return conn
48
+
49
+
50
+ def key(n: int) -> bytes:
51
+ return struct.pack("<q", n)
52
+
53
+
54
+ # ---------------------------------------------------------------- layout
55
+
56
+
57
+ @pytest.mark.parametrize("blob", [None, b"\x00" * 20, b"\x00" * 28, b"\x01" * 8])
58
+ def test_missing_or_tiny_store_is_empty(blob: bytes | None) -> None:
59
+ store = load_embedded_store(store_db(blob))
60
+ assert (store.profiles, store.tribes, store.errors) == ([], [], [])
61
+
62
+
63
+ def test_unknown_format_reports_one_error() -> None:
64
+ store = load_embedded_store(store_db(pack_store(pack_section([]), fmt=2)))
65
+ assert store.tribes == [] and len(store.errors) == 1
66
+ assert "format 2" in store.errors[0]
67
+
68
+
69
+ def test_truncated_index_reports_one_error() -> None:
70
+ blob = pack_store(pack_section([(key(1), b"x" * 40)]), terminator=False)[:-8]
71
+ store = load_embedded_store(store_db(blob))
72
+ assert store.tribes == [] and len(store.errors) == 1
73
+
74
+
75
+ def test_missing_terminator_reports_one_error() -> None:
76
+ blob = pack_store(pack_section([(key(1), b"z" * 40)]), terminator=False)
77
+ store = load_embedded_store(store_db(blob))
78
+ assert len(store.errors) == 1
79
+
80
+
81
+ def test_too_many_sections_reports_one_error() -> None:
82
+ section = pack_section([(key(1), b"y" * 40)])
83
+ store = load_embedded_store(store_db(pack_store(section, section, section, section)))
84
+ assert store.tribes == [] and len(store.errors) == 1
85
+ assert "sections" in store.errors[0]
86
+
87
+
88
+ def test_unknown_section_content_is_skipped() -> None:
89
+ junk = b"\x07\x00\x00\x00" + b"\x00" * 28 + struct.pack("<i", 8) + b"Unknown\x00"
90
+ store = load_embedded_store(store_db(pack_store(pack_section([(key(1), junk)]))))
91
+ assert store.tribes == [] and store.profiles == []
92
+ assert store.errors == ["section 0: unknown content, skipped"]
93
+
94
+
95
+ def test_ase_save_has_no_store() -> None:
96
+ assert WorldSave().read_embedded_store() == EmbeddedStore()
97
+
98
+
99
+ # ---------------------------------------------------------------- content
100
+
101
+
102
+ @pytest.fixture
103
+ def scorched_sidecars() -> tuple[list[bytes], list[bytes]]:
104
+ if not SCORCHED_DIR.exists():
105
+ pytest.skip(f"Fixture not available: {SCORCHED_DIR}")
106
+ tribes = [p.read_bytes() for p in sorted(SCORCHED_DIR.glob("*.arktribe"))[:2]]
107
+ profiles = [p.read_bytes() for p in sorted(SCORCHED_DIR.glob("*.arkprofile"))[:3]]
108
+ assert len(tribes) == 2 and len(profiles) == 3
109
+ return tribes, profiles
110
+
111
+
112
+ def build_blob(tribes: list[bytes], profiles: list[bytes]) -> bytes:
113
+ """Store with one stale (zero-key) copy per section ahead of the live ones."""
114
+ zero = b"\x00" * 8
115
+ tribe_entries = [(zero, tribes[0])] + [(key(i + 1), data) for i, data in enumerate(tribes)]
116
+ profile_entries = [(zero, profiles[0])] + [(key(i + 10), data) for i, data in enumerate(profiles)]
117
+ return pack_store(pack_section(tribe_entries), pack_section(profile_entries))
118
+
119
+
120
+ def test_current_entries_load(scorched_sidecars: tuple[list[bytes], list[bytes]]) -> None:
121
+ tribes, profiles = scorched_sidecars
122
+ store = load_embedded_store(store_db(build_blob(tribes, profiles)))
123
+ assert store.errors == []
124
+ assert (len(store.tribes), len(store.profiles)) == (2, 3)
125
+ assert (store.superseded_tribes, store.superseded_profiles) == (1, 1)
126
+ assert {t.tribe_id for t in store.tribes} == {Tribe.load(b).tribe_id for b in tribes}
127
+ assert {p.unique_id for p in store.profiles} == {Profile.load(b).unique_id for b in profiles}
128
+ assert all(p.source_path is None for p in store.profiles)
129
+
130
+
131
+ def test_duplicate_live_entries_keep_the_last(scorched_sidecars: tuple[list[bytes], list[bytes]]) -> None:
132
+ tribes = scorched_sidecars[0]
133
+ section = pack_section([(key(1), tribes[0]), (key(2), tribes[0])])
134
+ store = load_embedded_store(store_db(pack_store(section)))
135
+ assert len(store.tribes) == 1 and store.errors == []
136
+
137
+
138
+ def test_bad_entry_is_skipped_rest_load(scorched_sidecars: tuple[list[bytes], list[bytes]]) -> None:
139
+ tribes = scorched_sidecars[0]
140
+ good = pack_section([(key(1), tribes[0]), (key(2), tribes[1])])
141
+ # Point the second index entry past the end of the section's data.
142
+ bad_offset = len(good) - 8
143
+ broken = good[:bad_offset] + struct.pack("<ii", 10_000_000, 64)
144
+ store = load_embedded_store(store_db(pack_store(broken)))
145
+ assert len(store.tribes) == 1
146
+ assert len(store.errors) == 1 and store.errors[0].startswith("tribe #1:")
147
+
148
+
149
+ def minimal_save(tmp_path: Path, blob: bytes) -> Path:
150
+ """An ASA save with a real SaveHeader, no objects, and the given store."""
151
+ src = next(SCORCHED_DIR.glob("*_WP.ark"))
152
+ with sqlite3.connect(f"file:{src.as_posix()}?mode=ro", uri=True) as ref:
153
+ header = ref.execute("SELECT value FROM custom WHERE key = 'SaveHeader'").fetchone()[0]
154
+ path = tmp_path / "Store_WP.ark"
155
+ conn = sqlite3.connect(path)
156
+ conn.execute("CREATE TABLE game (key BLOB primary key not null, value blob not null)")
157
+ conn.execute("CREATE TABLE custom (key VARCHAR(64) primary key not null, value blob not null)")
158
+ conn.executemany("INSERT INTO custom VALUES (?, ?)", [("SaveHeader", header), (STORE_KEY, blob)])
159
+ conn.commit()
160
+ conn.close()
161
+ return path
162
+
163
+
164
+ @pytest.mark.parametrize("lazy", [False, True])
165
+ def test_world_save_reads_store_and_exports(
166
+ tmp_path: Path, scorched_sidecars: tuple[list[bytes], list[bytes]], lazy: bool
167
+ ) -> None:
168
+ tribes, profiles = scorched_sidecars
169
+ save = WorldSave.load(minimal_save(tmp_path, build_blob(tribes, profiles)), lazy_properties=lazy)
170
+ store = save.read_embedded_store()
171
+ assert (len(store.tribes), len(store.profiles), store.errors) == (2, 3, [])
172
+ save.tribes, save.profiles = store.tribes, store.profiles
173
+ data = export_all(save)
174
+ # Tribe members without a profile also get player rows, so check inclusion.
175
+ exported_ids = {str(r["playerid"]) for r in data["ASV_Players"]}
176
+ assert {str(p.player_id) for p in store.profiles} <= exported_ids
177
+ assert {r["tribeid"] for r in data["ASV_Tribes"]} >= {t.tribe_id for t in store.tribes}
178
+ if save._lazy_conn is not None:
179
+ save._lazy_conn.close()
180
+
181
+
182
+ # ---------------------------------------------------------------- real sample
183
+
184
+
185
+ def test_nitrado_sample(tmp_path: Path) -> None:
186
+ if not NITRADO_SAMPLE.exists():
187
+ pytest.skip(f"Local-only sample not available: {NITRADO_SAMPLE}")
188
+ copy = Path(shutil.copy2(NITRADO_SAMPLE, tmp_path / NITRADO_SAMPLE.name))
189
+ save = WorldSave.load(copy, lazy_properties=True)
190
+ store = save.read_embedded_store()
191
+ assert (len(store.tribes), len(store.profiles)) == (6, 8)
192
+ assert (store.superseded_tribes, store.superseded_profiles) == (13, 14)
193
+ save.tribes, save.profiles = store.tribes, store.profiles
194
+ logs = export_all(save)["ASV_TribeLogs"]
195
+ assert sum(len(r.get("logs") or []) for r in logs) == 835
196
+ assert save._lazy_conn is not None
197
+ save._lazy_conn.close()
File without changes
File without changes
File without changes
File without changes