pyaff4 2.0.0a20__py312-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyaff4/__init__.py +5 -0
- pyaff4/_version.py +42 -0
- pyaff4/aes_xts.py +39 -0
- pyaff4/aff4.py +406 -0
- pyaff4/aff4_directory.py +213 -0
- pyaff4/aff4_file.py +175 -0
- pyaff4/aff4_image.py +806 -0
- pyaff4/aff4_map.py +615 -0
- pyaff4/aff4_metadata.py +10 -0
- pyaff4/aff4_utils.py +30 -0
- pyaff4/block_hasher.py +614 -0
- pyaff4/cli/__init__.py +12 -0
- pyaff4/cli/__main__.py +12 -0
- pyaff4/cli/commands.py +499 -0
- pyaff4/cli/ls.py +261 -0
- pyaff4/container.py +693 -0
- pyaff4/data_store.py +949 -0
- pyaff4/dedup_container.py +199 -0
- pyaff4/encrypted_stream.py +516 -0
- pyaff4/escaping.py +154 -0
- pyaff4/hashes.py +87 -0
- pyaff4/hexdump.py +16 -0
- pyaff4/keybag.py +140 -0
- pyaff4/lexicon.py +298 -0
- pyaff4/linear_hasher.py +214 -0
- pyaff4/logical.py +165 -0
- pyaff4/plugins.py +19 -0
- pyaff4/rdfvalue.py +210 -0
- pyaff4/registry.py +21 -0
- pyaff4/statx.py +122 -0
- pyaff4/stream_factory.py +167 -0
- pyaff4/streams.py +43 -0
- pyaff4/struct_parser.py +86 -0
- pyaff4/symbolic_streams.py +111 -0
- pyaff4/turtle.py +24 -0
- pyaff4/version.py +62 -0
- pyaff4/zip.py +1319 -0
- pyaff4-2.0.0a20.dist-info/METADATA +144 -0
- pyaff4-2.0.0a20.dist-info/RECORD +52 -0
- pyaff4-2.0.0a20.dist-info/WHEEL +4 -0
- pyaff4-2.0.0a20.dist-info/entry_points.txt +3 -0
- pyaff4-2.0.0a20.dist-info/licenses/LICENSE +202 -0
- pyhdt2_cython/__init__.py +104 -0
- pyhdt2_cython/_kernels.py +1697 -0
- pyhdt2_cython/build.py +477 -0
- pyhdt2_cython/cli.py +67 -0
- pyhdt2_cython/hdt.py +529 -0
- rdflib_hdt_ng/__init__.py +13 -0
- rdflib_hdt_ng/hdt_store.py +130 -0
- rdflib_hdt_ng/rdflib_bridge.py +83 -0
- rdflib_hdt_ng/terms.py +63 -0
- rdflib_hdt_ng/union_store.py +91 -0
pyaff4/__init__.py
ADDED
pyaff4/_version.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Backwards-compatibility shim for legacy versioneer-style callers.
|
|
2
|
+
|
|
3
|
+
Pre-2.0 pyaff4 shipped a versioneer-generated `_version.py` that
|
|
4
|
+
exposed `raw_versions()` / `get_versions()` returning a dict of
|
|
5
|
+
git-derived version metadata. The uv_build migration removed
|
|
6
|
+
versioneer; the version is now sourced from pyproject.toml via
|
|
7
|
+
`importlib.metadata`. This module preserves the legacy function
|
|
8
|
+
shape so downstream consumers (e.g. mac-apt's LogLibraryVersions
|
|
9
|
+
hook calling `pyaff4._version.raw_versions()['version']`) continue
|
|
10
|
+
to work without code changes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from importlib.metadata import version
|
|
16
|
+
|
|
17
|
+
__version__: str = version("pyaff4")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def raw_versions() -> dict[str, object]:
|
|
21
|
+
"""Return version metadata under legacy keys.
|
|
22
|
+
|
|
23
|
+
The original versioneer dict carried git-derived fields (full SHA,
|
|
24
|
+
dirty flag, pep440 string). Releases now ship from clean git tags
|
|
25
|
+
with pyproject.toml as the single source of truth, so only the
|
|
26
|
+
`version` key carries meaningful information; the others are
|
|
27
|
+
populated with the same value (for `pep440`-indexing callers) or
|
|
28
|
+
benign defaults.
|
|
29
|
+
"""
|
|
30
|
+
return {
|
|
31
|
+
"version": __version__,
|
|
32
|
+
"pep440": __version__,
|
|
33
|
+
"version-pep440": __version__,
|
|
34
|
+
"full-revisionid": "",
|
|
35
|
+
"dirty": False,
|
|
36
|
+
"error": None,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_versions() -> dict[str, object]:
|
|
41
|
+
"""Versioneer-compatible alias for `raw_versions`."""
|
|
42
|
+
return raw_versions()
|
pyaff4/aes_xts.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright © 2026 Apple Inc.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under Apache License v2.0
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
6
|
+
#
|
|
7
|
+
|
|
8
|
+
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _pad_tweak(tweak: bytes) -> bytes:
|
|
12
|
+
if len(tweak) > 16:
|
|
13
|
+
raise ValueError(f"XTS tweak must be at most 16 bytes, got {len(tweak)}")
|
|
14
|
+
return tweak + b"\x00" * (16 - len(tweak))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AesXts:
|
|
18
|
+
"""AES-XTS cipher matching pycryptoplus's tweak-handling.
|
|
19
|
+
|
|
20
|
+
AFF4 encrypted streams write an 8-byte little-endian chunk-address
|
|
21
|
+
tweak; AES-XTS requires 16. pycryptoplus zero-pads on the right;
|
|
22
|
+
this class does the same so on-disk ciphertext stays byte-identical.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, key1: bytes, key2: bytes) -> None:
|
|
26
|
+
self._key = key1 + key2
|
|
27
|
+
|
|
28
|
+
def encrypt(self, data: bytes, tweak: bytes) -> bytes:
|
|
29
|
+
cipher = Cipher(algorithms.AES(self._key), modes.XTS(_pad_tweak(tweak)))
|
|
30
|
+
return cipher.encryptor().update(data)
|
|
31
|
+
|
|
32
|
+
def decrypt(self, data: bytes, tweak: bytes) -> bytes:
|
|
33
|
+
cipher = Cipher(algorithms.AES(self._key), modes.XTS(_pad_tweak(tweak)))
|
|
34
|
+
return cipher.decryptor().update(data)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def new(keys: tuple[bytes, bytes]) -> AesXts:
|
|
38
|
+
key1, key2 = keys
|
|
39
|
+
return AesXts(key1, key2)
|
pyaff4/aff4.py
ADDED
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
# Copyright 2014 Google Inc. All rights reserved.
|
|
2
|
+
# Copyright © 2026 Apple Inc.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
5
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
6
|
+
# the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
13
|
+
# License for the specific language governing permissions and limitations under
|
|
14
|
+
# the License.
|
|
15
|
+
|
|
16
|
+
"""This is the python AFF4 library."""
|
|
17
|
+
import platform
|
|
18
|
+
import sys
|
|
19
|
+
import time
|
|
20
|
+
import uuid
|
|
21
|
+
import weakref
|
|
22
|
+
|
|
23
|
+
from rdflib import URIRef
|
|
24
|
+
|
|
25
|
+
from pyaff4 import rdfvalue
|
|
26
|
+
from pyaff4 import lexicon
|
|
27
|
+
|
|
28
|
+
class NoneObject:
|
|
29
|
+
""" A magical object which is like None but swallows bad
|
|
30
|
+
dereferences, __getattr__, iterators etc to return itself.
|
|
31
|
+
|
|
32
|
+
Instantiate with the reason for the error.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, reason="", *args, **_):
|
|
36
|
+
# Often None objects are instantiated on purpose so its not really that
|
|
37
|
+
# important to see their reason.
|
|
38
|
+
self.reason = reason
|
|
39
|
+
self.args = args
|
|
40
|
+
|
|
41
|
+
def __str__(self):
|
|
42
|
+
return str(self).encode('utf-8')
|
|
43
|
+
|
|
44
|
+
def __unicode__(self):
|
|
45
|
+
return self.FormatReason()
|
|
46
|
+
|
|
47
|
+
def FormatReason(self):
|
|
48
|
+
if "%" in self.reason:
|
|
49
|
+
return self.reason % self.args
|
|
50
|
+
else:
|
|
51
|
+
return self.reason.format(*self.args)
|
|
52
|
+
|
|
53
|
+
def __repr__(self):
|
|
54
|
+
return "<%s>" % self.FormatReason()
|
|
55
|
+
|
|
56
|
+
## Behave like an empty set
|
|
57
|
+
def __iter__(self):
|
|
58
|
+
return iter([])
|
|
59
|
+
|
|
60
|
+
def __len__(self):
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
def __getattr__(self, attr):
|
|
64
|
+
# By returning self for any unknown attribute and ensuring the self is
|
|
65
|
+
# callable, we cover both properties and methods Override NotImplemented
|
|
66
|
+
# functions in object with self
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
def __bool__(self):
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
# Comparisons.
|
|
73
|
+
def __eq__(self, other):
|
|
74
|
+
return other is None
|
|
75
|
+
|
|
76
|
+
def __ne__(self, other):
|
|
77
|
+
return other is not None
|
|
78
|
+
|
|
79
|
+
def __gt__(self, _):
|
|
80
|
+
return False
|
|
81
|
+
|
|
82
|
+
__lt__ = __gt__
|
|
83
|
+
__le__ = __gt__
|
|
84
|
+
__ge__ = __gt__
|
|
85
|
+
|
|
86
|
+
def __call__(self, *arg, **kwargs):
|
|
87
|
+
return self
|
|
88
|
+
|
|
89
|
+
def __int__(self):
|
|
90
|
+
return -1
|
|
91
|
+
|
|
92
|
+
__add__ = __call__
|
|
93
|
+
__sub__ = __call__
|
|
94
|
+
__mul__ = __call__
|
|
95
|
+
__floordiv__ = __call__
|
|
96
|
+
__mod__ = __call__
|
|
97
|
+
__div__ = __call__
|
|
98
|
+
__divmod__ = __call__
|
|
99
|
+
__pow__ = __call__
|
|
100
|
+
__lshift__ = __call__
|
|
101
|
+
__rshift__ = __call__
|
|
102
|
+
__and__ = __call__
|
|
103
|
+
__xor__ = __call__
|
|
104
|
+
__or__ = __call__
|
|
105
|
+
|
|
106
|
+
__radd__ = __call__
|
|
107
|
+
__rsub__ = __call__
|
|
108
|
+
__rmul__ = __call__
|
|
109
|
+
__rfloordiv__ = __call__
|
|
110
|
+
__rmod__ = __call__
|
|
111
|
+
__rdivmod__ = __call__
|
|
112
|
+
__rpow__ = __call__
|
|
113
|
+
__rlshift__ = __call__
|
|
114
|
+
__rrshift__ = __call__
|
|
115
|
+
__rand__ = __call__
|
|
116
|
+
__rxor__ = __call__
|
|
117
|
+
__ror__ = __call__
|
|
118
|
+
|
|
119
|
+
# Override these methods too.
|
|
120
|
+
dereference_as = __call__
|
|
121
|
+
__getitem__ = __call__
|
|
122
|
+
|
|
123
|
+
def __enter__(self):
|
|
124
|
+
return self
|
|
125
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
126
|
+
# Do not allow exceptions to be propagated through us.
|
|
127
|
+
return True
|
|
128
|
+
|
|
129
|
+
def Close(self):
|
|
130
|
+
pass
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# Keep track of all the AFF4 objects which are alive right now. This helps in
|
|
134
|
+
# debugging memory leaks.
|
|
135
|
+
AFF4_OBJECT_REFS = {}
|
|
136
|
+
|
|
137
|
+
def newARN():
|
|
138
|
+
arn = "aff4://%s" % uuid.uuid4()
|
|
139
|
+
return rdfvalue.ARN(arn)
|
|
140
|
+
|
|
141
|
+
class AFF4StreamProperties:
|
|
142
|
+
seekable = True
|
|
143
|
+
sizeable = True
|
|
144
|
+
writable = False
|
|
145
|
+
|
|
146
|
+
class AFF4VolumeProperties:
|
|
147
|
+
supports_compression = True
|
|
148
|
+
writable = False
|
|
149
|
+
files_are_directories = True
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class AFF4Object:
|
|
153
|
+
def __init__(self, resolver, urn, parent, *args, version=None, **kwargs):
|
|
154
|
+
self.resolver = resolver
|
|
155
|
+
self._dirty = False
|
|
156
|
+
self.version = version
|
|
157
|
+
self.closed = False
|
|
158
|
+
self.parent = parent
|
|
159
|
+
|
|
160
|
+
if urn is None:
|
|
161
|
+
urn = "aff4://%s" % uuid.uuid4()
|
|
162
|
+
urn = rdfvalue.ARN(urn)
|
|
163
|
+
elif type(urn) == URIRef:
|
|
164
|
+
urn = rdfvalue.ARN(urn.toPython())
|
|
165
|
+
|
|
166
|
+
self.urn = urn
|
|
167
|
+
AFF4_OBJECT_REFS[id(self)] = weakref.proxy(
|
|
168
|
+
self, lambda _, id=id(self), ref=AFF4_OBJECT_REFS: ref.pop(id))
|
|
169
|
+
|
|
170
|
+
def __enter__(self):
|
|
171
|
+
return self
|
|
172
|
+
|
|
173
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
174
|
+
# Return ourselves to the resolver cache.
|
|
175
|
+
self.resolver.Return(self)
|
|
176
|
+
|
|
177
|
+
# Ensure exceptions aren't silently swallowed (are propagated)
|
|
178
|
+
if exc_value is not None:
|
|
179
|
+
return False
|
|
180
|
+
|
|
181
|
+
def LoadFromURN(self):
|
|
182
|
+
raise NotImplementedError
|
|
183
|
+
|
|
184
|
+
def Prepare(self):
|
|
185
|
+
pass
|
|
186
|
+
|
|
187
|
+
def Flush(self):
|
|
188
|
+
self._dirty = False
|
|
189
|
+
|
|
190
|
+
def IsDirty(self):
|
|
191
|
+
return self._dirty
|
|
192
|
+
|
|
193
|
+
def MarkDirty(self):
|
|
194
|
+
self._dirty = True
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class AFF4Volume(AFF4Object):
|
|
198
|
+
def __init__(self, *args, **kwargs):
|
|
199
|
+
super().__init__(*args, **kwargs)
|
|
200
|
+
self.properties = AFF4VolumeProperties()
|
|
201
|
+
|
|
202
|
+
def CreateMember(self, child):
|
|
203
|
+
raise NotImplementedError
|
|
204
|
+
|
|
205
|
+
class Image(AFF4Object):
|
|
206
|
+
|
|
207
|
+
def __init__(self, *args, **kwargs):
|
|
208
|
+
super().__init__(*args, **kwargs)
|
|
209
|
+
|
|
210
|
+
def getDTB(self):
|
|
211
|
+
# UGLY: At the moment, the CR3 returned by MacPMEM doesnt seem to work, so we need to rely on scanning
|
|
212
|
+
# that being the case, we dont return the dtb if it is OSX
|
|
213
|
+
|
|
214
|
+
kaslrSlide = list(self.resolver.QuerySubjectPredicate(self.urn, lexicon.standard.OSXKALSRSlide))
|
|
215
|
+
if len(kaslrSlide) == 1 and int(kaslrSlide[0]) != 0:
|
|
216
|
+
# it is Mac OS
|
|
217
|
+
return 0
|
|
218
|
+
else:
|
|
219
|
+
try:
|
|
220
|
+
dtb = next(self.resolver.QuerySubjectPredicate(self.urn, lexicon.standard.memoryPageTableEntryOffset))
|
|
221
|
+
return int(dtb)
|
|
222
|
+
except (StopIteration, ValueError):
|
|
223
|
+
# some early images generated by Rekall don't contain a CR3
|
|
224
|
+
return 0
|
|
225
|
+
|
|
226
|
+
class LogicalImage(AFF4Object):
|
|
227
|
+
|
|
228
|
+
def __init__(self, container, resolver, volume, urn, pathName):
|
|
229
|
+
super().__init__(resolver, urn, container)
|
|
230
|
+
self.volume = volume
|
|
231
|
+
self.pathName = pathName
|
|
232
|
+
self.container = container
|
|
233
|
+
|
|
234
|
+
def name(self):
|
|
235
|
+
return self.pathName
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class MemberContainer(AFF4Object):
|
|
239
|
+
def __init__(self, *args, **kwargs):
|
|
240
|
+
super().__init__(*args, **kwargs)
|
|
241
|
+
self.properties = AFF4StreamProperties()
|
|
242
|
+
|
|
243
|
+
def OpenMember(self, segment_urn):
|
|
244
|
+
pass
|
|
245
|
+
|
|
246
|
+
def CreateMember(self, child_urn):
|
|
247
|
+
pass
|
|
248
|
+
|
|
249
|
+
def ContainsMember(self, arn):
|
|
250
|
+
pass
|
|
251
|
+
|
|
252
|
+
def RemoveMember(self, child_urn):
|
|
253
|
+
pass
|
|
254
|
+
|
|
255
|
+
def RemoveMembers(self, child_urns):
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
SEEK_SET = 0
|
|
259
|
+
SEEK_CUR = 1
|
|
260
|
+
SEEK_END = 2
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class AFF4Stream(AFF4Object):
|
|
264
|
+
readptr = 0
|
|
265
|
+
writeptr = 0
|
|
266
|
+
size = 0
|
|
267
|
+
|
|
268
|
+
def __init__(self, *args, **kwargs):
|
|
269
|
+
super().__init__(*args, **kwargs)
|
|
270
|
+
self.properties = AFF4StreamProperties()
|
|
271
|
+
|
|
272
|
+
def Read(self, length):
|
|
273
|
+
raise NotImplementedError()
|
|
274
|
+
|
|
275
|
+
def Write(self, data):
|
|
276
|
+
raise NotImplementedError()
|
|
277
|
+
|
|
278
|
+
def WriteStream(self, source):
|
|
279
|
+
"""Writes into this stream from a stream.
|
|
280
|
+
|
|
281
|
+
The stream is a file-like object with read and tell() methods.
|
|
282
|
+
"""
|
|
283
|
+
raise NotImplementedError()
|
|
284
|
+
|
|
285
|
+
def SeekWrite(self, offset, whence=0):
|
|
286
|
+
if whence == SEEK_SET:
|
|
287
|
+
self.writeptr = offset
|
|
288
|
+
elif whence == SEEK_CUR:
|
|
289
|
+
self.writeptr += offset
|
|
290
|
+
elif whence == SEEK_END:
|
|
291
|
+
self.writeptr = offset + self.Size()
|
|
292
|
+
|
|
293
|
+
if self.writeptr < 0:
|
|
294
|
+
self.writeptr = 0
|
|
295
|
+
|
|
296
|
+
if self.size < self.writeptr:
|
|
297
|
+
self.size = self.writeptr
|
|
298
|
+
|
|
299
|
+
def SeekRead(self, offset, whence=0):
|
|
300
|
+
if whence == SEEK_SET:
|
|
301
|
+
self.readptr = offset
|
|
302
|
+
elif whence == SEEK_CUR:
|
|
303
|
+
self.readptr += offset
|
|
304
|
+
elif whence == SEEK_END:
|
|
305
|
+
self.readptr = offset + self.Size()
|
|
306
|
+
|
|
307
|
+
if self.readptr < 0:
|
|
308
|
+
self.readptr = 0
|
|
309
|
+
|
|
310
|
+
def TellRead(self):
|
|
311
|
+
return self.readptr
|
|
312
|
+
|
|
313
|
+
def TellWrite(self):
|
|
314
|
+
return self.writeptr
|
|
315
|
+
|
|
316
|
+
def Size(self):
|
|
317
|
+
return self.size
|
|
318
|
+
|
|
319
|
+
def read(self, length=1024*1024):
|
|
320
|
+
return self.Read(length)
|
|
321
|
+
|
|
322
|
+
def seek(self, offset, whence=0):
|
|
323
|
+
self.SeekRead(offset, whence=whence)
|
|
324
|
+
|
|
325
|
+
def write(self, data):
|
|
326
|
+
self.Write(data)
|
|
327
|
+
|
|
328
|
+
def tell(self):
|
|
329
|
+
return self.Tell()
|
|
330
|
+
|
|
331
|
+
def flush(self):
|
|
332
|
+
self.Flush()
|
|
333
|
+
|
|
334
|
+
def Prepare(self):
|
|
335
|
+
self.SeekRead(0)
|
|
336
|
+
#self.SeekWrite(0)
|
|
337
|
+
|
|
338
|
+
class EmptyStream(AFF4Stream):
|
|
339
|
+
def Read(self, length):
|
|
340
|
+
return b""
|
|
341
|
+
|
|
342
|
+
def LoadFromURN(self):
|
|
343
|
+
pass
|
|
344
|
+
|
|
345
|
+
def Close(self):
|
|
346
|
+
pass
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class ProgressContext:
|
|
351
|
+
last_time = 0
|
|
352
|
+
last_offset = 0
|
|
353
|
+
|
|
354
|
+
# The following are set in advance by users in order to get accurate
|
|
355
|
+
# progress reports.
|
|
356
|
+
|
|
357
|
+
# Start offset of this current range.
|
|
358
|
+
start = 0
|
|
359
|
+
length = 0
|
|
360
|
+
|
|
361
|
+
def __init__(self, length=0):
|
|
362
|
+
self.length = length
|
|
363
|
+
self.last_time = self.now()
|
|
364
|
+
|
|
365
|
+
def now(self):
|
|
366
|
+
return time.time() * 1e6
|
|
367
|
+
|
|
368
|
+
def Report(self, readptr):
|
|
369
|
+
"""This will be called periodically to report the progress.
|
|
370
|
+
|
|
371
|
+
Note that readptr is specified relative to the start of the range
|
|
372
|
+
operation (WriteStream and CopyToStream)
|
|
373
|
+
"""
|
|
374
|
+
readptr = readptr + self.start
|
|
375
|
+
now = self.now()
|
|
376
|
+
if now > self.last_time + 1000000 / 4:
|
|
377
|
+
# Rate in MB/s.
|
|
378
|
+
rate = ((readptr - self.last_offset) /
|
|
379
|
+
(now - self.last_time) * 1000000 / 1024/1024)
|
|
380
|
+
|
|
381
|
+
sys.stdout.write(" Reading %sMiB / %sMiB %s MiB/s\r\n" % (
|
|
382
|
+
readptr/1024/1024,
|
|
383
|
+
self.length/1024/1024,
|
|
384
|
+
rate))
|
|
385
|
+
sys.stdout.flush()
|
|
386
|
+
|
|
387
|
+
self.last_time = now
|
|
388
|
+
self.last_offset = readptr
|
|
389
|
+
|
|
390
|
+
if aff4_abort_signaled:
|
|
391
|
+
sys.stdout.write("\n\nAborted!\n")
|
|
392
|
+
raise RuntimeError("Aborted")
|
|
393
|
+
|
|
394
|
+
aff4_abort_signaled = False
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
class EmptyProgressContext(ProgressContext):
|
|
398
|
+
def Report(self, _):
|
|
399
|
+
pass
|
|
400
|
+
|
|
401
|
+
DEFAULT_PROGRESS = ProgressContext()
|
|
402
|
+
EMPTY_PROGRESS = EmptyProgressContext()
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
WIN32 = platform.system() == "Windows"
|
|
406
|
+
MacOS = platform.system() == "Darwin"
|
pyaff4/aff4_directory.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# Copyright 2014 Google Inc. All rights reserved.
|
|
2
|
+
# Copyright © 2026 Apple Inc.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
5
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
6
|
+
# the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
13
|
+
# License for the specific language governing permissions and limitations under
|
|
14
|
+
# the License.
|
|
15
|
+
|
|
16
|
+
"""This module implements the Directory AFF4 Volume."""
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
|
|
20
|
+
from rdflib import Literal, URIRef
|
|
21
|
+
|
|
22
|
+
from pyaff4 import aff4
|
|
23
|
+
from pyaff4 import aff4_file
|
|
24
|
+
from pyaff4 import aff4_utils
|
|
25
|
+
from pyaff4 import lexicon
|
|
26
|
+
from pyaff4 import rdfvalue
|
|
27
|
+
from pyaff4 import registry
|
|
28
|
+
from pyaff4 import escaping
|
|
29
|
+
|
|
30
|
+
LOGGER = logging.getLogger("pyaff4")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AFF4Directory(aff4.AFF4Volume):
|
|
34
|
+
|
|
35
|
+
root_path = ""
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def NewAFF4Directory(cls, resolver, version, root_urn, appendmode=None):
|
|
39
|
+
result = AFF4Directory(resolver, None, None)
|
|
40
|
+
result.version = version
|
|
41
|
+
result.root_path = root_urn.ToFilename()
|
|
42
|
+
result.mode = appendmode
|
|
43
|
+
|
|
44
|
+
if appendmode == "truncate":
|
|
45
|
+
aff4_utils.RemoveDirectory(result.root_path)
|
|
46
|
+
|
|
47
|
+
if not (os.path.isdir(result.root_path) or
|
|
48
|
+
os.path.isfile(result.root_path)):
|
|
49
|
+
if appendmode in ("truncate", "append"):
|
|
50
|
+
aff4_utils.MkDir(result.root_path)
|
|
51
|
+
elif appendmode is None:
|
|
52
|
+
# Read-only open of an existing directory volume; the path
|
|
53
|
+
# must already exist.
|
|
54
|
+
raise RuntimeError("AFF4Directory storage does not exist: %s"
|
|
55
|
+
% result.root_path)
|
|
56
|
+
else:
|
|
57
|
+
raise RuntimeError("Unknown mode: %s" % appendmode)
|
|
58
|
+
|
|
59
|
+
resolver.Set(lexicon.transient_graph, result.urn, lexicon.AFF4_TYPE,
|
|
60
|
+
URIRef(lexicon.AFF4_DIRECTORY_TYPE))
|
|
61
|
+
|
|
62
|
+
resolver.Set(lexicon.transient_graph, result.urn, lexicon.AFF4_STORED,
|
|
63
|
+
URIRef(root_urn))
|
|
64
|
+
|
|
65
|
+
result.LoadFromURN()
|
|
66
|
+
result.properties.writable = appendmode in ("truncate", "append")
|
|
67
|
+
|
|
68
|
+
return resolver.CachePut(result)
|
|
69
|
+
|
|
70
|
+
def __init__(self, *args, **kwargs):
|
|
71
|
+
super().__init__(*args, **kwargs)
|
|
72
|
+
self.children = set()
|
|
73
|
+
self.mode = None
|
|
74
|
+
|
|
75
|
+
def Close(self):
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
def ContainsSegment(self, child_arn):
|
|
79
|
+
localpath = escaping.member_name_for_file_iri(str(child_arn))
|
|
80
|
+
if os.path.exists(localpath):
|
|
81
|
+
return True
|
|
82
|
+
return False
|
|
83
|
+
|
|
84
|
+
def CreateMember(self, child_urn):
|
|
85
|
+
# Check that child is a relative path in our URN.
|
|
86
|
+
relative_path = self.urn.RelativePath(child_urn)
|
|
87
|
+
if relative_path is None or relative_path == str(child_urn):
|
|
88
|
+
raise IOError("Child URN is not within container URN.")
|
|
89
|
+
|
|
90
|
+
# Use this filename. Note that since filesystems can not typically
|
|
91
|
+
# represent files and directories as the same path component we can not
|
|
92
|
+
# allow slashes in the filename. Otherwise we will fail to create
|
|
93
|
+
# e.g. stream/0000000 and stream/0000000/index.
|
|
94
|
+
filename = escaping.member_name_for_urn(
|
|
95
|
+
child_urn, self.version, self.urn)
|
|
96
|
+
absolute_path = self.root_path + os.sep + filename
|
|
97
|
+
|
|
98
|
+
# Persist the member's type so it survives turtle round-trip;
|
|
99
|
+
# AFF4_FILE_NAME is just a session-only filesystem binding.
|
|
100
|
+
self.resolver.Set(self.urn, child_urn, lexicon.AFF4_TYPE,
|
|
101
|
+
URIRef(lexicon.AFF4_FILE_TYPE))
|
|
102
|
+
self.resolver.Set(self.urn, child_urn, URIRef(lexicon.AFF4_DIRECTORY_CHILD_FILENAME),
|
|
103
|
+
Literal(filename))
|
|
104
|
+
self.resolver.Set(lexicon.transient_graph,
|
|
105
|
+
child_urn, lexicon.AFF4_FILE_NAME,
|
|
106
|
+
Literal(absolute_path))
|
|
107
|
+
|
|
108
|
+
# Construct the FileBackedObject directly with appendmode="truncate"
|
|
109
|
+
# rather than going through AFF4FactoryOpen — the directory volume
|
|
110
|
+
# always wants to write fresh files, and AFF4FactoryOpen has no way
|
|
111
|
+
# to thread an appendmode kwarg through to the handler.
|
|
112
|
+
result = aff4_file.FileBackedObject(
|
|
113
|
+
self.resolver, child_urn, self, appendmode="truncate")
|
|
114
|
+
result.LoadFromURN()
|
|
115
|
+
self.resolver.ObjectCache.Put(result, True)
|
|
116
|
+
result.Prepare()
|
|
117
|
+
|
|
118
|
+
self.MarkDirty()
|
|
119
|
+
self.children.add(child_urn)
|
|
120
|
+
|
|
121
|
+
return result
|
|
122
|
+
|
|
123
|
+
def LoadFromURN(self):
|
|
124
|
+
self.storage = self.resolver.GetUnique(lexicon.transient_graph, self.urn, lexicon.AFF4_STORED)
|
|
125
|
+
if not self.storage:
|
|
126
|
+
LOGGER.error("Unable to find storage for AFF4Directory %s",
|
|
127
|
+
self.urn)
|
|
128
|
+
raise IOError("NOT_FOUND")
|
|
129
|
+
|
|
130
|
+
# The actual filename for the root directory.
|
|
131
|
+
if not isinstance(self.storage, rdfvalue.FileRef):
|
|
132
|
+
self.storage = rdfvalue.FileRef(str(self.storage))
|
|
133
|
+
self.root_path = self.storage.ToFilename()
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
# Read container.description directly from the filesystem; it
|
|
137
|
+
# holds the canonical aff4:// URN of the container which we
|
|
138
|
+
# adopt in place of our placeholder UUID.
|
|
139
|
+
container_desc_path = os.path.join(
|
|
140
|
+
self.root_path, lexicon.AFF4_CONTAINER_DESCRIPTION)
|
|
141
|
+
if os.path.exists(container_desc_path):
|
|
142
|
+
with open(container_desc_path, "rb") as f:
|
|
143
|
+
urn_string = f.read(1000).decode("utf-8").strip()
|
|
144
|
+
|
|
145
|
+
if urn_string and str(self.urn) != urn_string:
|
|
146
|
+
self.resolver.DeleteSubject(self.urn)
|
|
147
|
+
self.urn = rdfvalue.ARN(urn_string)
|
|
148
|
+
|
|
149
|
+
self.resolver.Set(lexicon.transient_graph, self.urn, lexicon.AFF4_TYPE,
|
|
150
|
+
URIRef(lexicon.AFF4_DIRECTORY_TYPE))
|
|
151
|
+
self.resolver.Set(lexicon.transient_graph, self.urn, lexicon.AFF4_STORED,
|
|
152
|
+
URIRef(self.storage))
|
|
153
|
+
LOGGER.info("AFF4Directory volume found: %s", self.urn)
|
|
154
|
+
|
|
155
|
+
# Try to load the RDF metadata file from the storage.
|
|
156
|
+
turtle_path = os.path.join(
|
|
157
|
+
self.root_path, lexicon.AFF4_CONTAINER_INFO_TURTLE)
|
|
158
|
+
if os.path.exists(turtle_path):
|
|
159
|
+
with open(turtle_path, "rb") as turtle_stream:
|
|
160
|
+
turtle_text = turtle_stream.read().decode("utf-8")
|
|
161
|
+
self.resolver.store.parse(data=turtle_text, format="turtle")
|
|
162
|
+
|
|
163
|
+
# Find all the contained objects and adjust their filenames
|
|
164
|
+
# so they map to the actual filesystem paths under the
|
|
165
|
+
# current root_path.
|
|
166
|
+
for subject in self.resolver.SelectSubjectsByPrefix(
|
|
167
|
+
lexicon.any, str(self.urn)):
|
|
168
|
+
child_filename = next(self.resolver.Get(
|
|
169
|
+
lexicon.any, subject, URIRef(lexicon.AFF4_DIRECTORY_CHILD_FILENAME)),
|
|
170
|
+
None)
|
|
171
|
+
if child_filename:
|
|
172
|
+
self.resolver.Set(lexicon.transient_graph,
|
|
173
|
+
subject, lexicon.AFF4_FILE_NAME,
|
|
174
|
+
Literal("%s%s%s" % (
|
|
175
|
+
self.root_path, os.sep, child_filename)))
|
|
176
|
+
|
|
177
|
+
except IOError:
|
|
178
|
+
pass
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def Flush(self):
|
|
182
|
+
if self.IsDirty():
|
|
183
|
+
# Flush all children before us. This ensures that metadata is fully
|
|
184
|
+
# generated for each child.
|
|
185
|
+
for child_urn in list(self.children):
|
|
186
|
+
obj = self.resolver.CacheGet(child_urn)
|
|
187
|
+
if obj:
|
|
188
|
+
obj.Flush()
|
|
189
|
+
|
|
190
|
+
# Mark the container with its URN
|
|
191
|
+
with self.CreateMember(
|
|
192
|
+
self.urn.Append(
|
|
193
|
+
lexicon.AFF4_CONTAINER_DESCRIPTION)) as desc:
|
|
194
|
+
desc.Truncate()
|
|
195
|
+
desc.Write(str(self.urn).encode("utf-8"))
|
|
196
|
+
desc.Flush() # Flush explicitly since we already flushed above.
|
|
197
|
+
|
|
198
|
+
# Dump the resolver into the directory.
|
|
199
|
+
with self.CreateMember(
|
|
200
|
+
self.urn.Append(
|
|
201
|
+
lexicon.AFF4_CONTAINER_INFO_TURTLE)) as turtle_stream:
|
|
202
|
+
# Overwrite the old turtle file with the newer data.
|
|
203
|
+
turtle_stream.Truncate()
|
|
204
|
+
turtle_text = self.resolver._DumpToTurtle(self.urn)
|
|
205
|
+
if isinstance(turtle_text, str):
|
|
206
|
+
turtle_text = turtle_text.encode("utf-8")
|
|
207
|
+
turtle_stream.Write(turtle_text)
|
|
208
|
+
turtle_stream.Flush()
|
|
209
|
+
|
|
210
|
+
return super().Flush()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
registry.AFF4_TYPE_MAP[lexicon.AFF4_DIRECTORY_TYPE] = AFF4Directory
|