dfindexeddb 20240324__tar.gz → 20240331a0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dfindexeddb-20240324/dfindexeddb.egg-info → dfindexeddb-20240331a0}/PKG-INFO +36 -6
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/README.md +35 -5
- dfindexeddb-20240331a0/dfindexeddb/indexeddb/chromium/__init__.py +13 -0
- {dfindexeddb-20240324/dfindexeddb/indexeddb → dfindexeddb-20240331a0/dfindexeddb/indexeddb/chromium}/blink.py +2 -2
- dfindexeddb-20240324/dfindexeddb/indexeddb/chromium.py → dfindexeddb-20240331a0/dfindexeddb/indexeddb/chromium/record.py +2 -2
- {dfindexeddb-20240324/dfindexeddb/indexeddb → dfindexeddb-20240331a0/dfindexeddb/indexeddb/chromium}/v8.py +1 -1
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/indexeddb/cli.py +19 -8
- dfindexeddb-20240331a0/dfindexeddb/indexeddb/firefox/__init__.py +13 -0
- dfindexeddb-20240331a0/dfindexeddb/indexeddb/safari/__init__.py +13 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/cli.py +66 -23
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/descriptor.py +51 -3
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/version.py +1 -1
- {dfindexeddb-20240324 → dfindexeddb-20240331a0/dfindexeddb.egg-info}/PKG-INFO +36 -6
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb.egg-info/SOURCES.txt +7 -4
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/pyproject.toml +9 -2
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/AUTHORS +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/LICENSE +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/__init__.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/errors.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/indexeddb/__init__.py +0 -0
- {dfindexeddb-20240324/dfindexeddb/indexeddb → dfindexeddb-20240331a0/dfindexeddb/indexeddb/chromium}/definitions.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/indexeddb/utils.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/__init__.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/definitions.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/ldb.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/log.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/record.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/leveldb/utils.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb/utils.py +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb.egg-info/dependency_links.txt +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb.egg-info/entry_points.txt +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb.egg-info/requires.txt +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/dfindexeddb.egg-info/top_level.txt +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/setup.cfg +0 -0
- {dfindexeddb-20240324 → dfindexeddb-20240331a0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dfindexeddb
|
|
3
|
-
Version:
|
|
3
|
+
Version: 20240331a0
|
|
4
4
|
Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
|
|
5
5
|
Author-email: Syd Pleno <sydp@google.com>
|
|
6
6
|
Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
|
|
@@ -283,7 +283,7 @@ installation:
|
|
|
283
283
|
|
|
284
284
|
```
|
|
285
285
|
$ dfindexeddb -h
|
|
286
|
-
usage: dfindexeddb [-h] -s SOURCE [
|
|
286
|
+
usage: dfindexeddb [-h] -s SOURCE [-o {json,jsonl,repr}]
|
|
287
287
|
|
|
288
288
|
A cli tool for parsing indexeddb files
|
|
289
289
|
|
|
@@ -291,7 +291,8 @@ options:
|
|
|
291
291
|
-h, --help show this help message and exit
|
|
292
292
|
-s SOURCE, --source SOURCE
|
|
293
293
|
The source leveldb folder
|
|
294
|
-
|
|
294
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
295
|
+
Output format. Default is json
|
|
295
296
|
```
|
|
296
297
|
|
|
297
298
|
### LevelDB
|
|
@@ -316,17 +317,46 @@ options:
|
|
|
316
317
|
To parse records from a LevelDB log (.log) file, use the following command:
|
|
317
318
|
|
|
318
319
|
```
|
|
319
|
-
$ dfleveldb log
|
|
320
|
+
$ dfleveldb log -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,write_batches,parsed_internal_key}]
|
|
321
|
+
|
|
322
|
+
options:
|
|
323
|
+
-h, --help show this help message and exit
|
|
324
|
+
-s SOURCE, --source SOURCE
|
|
325
|
+
The source leveldb file
|
|
326
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
327
|
+
Output format. Default is json
|
|
328
|
+
-t {blocks,physical_records,write_batches,parsed_internal_key}, --structure_type {blocks,physical_records,write_batches,parsed_internal_key}
|
|
329
|
+
Parses the specified structure. Default is parsed_internal_key.
|
|
320
330
|
```
|
|
321
331
|
|
|
322
332
|
To parse records from a LevelDB table (.ldb) file, use the following command:
|
|
323
333
|
|
|
324
334
|
```
|
|
325
|
-
$ dfleveldb ldb -s
|
|
335
|
+
$ dfleveldb ldb -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,records}]
|
|
336
|
+
|
|
337
|
+
options:
|
|
338
|
+
-h, --help show this help message and exit
|
|
339
|
+
-s SOURCE, --source SOURCE
|
|
340
|
+
The source leveldb file
|
|
341
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
342
|
+
Output format. Default is json
|
|
343
|
+
-t {blocks,records}, --structure_type {blocks,records}
|
|
344
|
+
Parses the specified structure. Default is records.
|
|
326
345
|
```
|
|
327
346
|
|
|
328
347
|
To parse version edit records from a Descriptor (MANIFEST) file:
|
|
329
348
|
|
|
330
349
|
```
|
|
331
|
-
$ dfleveldb descriptor -s
|
|
350
|
+
$ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,versionedit} | -v]
|
|
351
|
+
|
|
352
|
+
options:
|
|
353
|
+
-h, --help show this help message and exit
|
|
354
|
+
-s SOURCE, --source SOURCE
|
|
355
|
+
The source leveldb file
|
|
356
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
357
|
+
Output format. Default is json
|
|
358
|
+
-t {blocks,physical_records,versionedit}, --structure_type {blocks,physical_records,versionedit}
|
|
359
|
+
Parses the specified structure. Default is versionedit.
|
|
360
|
+
-v, --version_history
|
|
361
|
+
Parses the leveldb version history.
|
|
332
362
|
```
|
|
@@ -61,7 +61,7 @@ installation:
|
|
|
61
61
|
|
|
62
62
|
```
|
|
63
63
|
$ dfindexeddb -h
|
|
64
|
-
usage: dfindexeddb [-h] -s SOURCE [
|
|
64
|
+
usage: dfindexeddb [-h] -s SOURCE [-o {json,jsonl,repr}]
|
|
65
65
|
|
|
66
66
|
A cli tool for parsing indexeddb files
|
|
67
67
|
|
|
@@ -69,7 +69,8 @@ options:
|
|
|
69
69
|
-h, --help show this help message and exit
|
|
70
70
|
-s SOURCE, --source SOURCE
|
|
71
71
|
The source leveldb folder
|
|
72
|
-
|
|
72
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
73
|
+
Output format. Default is json
|
|
73
74
|
```
|
|
74
75
|
|
|
75
76
|
### LevelDB
|
|
@@ -94,17 +95,46 @@ options:
|
|
|
94
95
|
To parse records from a LevelDB log (.log) file, use the following command:
|
|
95
96
|
|
|
96
97
|
```
|
|
97
|
-
$ dfleveldb log
|
|
98
|
+
$ dfleveldb log -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,write_batches,parsed_internal_key}]
|
|
99
|
+
|
|
100
|
+
options:
|
|
101
|
+
-h, --help show this help message and exit
|
|
102
|
+
-s SOURCE, --source SOURCE
|
|
103
|
+
The source leveldb file
|
|
104
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
105
|
+
Output format. Default is json
|
|
106
|
+
-t {blocks,physical_records,write_batches,parsed_internal_key}, --structure_type {blocks,physical_records,write_batches,parsed_internal_key}
|
|
107
|
+
Parses the specified structure. Default is parsed_internal_key.
|
|
98
108
|
```
|
|
99
109
|
|
|
100
110
|
To parse records from a LevelDB table (.ldb) file, use the following command:
|
|
101
111
|
|
|
102
112
|
```
|
|
103
|
-
$ dfleveldb ldb -s
|
|
113
|
+
$ dfleveldb ldb -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,records}]
|
|
114
|
+
|
|
115
|
+
options:
|
|
116
|
+
-h, --help show this help message and exit
|
|
117
|
+
-s SOURCE, --source SOURCE
|
|
118
|
+
The source leveldb file
|
|
119
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
120
|
+
Output format. Default is json
|
|
121
|
+
-t {blocks,records}, --structure_type {blocks,records}
|
|
122
|
+
Parses the specified structure. Default is records.
|
|
104
123
|
```
|
|
105
124
|
|
|
106
125
|
To parse version edit records from a Descriptor (MANIFEST) file:
|
|
107
126
|
|
|
108
127
|
```
|
|
109
|
-
$ dfleveldb descriptor -s
|
|
128
|
+
$ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,versionedit} | -v]
|
|
129
|
+
|
|
130
|
+
options:
|
|
131
|
+
-h, --help show this help message and exit
|
|
132
|
+
-s SOURCE, --source SOURCE
|
|
133
|
+
The source leveldb file
|
|
134
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
135
|
+
Output format. Default is json
|
|
136
|
+
-t {blocks,physical_records,versionedit}, --structure_type {blocks,physical_records,versionedit}
|
|
137
|
+
Parses the specified structure. Default is versionedit.
|
|
138
|
+
-v, --version_history
|
|
139
|
+
Parses the leveldb version history.
|
|
110
140
|
```
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
@@ -17,8 +17,8 @@ import io
|
|
|
17
17
|
from typing import Any
|
|
18
18
|
|
|
19
19
|
from dfindexeddb import utils
|
|
20
|
-
from dfindexeddb.indexeddb import definitions
|
|
21
|
-
from dfindexeddb.indexeddb import v8
|
|
20
|
+
from dfindexeddb.indexeddb.chromium import definitions
|
|
21
|
+
from dfindexeddb.indexeddb.chromium import v8
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
|
|
@@ -20,8 +20,8 @@ import io
|
|
|
20
20
|
from typing import Any, BinaryIO, Optional, Tuple, Type, TypeVar, Union
|
|
21
21
|
|
|
22
22
|
from dfindexeddb import errors
|
|
23
|
-
from dfindexeddb.indexeddb import blink
|
|
24
|
-
from dfindexeddb.indexeddb import definitions
|
|
23
|
+
from dfindexeddb.indexeddb.chromium import blink
|
|
24
|
+
from dfindexeddb.indexeddb.chromium import definitions
|
|
25
25
|
from dfindexeddb.leveldb import ldb
|
|
26
26
|
from dfindexeddb.leveldb import log
|
|
27
27
|
from dfindexeddb.leveldb import utils
|
|
@@ -21,7 +21,7 @@ from typing import Any, BinaryIO, Dict, Optional, Set, Tuple, Union
|
|
|
21
21
|
|
|
22
22
|
from dfindexeddb import errors
|
|
23
23
|
from dfindexeddb import utils
|
|
24
|
-
from dfindexeddb.indexeddb import definitions
|
|
24
|
+
from dfindexeddb.indexeddb.chromium import definitions
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
@dataclass
|
|
@@ -24,8 +24,8 @@ import traceback
|
|
|
24
24
|
from dfindexeddb import errors
|
|
25
25
|
from dfindexeddb import version
|
|
26
26
|
from dfindexeddb.leveldb import record as leveldb_record
|
|
27
|
-
from dfindexeddb.indexeddb import
|
|
28
|
-
from dfindexeddb.indexeddb import v8
|
|
27
|
+
from dfindexeddb.indexeddb.chromium import record as chromium_record
|
|
28
|
+
from dfindexeddb.indexeddb.chromium import v8
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
_VALID_PRINTABLE_CHARACTERS = (
|
|
@@ -60,11 +60,13 @@ class Encoder(json.JSONEncoder):
|
|
|
60
60
|
return json.JSONEncoder.default(self, o)
|
|
61
61
|
|
|
62
62
|
|
|
63
|
-
def _Output(structure,
|
|
63
|
+
def _Output(structure, output):
|
|
64
64
|
"""Helper method to output parsed structure to stdout."""
|
|
65
|
-
if
|
|
65
|
+
if output == 'json':
|
|
66
66
|
print(json.dumps(structure, indent=2, cls=Encoder))
|
|
67
|
-
|
|
67
|
+
elif output == 'jsonl':
|
|
68
|
+
print(json.dumps(structure, cls=Encoder))
|
|
69
|
+
elif output == 'repr':
|
|
68
70
|
print(structure)
|
|
69
71
|
|
|
70
72
|
|
|
@@ -73,7 +75,8 @@ def IndexeddbCommand(args):
|
|
|
73
75
|
for db_record in leveldb_record.LevelDBRecord.FromDir(args.source):
|
|
74
76
|
record = db_record.record
|
|
75
77
|
try:
|
|
76
|
-
db_record.record =
|
|
78
|
+
db_record.record = chromium_record.IndexedDBRecord.FromLevelDBRecord(
|
|
79
|
+
record)
|
|
77
80
|
except(
|
|
78
81
|
errors.ParserError,
|
|
79
82
|
errors.DecoderError,
|
|
@@ -82,7 +85,7 @@ def IndexeddbCommand(args):
|
|
|
82
85
|
(f'Error parsing blink value: {err} for {record.__class__.__name__} '
|
|
83
86
|
f'at offset {record.offset} in {db_record.path}'), file=sys.stderr)
|
|
84
87
|
print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
|
|
85
|
-
_Output(db_record,
|
|
88
|
+
_Output(db_record, output=args.output)
|
|
86
89
|
|
|
87
90
|
|
|
88
91
|
def App():
|
|
@@ -94,7 +97,15 @@ def App():
|
|
|
94
97
|
parser.add_argument(
|
|
95
98
|
'-s', '--source', required=True, type=pathlib.Path,
|
|
96
99
|
help='The source leveldb folder')
|
|
97
|
-
parser.add_argument(
|
|
100
|
+
parser.add_argument(
|
|
101
|
+
'-o',
|
|
102
|
+
'--output',
|
|
103
|
+
choices=[
|
|
104
|
+
'json',
|
|
105
|
+
'jsonl',
|
|
106
|
+
'repr'],
|
|
107
|
+
default='json',
|
|
108
|
+
help='Output format. Default is json')
|
|
98
109
|
parser.set_defaults(func=IndexeddbCommand)
|
|
99
110
|
|
|
100
111
|
args = parser.parse_args()
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
@@ -54,18 +54,20 @@ class Encoder(json.JSONEncoder):
|
|
|
54
54
|
return json.JSONEncoder.default(self, o)
|
|
55
55
|
|
|
56
56
|
|
|
57
|
-
def _Output(structure,
|
|
57
|
+
def _Output(structure, output):
|
|
58
58
|
"""Helper method to output parsed structure to stdout."""
|
|
59
|
-
if
|
|
59
|
+
if output == 'json':
|
|
60
60
|
print(json.dumps(structure, indent=2, cls=Encoder))
|
|
61
|
-
|
|
61
|
+
elif output == 'jsonl':
|
|
62
|
+
print(json.dumps(structure, cls=Encoder))
|
|
63
|
+
elif output == 'repr':
|
|
62
64
|
print(structure)
|
|
63
65
|
|
|
64
66
|
|
|
65
67
|
def DbCommand(args):
|
|
66
68
|
"""The CLI for processing leveldb folders."""
|
|
67
69
|
for rec in record.LevelDBRecord.FromDir(args.source):
|
|
68
|
-
_Output(rec,
|
|
70
|
+
_Output(rec, output=args.output)
|
|
69
71
|
|
|
70
72
|
|
|
71
73
|
def LdbCommand(args):
|
|
@@ -75,12 +77,12 @@ def LdbCommand(args):
|
|
|
75
77
|
if args.structure_type == 'blocks':
|
|
76
78
|
# Prints block information.
|
|
77
79
|
for block in ldb_file.GetBlocks():
|
|
78
|
-
_Output(block,
|
|
80
|
+
_Output(block, output=args.output)
|
|
79
81
|
|
|
80
82
|
elif args.structure_type == 'records' or not args.structure_type:
|
|
81
83
|
# Prints key value record information.
|
|
82
84
|
for key_value_record in ldb_file.GetKeyValueRecords():
|
|
83
|
-
_Output(key_value_record,
|
|
85
|
+
_Output(key_value_record, output=args.output)
|
|
84
86
|
|
|
85
87
|
else:
|
|
86
88
|
print(f'{args.structure_type} is not supported for ldb files.')
|
|
@@ -93,23 +95,23 @@ def LogCommand(args):
|
|
|
93
95
|
if args.structure_type == 'blocks':
|
|
94
96
|
# Prints block information.
|
|
95
97
|
for block in log_file.GetBlocks():
|
|
96
|
-
_Output(block,
|
|
98
|
+
_Output(block, output=args.output)
|
|
97
99
|
|
|
98
100
|
elif args.structure_type == 'physical_records':
|
|
99
101
|
# Prints log file physical record information.
|
|
100
102
|
for log_file_record in log_file.GetPhysicalRecords():
|
|
101
|
-
_Output(log_file_record,
|
|
103
|
+
_Output(log_file_record, output=args.output)
|
|
102
104
|
|
|
103
105
|
elif args.structure_type == 'write_batches':
|
|
104
106
|
# Prints log file batch information.
|
|
105
107
|
for batch in log_file.GetWriteBatches():
|
|
106
|
-
_Output(batch,
|
|
108
|
+
_Output(batch, output=args.output)
|
|
107
109
|
|
|
108
110
|
elif (args.structure_type in ('parsed_internal_key', 'records')
|
|
109
111
|
or not args.structure_type):
|
|
110
112
|
# Prints key value record information.
|
|
111
113
|
for internal_key_record in log_file.GetParsedInternalKeys():
|
|
112
|
-
_Output(internal_key_record,
|
|
114
|
+
_Output(internal_key_record, output=args.output)
|
|
113
115
|
|
|
114
116
|
else:
|
|
115
117
|
print(f'{args.structure_type} is not supported for log files.')
|
|
@@ -119,20 +121,24 @@ def DescriptorCommand(args):
|
|
|
119
121
|
"""The CLI for processing descriptor (MANIFEST) files."""
|
|
120
122
|
manifest_file = descriptor.FileReader(args.source)
|
|
121
123
|
|
|
122
|
-
if args.
|
|
124
|
+
if args.version_history:
|
|
125
|
+
for levels in manifest_file.GetVersions():
|
|
126
|
+
_Output(levels, output=args.output)
|
|
127
|
+
|
|
128
|
+
elif args.structure_type == 'blocks':
|
|
123
129
|
# Prints block information.
|
|
124
130
|
for block in manifest_file.GetBlocks():
|
|
125
|
-
_Output(block,
|
|
131
|
+
_Output(block, output=args.output)
|
|
126
132
|
|
|
127
133
|
elif args.structure_type == 'physical_records':
|
|
128
134
|
# Prints log file physical record information.
|
|
129
135
|
for log_file_record in manifest_file.GetPhysicalRecords():
|
|
130
|
-
_Output(log_file_record,
|
|
136
|
+
_Output(log_file_record, output=args.output)
|
|
131
137
|
|
|
132
138
|
elif (args.structure_type == 'versionedit'
|
|
133
139
|
or not args.structure_type):
|
|
134
140
|
for version_edit in manifest_file.GetVersionEdits():
|
|
135
|
-
_Output(version_edit,
|
|
141
|
+
_Output(version_edit, output=args.output)
|
|
136
142
|
|
|
137
143
|
else:
|
|
138
144
|
print(f'{args.structure_type} is not supported for descriptor files.')
|
|
@@ -154,8 +160,14 @@ def App():
|
|
|
154
160
|
type=pathlib.Path,
|
|
155
161
|
help='The source leveldb directory')
|
|
156
162
|
parser_db.add_argument(
|
|
157
|
-
'
|
|
158
|
-
|
|
163
|
+
'-o',
|
|
164
|
+
'--output',
|
|
165
|
+
choices=[
|
|
166
|
+
'json',
|
|
167
|
+
'jsonl',
|
|
168
|
+
'repr'],
|
|
169
|
+
default='json',
|
|
170
|
+
help='Output format. Default is json')
|
|
159
171
|
|
|
160
172
|
parser_log = subparsers.add_parser(
|
|
161
173
|
'log', help='Parse a leveldb log file.')
|
|
@@ -165,7 +177,14 @@ def App():
|
|
|
165
177
|
type=pathlib.Path,
|
|
166
178
|
help='The source leveldb file')
|
|
167
179
|
parser_log.add_argument(
|
|
168
|
-
'
|
|
180
|
+
'-o',
|
|
181
|
+
'--output',
|
|
182
|
+
choices=[
|
|
183
|
+
'json',
|
|
184
|
+
'jsonl',
|
|
185
|
+
'repr'],
|
|
186
|
+
default='json',
|
|
187
|
+
help='Output format. Default is json')
|
|
169
188
|
parser_log.add_argument(
|
|
170
189
|
'-t',
|
|
171
190
|
'--structure_type',
|
|
@@ -173,7 +192,8 @@ def App():
|
|
|
173
192
|
'blocks',
|
|
174
193
|
'physical_records',
|
|
175
194
|
'write_batches',
|
|
176
|
-
'parsed_internal_key']
|
|
195
|
+
'parsed_internal_key'],
|
|
196
|
+
help='Parses the specified structure. Default is parsed_internal_key.')
|
|
177
197
|
parser_log.set_defaults(func=LogCommand)
|
|
178
198
|
|
|
179
199
|
parser_ldb = subparsers.add_parser(
|
|
@@ -184,13 +204,21 @@ def App():
|
|
|
184
204
|
type=pathlib.Path,
|
|
185
205
|
help='The source leveldb file')
|
|
186
206
|
parser_ldb.add_argument(
|
|
187
|
-
'
|
|
207
|
+
'-o',
|
|
208
|
+
'--output',
|
|
209
|
+
choices=[
|
|
210
|
+
'json',
|
|
211
|
+
'jsonl',
|
|
212
|
+
'repr'],
|
|
213
|
+
default='json',
|
|
214
|
+
help='Output format. Default is json')
|
|
188
215
|
parser_ldb.add_argument(
|
|
189
216
|
'-t',
|
|
190
217
|
'--structure_type',
|
|
191
218
|
choices=[
|
|
192
219
|
'blocks',
|
|
193
|
-
'records']
|
|
220
|
+
'records'],
|
|
221
|
+
help='Parses the specified structure. Default is records.')
|
|
194
222
|
parser_ldb.set_defaults(func=LdbCommand)
|
|
195
223
|
|
|
196
224
|
parser_descriptor = subparsers.add_parser(
|
|
@@ -201,12 +229,27 @@ def App():
|
|
|
201
229
|
type=pathlib.Path,
|
|
202
230
|
help='The source leveldb file')
|
|
203
231
|
parser_descriptor.add_argument(
|
|
204
|
-
'
|
|
205
|
-
|
|
232
|
+
'-o',
|
|
233
|
+
'--output',
|
|
234
|
+
choices=[
|
|
235
|
+
'json',
|
|
236
|
+
'jsonl',
|
|
237
|
+
'repr'],
|
|
238
|
+
default='json',
|
|
239
|
+
help='Output format. Default is json')
|
|
240
|
+
db_group = parser_descriptor.add_mutually_exclusive_group()
|
|
241
|
+
db_group.add_argument(
|
|
206
242
|
'-t',
|
|
207
243
|
'--structure_type',
|
|
208
244
|
choices=[
|
|
209
|
-
'blocks', 'physical_records', 'versionedit']
|
|
245
|
+
'blocks', 'physical_records', 'versionedit'],
|
|
246
|
+
help='Parses the specified structure. Default is versionedit.')
|
|
247
|
+
db_group.add_argument(
|
|
248
|
+
'-v',
|
|
249
|
+
'--version_history',
|
|
250
|
+
action='store_true',
|
|
251
|
+
help='Parses the leveldb version history.'
|
|
252
|
+
)
|
|
210
253
|
parser_descriptor.set_defaults(func=DescriptorCommand)
|
|
211
254
|
|
|
212
255
|
args = parser.parse_args()
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
"""Parser for LevelDB Descriptor (MANIFEST) files."""
|
|
16
16
|
from __future__ import annotations
|
|
17
|
-
|
|
17
|
+
from collections import defaultdict
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
19
|
from typing import Generator, Optional
|
|
20
20
|
|
|
@@ -35,7 +35,7 @@ class InternalKey:
|
|
|
35
35
|
key_type: the key type.
|
|
36
36
|
"""
|
|
37
37
|
offset: int
|
|
38
|
-
user_key: bytes
|
|
38
|
+
user_key: bytes
|
|
39
39
|
sequence_number: int
|
|
40
40
|
key_type: int
|
|
41
41
|
|
|
@@ -127,7 +127,7 @@ class CompactPointer(utils.FromDecoderMixin):
|
|
|
127
127
|
"""
|
|
128
128
|
offset: int
|
|
129
129
|
level: int
|
|
130
|
-
key: bytes
|
|
130
|
+
key: bytes
|
|
131
131
|
|
|
132
132
|
@classmethod
|
|
133
133
|
def FromDecoder(
|
|
@@ -258,6 +258,27 @@ class VersionEdit(utils.FromDecoderMixin):
|
|
|
258
258
|
return version_edit
|
|
259
259
|
|
|
260
260
|
|
|
261
|
+
@dataclass
|
|
262
|
+
class LevelDBVersion:
|
|
263
|
+
"""A LevelDBVersion.
|
|
264
|
+
|
|
265
|
+
A LevelDBVersion represents the current state of the table files and log file
|
|
266
|
+
that are currently "active". The current state is determined by the sequence
|
|
267
|
+
of VersionEdits that are parsed from a descriptor file.
|
|
268
|
+
|
|
269
|
+
Active files can contain overlapping keys in the current log file and the
|
|
270
|
+
"young" or 0-level.
|
|
271
|
+
|
|
272
|
+
"Deleted files" will typically no longer exist but may be forensically
|
|
273
|
+
recoverable.
|
|
274
|
+
"""
|
|
275
|
+
current_log: str
|
|
276
|
+
version_edit_offset: int
|
|
277
|
+
last_sequence: int
|
|
278
|
+
active_files: dict[int, dict[int, NewFile]]
|
|
279
|
+
deleted_files: dict[int, dict[int, DeletedFile]]
|
|
280
|
+
|
|
281
|
+
|
|
261
282
|
class FileReader:
|
|
262
283
|
"""A reader for Descriptor files.
|
|
263
284
|
|
|
@@ -332,3 +353,30 @@ class FileReader:
|
|
|
332
353
|
version_edit = VersionEdit.FromBytes(buffer, base_offset=offset)
|
|
333
354
|
yield version_edit
|
|
334
355
|
buffer = bytearray()
|
|
356
|
+
|
|
357
|
+
def GetVersions(self) -> Generator[LevelDBVersion, None, None]:
|
|
358
|
+
"""Returns an iterator of LevelDBVersion instances.
|
|
359
|
+
|
|
360
|
+
Yields:
|
|
361
|
+
LevelDBVersion
|
|
362
|
+
"""
|
|
363
|
+
active_files = defaultdict(dict)
|
|
364
|
+
deleted_files = defaultdict(set)
|
|
365
|
+
current_log = None
|
|
366
|
+
|
|
367
|
+
for version_edit in self.GetVersionEdits():
|
|
368
|
+
current_log = f'{version_edit.log_number:06d}.log'
|
|
369
|
+
|
|
370
|
+
for new_file in version_edit.new_files:
|
|
371
|
+
active_files[new_file.level][f'{new_file.number:06d}.ldb'] = new_file
|
|
372
|
+
|
|
373
|
+
for deleted_file in version_edit.deleted_files:
|
|
374
|
+
active_files[deleted_file.level].pop(f'{deleted_file.number:06d}.ldb')
|
|
375
|
+
deleted_files[deleted_file.level].add(f'{deleted_file.number:06d}.ldb')
|
|
376
|
+
|
|
377
|
+
yield LevelDBVersion(
|
|
378
|
+
current_log=current_log,
|
|
379
|
+
active_files=dict(active_files),
|
|
380
|
+
deleted_files=dict(deleted_files),
|
|
381
|
+
version_edit_offset=version_edit.offset,
|
|
382
|
+
last_sequence=version_edit.last_sequence)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dfindexeddb
|
|
3
|
-
Version:
|
|
3
|
+
Version: 20240331a0
|
|
4
4
|
Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
|
|
5
5
|
Author-email: Syd Pleno <sydp@google.com>
|
|
6
6
|
Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
|
|
@@ -283,7 +283,7 @@ installation:
|
|
|
283
283
|
|
|
284
284
|
```
|
|
285
285
|
$ dfindexeddb -h
|
|
286
|
-
usage: dfindexeddb [-h] -s SOURCE [
|
|
286
|
+
usage: dfindexeddb [-h] -s SOURCE [-o {json,jsonl,repr}]
|
|
287
287
|
|
|
288
288
|
A cli tool for parsing indexeddb files
|
|
289
289
|
|
|
@@ -291,7 +291,8 @@ options:
|
|
|
291
291
|
-h, --help show this help message and exit
|
|
292
292
|
-s SOURCE, --source SOURCE
|
|
293
293
|
The source leveldb folder
|
|
294
|
-
|
|
294
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
295
|
+
Output format. Default is json
|
|
295
296
|
```
|
|
296
297
|
|
|
297
298
|
### LevelDB
|
|
@@ -316,17 +317,46 @@ options:
|
|
|
316
317
|
To parse records from a LevelDB log (.log) file, use the following command:
|
|
317
318
|
|
|
318
319
|
```
|
|
319
|
-
$ dfleveldb log
|
|
320
|
+
$ dfleveldb log -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,write_batches,parsed_internal_key}]
|
|
321
|
+
|
|
322
|
+
options:
|
|
323
|
+
-h, --help show this help message and exit
|
|
324
|
+
-s SOURCE, --source SOURCE
|
|
325
|
+
The source leveldb file
|
|
326
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
327
|
+
Output format. Default is json
|
|
328
|
+
-t {blocks,physical_records,write_batches,parsed_internal_key}, --structure_type {blocks,physical_records,write_batches,parsed_internal_key}
|
|
329
|
+
Parses the specified structure. Default is parsed_internal_key.
|
|
320
330
|
```
|
|
321
331
|
|
|
322
332
|
To parse records from a LevelDB table (.ldb) file, use the following command:
|
|
323
333
|
|
|
324
334
|
```
|
|
325
|
-
$ dfleveldb ldb -s
|
|
335
|
+
$ dfleveldb ldb -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,records}]
|
|
336
|
+
|
|
337
|
+
options:
|
|
338
|
+
-h, --help show this help message and exit
|
|
339
|
+
-s SOURCE, --source SOURCE
|
|
340
|
+
The source leveldb file
|
|
341
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
342
|
+
Output format. Default is json
|
|
343
|
+
-t {blocks,records}, --structure_type {blocks,records}
|
|
344
|
+
Parses the specified structure. Default is records.
|
|
326
345
|
```
|
|
327
346
|
|
|
328
347
|
To parse version edit records from a Descriptor (MANIFEST) file:
|
|
329
348
|
|
|
330
349
|
```
|
|
331
|
-
$ dfleveldb descriptor -s
|
|
350
|
+
$ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,versionedit} | -v]
|
|
351
|
+
|
|
352
|
+
options:
|
|
353
|
+
-h, --help show this help message and exit
|
|
354
|
+
-s SOURCE, --source SOURCE
|
|
355
|
+
The source leveldb file
|
|
356
|
+
-o {json,jsonl,repr}, --output {json,jsonl,repr}
|
|
357
|
+
Output format. Default is json
|
|
358
|
+
-t {blocks,physical_records,versionedit}, --structure_type {blocks,physical_records,versionedit}
|
|
359
|
+
Parses the specified structure. Default is versionedit.
|
|
360
|
+
-v, --version_history
|
|
361
|
+
Parses the leveldb version history.
|
|
332
362
|
```
|
|
@@ -14,12 +14,15 @@ dfindexeddb.egg-info/entry_points.txt
|
|
|
14
14
|
dfindexeddb.egg-info/requires.txt
|
|
15
15
|
dfindexeddb.egg-info/top_level.txt
|
|
16
16
|
dfindexeddb/indexeddb/__init__.py
|
|
17
|
-
dfindexeddb/indexeddb/blink.py
|
|
18
|
-
dfindexeddb/indexeddb/chromium.py
|
|
19
17
|
dfindexeddb/indexeddb/cli.py
|
|
20
|
-
dfindexeddb/indexeddb/definitions.py
|
|
21
18
|
dfindexeddb/indexeddb/utils.py
|
|
22
|
-
dfindexeddb/indexeddb/
|
|
19
|
+
dfindexeddb/indexeddb/chromium/__init__.py
|
|
20
|
+
dfindexeddb/indexeddb/chromium/blink.py
|
|
21
|
+
dfindexeddb/indexeddb/chromium/definitions.py
|
|
22
|
+
dfindexeddb/indexeddb/chromium/record.py
|
|
23
|
+
dfindexeddb/indexeddb/chromium/v8.py
|
|
24
|
+
dfindexeddb/indexeddb/firefox/__init__.py
|
|
25
|
+
dfindexeddb/indexeddb/safari/__init__.py
|
|
23
26
|
dfindexeddb/leveldb/__init__.py
|
|
24
27
|
dfindexeddb/leveldb/cli.py
|
|
25
28
|
dfindexeddb/leveldb/definitions.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "dfindexeddb"
|
|
7
|
-
version = "
|
|
7
|
+
version = "20240331a"
|
|
8
8
|
requires-python = ">=3.8"
|
|
9
9
|
description = "dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files."
|
|
10
10
|
license = {file = "LICENSE"}
|
|
@@ -27,7 +27,14 @@ dfindexeddb = "dfindexeddb.indexeddb.cli:App"
|
|
|
27
27
|
dfleveldb = "dfindexeddb.leveldb.cli:App"
|
|
28
28
|
|
|
29
29
|
[tool.setuptools]
|
|
30
|
-
packages = [
|
|
30
|
+
packages = [
|
|
31
|
+
"dfindexeddb",
|
|
32
|
+
"dfindexeddb.indexeddb",
|
|
33
|
+
"dfindexeddb.indexeddb.chromium",
|
|
34
|
+
"dfindexeddb.indexeddb.firefox",
|
|
35
|
+
"dfindexeddb.indexeddb.safari",
|
|
36
|
+
"dfindexeddb.leveldb",
|
|
37
|
+
]
|
|
31
38
|
|
|
32
39
|
[project.urls]
|
|
33
40
|
Homepage = "https://github.com/google/dfindexeddb"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|