lupaxa-magic-bytes 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa_magic_bytes-0.1.0/.gitignore +26 -0
- lupaxa_magic_bytes-0.1.0/LICENCE +21 -0
- lupaxa_magic_bytes-0.1.0/PKG-INFO +261 -0
- lupaxa_magic_bytes-0.1.0/README.md +199 -0
- lupaxa_magic_bytes-0.1.0/docs/README.md +23 -0
- lupaxa_magic_bytes-0.1.0/pyproject.toml +140 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/__init__.py +1 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/__init__.py +5 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/__main__.py +6 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/analyse.py +276 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/cli.py +928 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/compare.py +117 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/config.py +153 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/errors.py +75 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/formats.py +1302 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/hexview.py +96 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/htmlreport.py +79 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/io.py +173 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/models.py +191 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/py.typed +0 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/scan.py +160 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/signatures.py +727 -0
- lupaxa_magic_bytes-0.1.0/src/lupaxa/magic_bytes/version.py +8 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# OS
|
|
2
|
+
.DS_Store
|
|
3
|
+
|
|
4
|
+
# Cursor
|
|
5
|
+
.cursor/
|
|
6
|
+
.superpowers/
|
|
7
|
+
.worktrees/
|
|
8
|
+
cursor-docs
|
|
9
|
+
|
|
10
|
+
# Make
|
|
11
|
+
.makefiles/
|
|
12
|
+
|
|
13
|
+
# MkDocs
|
|
14
|
+
site/
|
|
15
|
+
|
|
16
|
+
# Python
|
|
17
|
+
.venv/
|
|
18
|
+
__pycache__/
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.ruff_cache/
|
|
21
|
+
.pytest_cache/
|
|
22
|
+
dist/
|
|
23
|
+
build/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.hypothesis/
|
|
26
|
+
.audit-env/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) The Lupaxa Project
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lupaxa-magic-bytes
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline command-line tool for identifying, validating, and extracting binary file structures.
|
|
5
|
+
Project-URL: Homepage, https://github.com/lupaxa-security-toolbox/magic-bytes
|
|
6
|
+
Project-URL: Repository, https://github.com/lupaxa-security-toolbox/magic-bytes
|
|
7
|
+
Project-URL: Issues, https://github.com/lupaxa-security-toolbox/magic-bytes/issues
|
|
8
|
+
Author: The Lupaxa Project
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) The Lupaxa Project
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENCE
|
|
31
|
+
Keywords: binary,cli,file-formats,forensics
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: Environment :: Console
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: MacOS
|
|
36
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
37
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
38
|
+
Classifier: Programming Language :: Python :: 3
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
42
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
43
|
+
Classifier: Topic :: Security
|
|
44
|
+
Classifier: Topic :: System :: Filesystems
|
|
45
|
+
Classifier: Topic :: Utilities
|
|
46
|
+
Requires-Python: >=3.11
|
|
47
|
+
Requires-Dist: platformdirs>=4.2.0
|
|
48
|
+
Requires-Dist: rich>=13.0.0
|
|
49
|
+
Requires-Dist: typer>=0.12.0
|
|
50
|
+
Provides-Extra: dev
|
|
51
|
+
Requires-Dist: bump-my-version>=1.2.0; extra == 'dev'
|
|
52
|
+
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
53
|
+
Requires-Dist: mypy>=1.12.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
55
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
56
|
+
Requires-Dist: ruff>=0.6.0; extra == 'dev'
|
|
57
|
+
Provides-Extra: test
|
|
58
|
+
Requires-Dist: hypothesis>=6.100; extra == 'test'
|
|
59
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'test'
|
|
60
|
+
Requires-Dist: pytest>=8.0; extra == 'test'
|
|
61
|
+
Description-Content-Type: text/markdown
|
|
62
|
+
|
|
63
|
+
<p align="center">
|
|
64
|
+
<a href="https://github.com/lupaxa-security-toolbox">
|
|
65
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/organisations/security-toolbox/readme-logo.png" alt="Security Toolbox" />
|
|
66
|
+
</a>
|
|
67
|
+
</p>
|
|
68
|
+
|
|
69
|
+
<h1 align="center">Magic Bytes</h1>
|
|
70
|
+
|
|
71
|
+
Magic Bytes identifies, checks, and extracts binary file structures. It runs offline and does not use the network.
|
|
72
|
+
|
|
73
|
+
A signature is not proof of a valid file. Validation is not proof that a file is safe to open. The tool never executes what it inspects or extracts.
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
Python 3.11 or newer:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install lupaxa-magic-bytes
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The command is `magic-bytes`.
|
|
84
|
+
|
|
85
|
+
## Quick Start
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
magic-bytes identify mystery.dat
|
|
89
|
+
magic-bytes validate picture.png --level standard
|
|
90
|
+
magic-bytes carve suspicious.bin --list
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## What it Does
|
|
94
|
+
|
|
95
|
+
- Identifies formats from bytes, including files whose extension is wrong or missing.
|
|
96
|
+
- Validates PNG, JPEG, GIF, BMP, ZIP, PDF, WAV, MP3, ELF, PE, SQLite, and GZIP.
|
|
97
|
+
- Reports other common formats from signatures when no structural validator exists.
|
|
98
|
+
- Finds embedded objects, trailing bytes, and overlapping interpretations.
|
|
99
|
+
- Carves recoverable ranges without modifying the source.
|
|
100
|
+
- Maps layouts, measures entropy, dumps hex, compares files, and scans directories.
|
|
101
|
+
- Accepts declarative TOML signatures. Rules cannot run programs.
|
|
102
|
+
|
|
103
|
+
## Confidence
|
|
104
|
+
|
|
105
|
+
Confidence is a label, not a probability. A file can be confidently identified and still be truncated.
|
|
106
|
+
|
|
107
|
+
| Confidence | Meaning |
|
|
108
|
+
| ----------- | ------------------------------------------------ |
|
|
109
|
+
| `confirmed` | Structural validation passed |
|
|
110
|
+
| `high` | Strong signature, or validation was only partial |
|
|
111
|
+
| `medium` | A recognised signature with limited support |
|
|
112
|
+
| `low` | A weak or ambiguous signature |
|
|
113
|
+
| `unknown` | Not enough evidence |
|
|
114
|
+
|
|
115
|
+
## Commands
|
|
116
|
+
|
|
117
|
+
Global options come before the subcommand.
|
|
118
|
+
|
|
119
|
+
| Option | Effect |
|
|
120
|
+
| ----------------------- | ------------------------------- |
|
|
121
|
+
| `--config PATH` | Load a TOML file |
|
|
122
|
+
| `--json` | Print one JSON document |
|
|
123
|
+
| `--quiet` | Hide the human report |
|
|
124
|
+
| `--verbose` | More diagnostic detail |
|
|
125
|
+
| `--debug` | Trace unexpected errors |
|
|
126
|
+
| `--no-colour` | Plain text |
|
|
127
|
+
| `--no-progress` | Hide progress |
|
|
128
|
+
| `--jobs N` | Worker count for scans |
|
|
129
|
+
| `--max-file-size SIZE` | Refuse larger inputs |
|
|
130
|
+
| `--max-scan-bytes SIZE` | Bound a deep scan |
|
|
131
|
+
| `--timeout SECONDS` | Stop bounded work |
|
|
132
|
+
| `--force` | Replace an existing output file |
|
|
133
|
+
| `--version` | Print the version |
|
|
134
|
+
| `--help` | Show help |
|
|
135
|
+
|
|
136
|
+
Sizes accept integers, `0x` hex, and `K`, `M`, or `G` suffixes.
|
|
137
|
+
|
|
138
|
+
| Command | Purpose | Main options | | |
|
|
139
|
+
| ------------ | ------------------------------- | ---------------------------------------------------------------------------- | | |
|
|
140
|
+
| `identify` | Name the likely formats | `--all`, `--deep`, `--offset`, `--signatures-only` | | |
|
|
141
|
+
| `inspect` | Combine structure and findings | `--deep`, `--embedded`, `--trailing`, `--entropy`, `--structure`, `--report` | | |
|
|
142
|
+
| `validate` | Check one recognised structure | `--format`, `--level basic\ | standard\ | deep`, `--strict`, `--report` |
|
|
143
|
+
| `scan` | Walk a directory | `--recursive`, `--mismatches`, `--invalid`, `--embedded`, `--trailing` | | |
|
|
144
|
+
| `carve` | Copy a range out of a file | `--list`, `--id`, `--all`, `--offset`, `--length`, `--validated-only` | | |
|
|
145
|
+
| `compare` | Diff two files | `--structural`, `--bytewise`, `--format text\ | json\ | html` |
|
|
146
|
+
| `entropy` | Shannon bits per byte | `--block-size`, `--window-size`, `--step`, `--offset`, `--length`, `--chart` | | |
|
|
147
|
+
| `hexdump` | Show or search bytes | `--offset`, `--length`, `--width`, `--find`, `--utf8` | | |
|
|
148
|
+
| `map` | Draw regions | `--format terminal\ | html\ | json`, `--show-entropy`, `--show-embedded` |
|
|
149
|
+
| `signatures` | List and manage TOML rules | `list`, `show`, `add`, `remove`, `validate`, `test` | | |
|
|
150
|
+
|
|
151
|
+
`identify` keeps competing candidates. The primary one is the strongest, not merely the first. An extension mismatch is an observation. High entropy is not evidence of encryption, and a PE overlay is not evidence of malice.
|
|
152
|
+
|
|
153
|
+
## Validation
|
|
154
|
+
|
|
155
|
+
| Level | Behaviour |
|
|
156
|
+
| ---------- | ------------------------------------------------------------ |
|
|
157
|
+
| `basic` | Signature and essential header checks |
|
|
158
|
+
| `standard` | Boundaries, required chunks, and declared sizes |
|
|
159
|
+
| `deep` | CRCs, deflate checks, and a read-only SQLite integrity check |
|
|
160
|
+
|
|
161
|
+
`--strict` turns warnings into failures. A failed validation exits 6. That is different from the validator failing to run.
|
|
162
|
+
|
|
163
|
+
PDF reports say when a full semantic parse was not done. ZIP64 and xref streams are called out instead of being treated as corruption. SQLite deep checks open the file read-only and immutable so they do not create a journal.
|
|
164
|
+
|
|
165
|
+
## Carving
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
magic-bytes carve suspicious.bin --list
|
|
169
|
+
magic-bytes carve suspicious.bin --id 2 --output recovered.bin
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Boundaries are `exact`, `estimated`, or `user-selected`. Only an exact validated extraction is described as fully validated. Output is never executed. Existing files are kept unless `--force` is set. Each extraction reports the source
|
|
173
|
+
offset, length, and SHA-256.
|
|
174
|
+
|
|
175
|
+
## Custom Signatures
|
|
176
|
+
|
|
177
|
+
```toml
|
|
178
|
+
[[signature]]
|
|
179
|
+
id = "custom-data"
|
|
180
|
+
name = "Custom Data File"
|
|
181
|
+
mime = "application/x-custom"
|
|
182
|
+
extensions = [".dat"]
|
|
183
|
+
min_size = 16
|
|
184
|
+
|
|
185
|
+
[[signature.match]]
|
|
186
|
+
offset = 0
|
|
187
|
+
hex = "4C 55 50 41 ?? 41"
|
|
188
|
+
required = true
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
`hex` may also be a list of alternative patterns. `??` is a wildcard. `end_relative` measures from the end of the file. Built-in signatures cannot be replaced. A rule file that sets `command`, `exec`, `script`, `python`, or `shell` is
|
|
192
|
+
rejected.
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
magic-bytes signatures validate custom.toml
|
|
196
|
+
magic-bytes signatures test custom.toml sample.dat
|
|
197
|
+
magic-bytes signatures add custom.toml
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## JSON
|
|
201
|
+
|
|
202
|
+
`--json` prints one object and nothing else on standard output. Offsets are integers. Reports include `catalogue_version` and `validator_version`.
|
|
203
|
+
|
|
204
|
+
```json
|
|
205
|
+
{
|
|
206
|
+
"schema_version": 1,
|
|
207
|
+
"command": "identify",
|
|
208
|
+
"success": true,
|
|
209
|
+
"results": []
|
|
210
|
+
}
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
## Exit Codes
|
|
214
|
+
|
|
215
|
+
| Code | Meaning |
|
|
216
|
+
| ---- | ---------------------------------- |
|
|
217
|
+
| 0 | Success |
|
|
218
|
+
| 1 | General failure |
|
|
219
|
+
| 2 | Invalid arguments or configuration |
|
|
220
|
+
| 3 | Unsupported format or operation |
|
|
221
|
+
| 4 | Input or output error |
|
|
222
|
+
| 5 | No recognised format |
|
|
223
|
+
| 6 | Structural validation failed |
|
|
224
|
+
| 7 | Extraction failed |
|
|
225
|
+
| 8 | Resource limit exceeded |
|
|
226
|
+
| 9 | Invalid custom signature |
|
|
227
|
+
| 10 | Partial batch failure |
|
|
228
|
+
|
|
229
|
+
## Configuration
|
|
230
|
+
|
|
231
|
+
A file is optional. The user file is the platform config directory for `magic-bytes` / `Lupaxa` / `config.toml`. Environment variables use the `MAGIC_BYTES_` prefix and override the file. Explicit CLI options override both.
|
|
232
|
+
|
|
233
|
+
```toml
|
|
234
|
+
validation_level = "standard"
|
|
235
|
+
max_file_size = 536870912
|
|
236
|
+
max_scan_bytes = 536870912
|
|
237
|
+
max_depth = 4
|
|
238
|
+
max_candidates = 256
|
|
239
|
+
max_extract_size = 268435456
|
|
240
|
+
workers = 4
|
|
241
|
+
entropy_block_size = 4096
|
|
242
|
+
colour = true
|
|
243
|
+
follow_symlinks = false
|
|
244
|
+
timeout_seconds = 60
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Security
|
|
248
|
+
|
|
249
|
+
Inputs are untrusted. Declared lengths are checked before they are used. Decompression and candidate counts stop at configured limits. Archive carving copies the archive object; it does not unpack members or follow paths inside them. HTML
|
|
250
|
+
reports are self-contained and escape file-derived text.
|
|
251
|
+
|
|
252
|
+
## Troubleshooting
|
|
253
|
+
|
|
254
|
+
- `validate` on a signature-only format exits 3 because there is no structural validator.
|
|
255
|
+
- A ZIP or PDF with a prefix can still be valid. The report names the prefix.
|
|
256
|
+
- Deep scans can stop early when the candidate or byte limit is reached. The report then says the analysis is incomplete.
|
|
257
|
+
- `carve` without `--list`, `--id`, `--all`, or `--offset` only lists objects.
|
|
258
|
+
|
|
259
|
+
<a href="https://github.com/the-lupaxa-project">
|
|
260
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/components/footer-for-child-orgs.svg" alt="The Lupaxa Project Footer" width="100%" />
|
|
261
|
+
</a>
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://github.com/lupaxa-security-toolbox">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/organisations/security-toolbox/readme-logo.png" alt="Security Toolbox" />
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<h1 align="center">Magic Bytes</h1>
|
|
8
|
+
|
|
9
|
+
Magic Bytes identifies, checks, and extracts binary file structures. It runs offline and does not use the network.
|
|
10
|
+
|
|
11
|
+
A signature is not proof of a valid file. Validation is not proof that a file is safe to open. The tool never executes what it inspects or extracts.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
Python 3.11 or newer:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install lupaxa-magic-bytes
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The command is `magic-bytes`.
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
magic-bytes identify mystery.dat
|
|
27
|
+
magic-bytes validate picture.png --level standard
|
|
28
|
+
magic-bytes carve suspicious.bin --list
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## What it Does
|
|
32
|
+
|
|
33
|
+
- Identifies formats from bytes, including files whose extension is wrong or missing.
|
|
34
|
+
- Validates PNG, JPEG, GIF, BMP, ZIP, PDF, WAV, MP3, ELF, PE, SQLite, and GZIP.
|
|
35
|
+
- Reports other common formats from signatures when no structural validator exists.
|
|
36
|
+
- Finds embedded objects, trailing bytes, and overlapping interpretations.
|
|
37
|
+
- Carves recoverable ranges without modifying the source.
|
|
38
|
+
- Maps layouts, measures entropy, dumps hex, compares files, and scans directories.
|
|
39
|
+
- Accepts declarative TOML signatures. Rules cannot run programs.
|
|
40
|
+
|
|
41
|
+
## Confidence
|
|
42
|
+
|
|
43
|
+
Confidence is a label, not a probability. A file can be confidently identified and still be truncated.
|
|
44
|
+
|
|
45
|
+
| Confidence | Meaning |
|
|
46
|
+
| ----------- | ------------------------------------------------ |
|
|
47
|
+
| `confirmed` | Structural validation passed |
|
|
48
|
+
| `high` | Strong signature, or validation was only partial |
|
|
49
|
+
| `medium` | A recognised signature with limited support |
|
|
50
|
+
| `low` | A weak or ambiguous signature |
|
|
51
|
+
| `unknown` | Not enough evidence |
|
|
52
|
+
|
|
53
|
+
## Commands
|
|
54
|
+
|
|
55
|
+
Global options come before the subcommand.
|
|
56
|
+
|
|
57
|
+
| Option | Effect |
|
|
58
|
+
| ----------------------- | ------------------------------- |
|
|
59
|
+
| `--config PATH` | Load a TOML file |
|
|
60
|
+
| `--json` | Print one JSON document |
|
|
61
|
+
| `--quiet` | Hide the human report |
|
|
62
|
+
| `--verbose` | More diagnostic detail |
|
|
63
|
+
| `--debug` | Trace unexpected errors |
|
|
64
|
+
| `--no-colour` | Plain text |
|
|
65
|
+
| `--no-progress` | Hide progress |
|
|
66
|
+
| `--jobs N` | Worker count for scans |
|
|
67
|
+
| `--max-file-size SIZE` | Refuse larger inputs |
|
|
68
|
+
| `--max-scan-bytes SIZE` | Bound a deep scan |
|
|
69
|
+
| `--timeout SECONDS` | Stop bounded work |
|
|
70
|
+
| `--force` | Replace an existing output file |
|
|
71
|
+
| `--version` | Print the version |
|
|
72
|
+
| `--help` | Show help |
|
|
73
|
+
|
|
74
|
+
Sizes accept integers, `0x` hex, and `K`, `M`, or `G` suffixes.
|
|
75
|
+
|
|
76
|
+
| Command | Purpose | Main options | | |
|
|
77
|
+
| ------------ | ------------------------------- | ---------------------------------------------------------------------------- | | |
|
|
78
|
+
| `identify` | Name the likely formats | `--all`, `--deep`, `--offset`, `--signatures-only` | | |
|
|
79
|
+
| `inspect` | Combine structure and findings | `--deep`, `--embedded`, `--trailing`, `--entropy`, `--structure`, `--report` | | |
|
|
80
|
+
| `validate` | Check one recognised structure | `--format`, `--level basic\ | standard\ | deep`, `--strict`, `--report` |
|
|
81
|
+
| `scan` | Walk a directory | `--recursive`, `--mismatches`, `--invalid`, `--embedded`, `--trailing` | | |
|
|
82
|
+
| `carve` | Copy a range out of a file | `--list`, `--id`, `--all`, `--offset`, `--length`, `--validated-only` | | |
|
|
83
|
+
| `compare` | Diff two files | `--structural`, `--bytewise`, `--format text\ | json\ | html` |
|
|
84
|
+
| `entropy` | Shannon bits per byte | `--block-size`, `--window-size`, `--step`, `--offset`, `--length`, `--chart` | | |
|
|
85
|
+
| `hexdump` | Show or search bytes | `--offset`, `--length`, `--width`, `--find`, `--utf8` | | |
|
|
86
|
+
| `map` | Draw regions | `--format terminal\ | html\ | json`, `--show-entropy`, `--show-embedded` |
|
|
87
|
+
| `signatures` | List and manage TOML rules | `list`, `show`, `add`, `remove`, `validate`, `test` | | |
|
|
88
|
+
|
|
89
|
+
`identify` keeps competing candidates. The primary one is the strongest, not merely the first. An extension mismatch is an observation. High entropy is not evidence of encryption, and a PE overlay is not evidence of malice.
|
|
90
|
+
|
|
91
|
+
## Validation
|
|
92
|
+
|
|
93
|
+
| Level | Behaviour |
|
|
94
|
+
| ---------- | ------------------------------------------------------------ |
|
|
95
|
+
| `basic` | Signature and essential header checks |
|
|
96
|
+
| `standard` | Boundaries, required chunks, and declared sizes |
|
|
97
|
+
| `deep` | CRCs, deflate checks, and a read-only SQLite integrity check |
|
|
98
|
+
|
|
99
|
+
`--strict` turns warnings into failures. A failed validation exits 6. That is different from the validator failing to run.
|
|
100
|
+
|
|
101
|
+
PDF reports say when a full semantic parse was not done. ZIP64 and xref streams are called out instead of being treated as corruption. SQLite deep checks open the file read-only and immutable so they do not create a journal.
|
|
102
|
+
|
|
103
|
+
## Carving
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
magic-bytes carve suspicious.bin --list
|
|
107
|
+
magic-bytes carve suspicious.bin --id 2 --output recovered.bin
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Boundaries are `exact`, `estimated`, or `user-selected`. Only an exact validated extraction is described as fully validated. Output is never executed. Existing files are kept unless `--force` is set. Each extraction reports the source
|
|
111
|
+
offset, length, and SHA-256.
|
|
112
|
+
|
|
113
|
+
## Custom Signatures
|
|
114
|
+
|
|
115
|
+
```toml
|
|
116
|
+
[[signature]]
|
|
117
|
+
id = "custom-data"
|
|
118
|
+
name = "Custom Data File"
|
|
119
|
+
mime = "application/x-custom"
|
|
120
|
+
extensions = [".dat"]
|
|
121
|
+
min_size = 16
|
|
122
|
+
|
|
123
|
+
[[signature.match]]
|
|
124
|
+
offset = 0
|
|
125
|
+
hex = "4C 55 50 41 ?? 41"
|
|
126
|
+
required = true
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
`hex` may also be a list of alternative patterns. `??` is a wildcard. `end_relative` measures from the end of the file. Built-in signatures cannot be replaced. A rule file that sets `command`, `exec`, `script`, `python`, or `shell` is
|
|
130
|
+
rejected.
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
magic-bytes signatures validate custom.toml
|
|
134
|
+
magic-bytes signatures test custom.toml sample.dat
|
|
135
|
+
magic-bytes signatures add custom.toml
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## JSON
|
|
139
|
+
|
|
140
|
+
`--json` prints one object and nothing else on standard output. Offsets are integers. Reports include `catalogue_version` and `validator_version`.
|
|
141
|
+
|
|
142
|
+
```json
|
|
143
|
+
{
|
|
144
|
+
"schema_version": 1,
|
|
145
|
+
"command": "identify",
|
|
146
|
+
"success": true,
|
|
147
|
+
"results": []
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Exit Codes
|
|
152
|
+
|
|
153
|
+
| Code | Meaning |
|
|
154
|
+
| ---- | ---------------------------------- |
|
|
155
|
+
| 0 | Success |
|
|
156
|
+
| 1 | General failure |
|
|
157
|
+
| 2 | Invalid arguments or configuration |
|
|
158
|
+
| 3 | Unsupported format or operation |
|
|
159
|
+
| 4 | Input or output error |
|
|
160
|
+
| 5 | No recognised format |
|
|
161
|
+
| 6 | Structural validation failed |
|
|
162
|
+
| 7 | Extraction failed |
|
|
163
|
+
| 8 | Resource limit exceeded |
|
|
164
|
+
| 9 | Invalid custom signature |
|
|
165
|
+
| 10 | Partial batch failure |
|
|
166
|
+
|
|
167
|
+
## Configuration
|
|
168
|
+
|
|
169
|
+
A file is optional. The user file is the platform config directory for `magic-bytes` / `Lupaxa` / `config.toml`. Environment variables use the `MAGIC_BYTES_` prefix and override the file. Explicit CLI options override both.
|
|
170
|
+
|
|
171
|
+
```toml
|
|
172
|
+
validation_level = "standard"
|
|
173
|
+
max_file_size = 536870912
|
|
174
|
+
max_scan_bytes = 536870912
|
|
175
|
+
max_depth = 4
|
|
176
|
+
max_candidates = 256
|
|
177
|
+
max_extract_size = 268435456
|
|
178
|
+
workers = 4
|
|
179
|
+
entropy_block_size = 4096
|
|
180
|
+
colour = true
|
|
181
|
+
follow_symlinks = false
|
|
182
|
+
timeout_seconds = 60
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Security
|
|
186
|
+
|
|
187
|
+
Inputs are untrusted. Declared lengths are checked before they are used. Decompression and candidate counts stop at configured limits. Archive carving copies the archive object; it does not unpack members or follow paths inside them. HTML
|
|
188
|
+
reports are self-contained and escape file-derived text.
|
|
189
|
+
|
|
190
|
+
## Troubleshooting
|
|
191
|
+
|
|
192
|
+
- `validate` on a signature-only format exits 3 because there is no structural validator.
|
|
193
|
+
- A ZIP or PDF with a prefix can still be valid. The report names the prefix.
|
|
194
|
+
- Deep scans can stop early when the candidate or byte limit is reached. The report then says the analysis is incomplete.
|
|
195
|
+
- `carve` without `--list`, `--id`, `--all`, or `--offset` only lists objects.
|
|
196
|
+
|
|
197
|
+
<a href="https://github.com/the-lupaxa-project">
|
|
198
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/components/footer-for-child-orgs.svg" alt="The Lupaxa Project Footer" width="100%" />
|
|
199
|
+
</a>
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://github.com/lupaxa-security-toolbox">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/organisations/security-toolbox/readme-logo.png" alt="Organisation Logo" />
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<h1 align="center">Core Documentation</h1>
|
|
8
|
+
|
|
9
|
+
The following documents define the standards that apply across **The Lupaxa Project**:
|
|
10
|
+
|
|
11
|
+
| Document | Purpose |
|
|
12
|
+
| :------------------------------------ | :---------------------------------------------------------------- |
|
|
13
|
+
| [Code of Conduct](CODE_OF_CONDUCT.md) | Expected behaviour and reporting procedures. |
|
|
14
|
+
| [How to Contribute](CONTRIBUTING.md) | Development workflow, coding standards, and contribution process. |
|
|
15
|
+
| [Governance](GOVERNANCE.md) | Project governance, roles, and decision-making. |
|
|
16
|
+
| [Maintainers](MAINTAINERS.md) | Maintainer responsibilities and expectations. |
|
|
17
|
+
| [Projects Overview](PROJECTS.md) | Overview of organisations, repositories, and project structure. |
|
|
18
|
+
| [Security Policy](SECURITY.md) | Responsible disclosure and secure development practices. |
|
|
19
|
+
| [Support Guide](SUPPORT.md) | How to obtain help and support. |
|
|
20
|
+
|
|
21
|
+
<a href="https://github.com/the-lupaxa-project">
|
|
22
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/components/footer-for-child-orgs.svg" alt="The Lupaxa Project Footer" width="100%" />
|
|
23
|
+
</a>
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.18"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "lupaxa-magic-bytes"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Offline command-line tool for identifying, validating, and extracting binary file structures."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { file = "LICENCE" }
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "The Lupaxa Project" },
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"binary",
|
|
17
|
+
"file-formats",
|
|
18
|
+
"forensics",
|
|
19
|
+
"cli",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"platformdirs>=4.2.0",
|
|
23
|
+
"rich>=13.0.0",
|
|
24
|
+
"typer>=0.12.0",
|
|
25
|
+
]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 3 - Alpha",
|
|
28
|
+
"Environment :: Console",
|
|
29
|
+
"License :: OSI Approved :: MIT License",
|
|
30
|
+
"Operating System :: MacOS",
|
|
31
|
+
"Operating System :: Microsoft :: Windows",
|
|
32
|
+
"Operating System :: POSIX :: Linux",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.11",
|
|
35
|
+
"Programming Language :: Python :: 3.12",
|
|
36
|
+
"Programming Language :: Python :: 3.13",
|
|
37
|
+
"Programming Language :: Python :: 3.14",
|
|
38
|
+
"Topic :: Security",
|
|
39
|
+
"Topic :: System :: Filesystems",
|
|
40
|
+
"Topic :: Utilities",
|
|
41
|
+
]
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/lupaxa-security-toolbox/magic-bytes"
|
|
44
|
+
Repository = "https://github.com/lupaxa-security-toolbox/magic-bytes"
|
|
45
|
+
Issues = "https://github.com/lupaxa-security-toolbox/magic-bytes/issues"
|
|
46
|
+
|
|
47
|
+
[project.scripts]
|
|
48
|
+
magic-bytes = "lupaxa.magic_bytes.cli:main"
|
|
49
|
+
|
|
50
|
+
[project.optional-dependencies]
|
|
51
|
+
dev = [
|
|
52
|
+
"bump-my-version>=1.2.0",
|
|
53
|
+
"hypothesis>=6.100",
|
|
54
|
+
"mypy>=1.12.0",
|
|
55
|
+
"pytest>=8.0",
|
|
56
|
+
"pytest-cov>=5.0",
|
|
57
|
+
"ruff>=0.6.0",
|
|
58
|
+
]
|
|
59
|
+
test = [
|
|
60
|
+
"hypothesis>=6.100",
|
|
61
|
+
"pytest>=8.0",
|
|
62
|
+
"pytest-cov>=5.0",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[tool.hatch.version]
|
|
66
|
+
path = "src/lupaxa/magic_bytes/version.py"
|
|
67
|
+
|
|
68
|
+
[tool.hatch.build.targets.sdist]
|
|
69
|
+
include = [
|
|
70
|
+
"src/**",
|
|
71
|
+
"README.md",
|
|
72
|
+
"LICENCE",
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
[tool.hatch.build.targets.wheel]
|
|
76
|
+
packages = ["src/lupaxa"]
|
|
77
|
+
|
|
78
|
+
[tool.ruff]
|
|
79
|
+
line-length = 120
|
|
80
|
+
target-version = "py311"
|
|
81
|
+
src = ["src", "tests"]
|
|
82
|
+
|
|
83
|
+
[tool.ruff.lint]
|
|
84
|
+
select = [
|
|
85
|
+
"E",
|
|
86
|
+
"F",
|
|
87
|
+
"I",
|
|
88
|
+
"B",
|
|
89
|
+
"S",
|
|
90
|
+
"UP",
|
|
91
|
+
"SIM",
|
|
92
|
+
"C4",
|
|
93
|
+
"ISC",
|
|
94
|
+
"D",
|
|
95
|
+
"ANN",
|
|
96
|
+
"PT",
|
|
97
|
+
]
|
|
98
|
+
ignore = [
|
|
99
|
+
"D100",
|
|
100
|
+
"D104",
|
|
101
|
+
"D107",
|
|
102
|
+
"D200",
|
|
103
|
+
"D201",
|
|
104
|
+
"D202",
|
|
105
|
+
"D203",
|
|
106
|
+
"D204",
|
|
107
|
+
"D205",
|
|
108
|
+
"D206",
|
|
109
|
+
"D207",
|
|
110
|
+
"D208",
|
|
111
|
+
"D209",
|
|
112
|
+
"D210",
|
|
113
|
+
"D211",
|
|
114
|
+
"D212",
|
|
115
|
+
"D213",
|
|
116
|
+
"S311",
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
[tool.ruff.lint.per-file-ignores]
|
|
120
|
+
"tests/**" = ["S101", "S105", "S106", "ANN001", "ANN201", "D103"]
|
|
121
|
+
"src/lupaxa/magic_bytes/cli.py" = ["B008"]
|
|
122
|
+
|
|
123
|
+
[tool.ruff.lint.pydocstyle]
|
|
124
|
+
convention = "numpy"
|
|
125
|
+
|
|
126
|
+
[tool.mypy]
|
|
127
|
+
python_version = "3.11"
|
|
128
|
+
warn_unused_configs = true
|
|
129
|
+
warn_return_any = true
|
|
130
|
+
warn_unused_ignores = true
|
|
131
|
+
disallow_untyped_defs = true
|
|
132
|
+
follow_imports = "normal"
|
|
133
|
+
ignore_missing_imports = true
|
|
134
|
+
packages = ["lupaxa.magic_bytes"]
|
|
135
|
+
mypy_path = "src"
|
|
136
|
+
|
|
137
|
+
[tool.pytest.ini_options]
|
|
138
|
+
addopts = "-ra --disable-warnings"
|
|
139
|
+
testpaths = ["tests"]
|
|
140
|
+
pythonpath = ["."]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Lupaxa namespace package."""
|