modmanifest 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ """ModManifest: read-only inspection, validation and metadata auditing for mod packages."""
2
+
3
+ __version__ = "0.1.0"
4
+ SCHEMA_VERSION = 1
@@ -0,0 +1,4 @@
1
+ from modmanifest.cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
@@ -0,0 +1 @@
1
+ """Package marker."""
@@ -0,0 +1,36 @@
1
+ """Configurable scan limits and the scan deadline."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from dataclasses import dataclass
7
+
8
+ MB = 1024 * 1024
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class ScanLimits:
13
+ max_entries: int = 20_000
14
+ max_archive_bytes: int = 1024 * MB
15
+ max_total_bytes: int = 500 * MB
16
+ max_entry_bytes: int = 50 * MB
17
+ max_ratio: float = 100.0
18
+ max_metadata_bytes: int = 1 * MB
19
+ max_image_bytes: int = 20 * MB
20
+ timeout_s: float = 30.0
21
+ # Ratio warnings are skipped for tiny entries, which compress extremely well legitimately.
22
+ ratio_min_bytes: int = 64 * 1024
23
+
24
+
25
+ class ScanTimeout(Exception):
26
+ """Raised when the scan deadline passes."""
27
+
28
+
29
+ class Deadline:
30
+ def __init__(self, seconds: float) -> None:
31
+ self._end = time.monotonic() + seconds
32
+ self.seconds = seconds
33
+
34
+ def check(self) -> None:
35
+ if time.monotonic() > self._end:
36
+ raise ScanTimeout(f"scan exceeded the {self.seconds:g}s time limit")
@@ -0,0 +1,47 @@
1
+ """Structural file-signature checks for extensions where a signature is well defined."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from modmanifest.context import ScanContext
6
+ from modmanifest.models import Finding, Status
7
+
8
+ _ZIP = (b"PK\x03\x04", b"PK\x05\x06")
9
+ SIGNATURES: dict[str, tuple[bytes, ...]] = {
10
+ ".png": (b"\x89PNG\r\n\x1a\n",),
11
+ ".jpg": (b"\xff\xd8\xff",),
12
+ ".jpeg": (b"\xff\xd8\xff",),
13
+ ".zip": _ZIP,
14
+ ".ts4script": _ZIP,
15
+ }
16
+ # .package signatures are verified by the Sims 4 DBPF parser, not here.
17
+
18
+
19
+ def check_signatures(ctx: ScanContext) -> None:
20
+ checked = 0
21
+ for entry in ctx.source.files():
22
+ ext = "." + entry.path.rsplit(".", 1)[-1].lower() if "." in entry.path else ""
23
+ sigs = SIGNATURES.get(ext)
24
+ is_webp = ext == ".webp"
25
+ if sigs is None and not is_webp:
26
+ continue
27
+ ctx.deadline.check()
28
+ head = ctx.head(entry.path, 12)
29
+ if head is None:
30
+ continue
31
+ checked += 1
32
+ ok = (
33
+ head[:4] == b"RIFF" and head[8:12] == b"WEBP"
34
+ if is_webp
35
+ else any(head.startswith(s) for s in sigs or ())
36
+ )
37
+ if not ok:
38
+ ctx.report.add(
39
+ Finding(
40
+ "MM-ARC-013",
41
+ Status.WARNING,
42
+ f"extension '{ext}' does not match the file signature",
43
+ source=entry.path,
44
+ )
45
+ )
46
+ if checked:
47
+ ctx.report.capability("archive.signature_check")
@@ -0,0 +1,196 @@
1
+ """Package sources: a uniform, read-only view over folders, single files and ZIP archives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import stat
7
+ from collections.abc import Iterator
8
+ from dataclasses import dataclass
9
+ from typing import BinaryIO
10
+
11
+ from modmanifest.archive.limits import Deadline, ScanLimits
12
+ from modmanifest.models import Finding, Report, Status
13
+
14
+ _REPARSE_POINT = 0x400
15
+
16
+
17
+ class SourceReadError(Exception):
18
+ """An entry could not be read."""
19
+
20
+
21
+ class EntryTooLarge(SourceReadError):
22
+ """An entry exceeds the allowed read size."""
23
+
24
+
25
+ class CorruptEntry(SourceReadError):
26
+ """An entry's data failed an integrity check."""
27
+
28
+
29
+ @dataclass(slots=True)
30
+ class Entry:
31
+ path: str
32
+ size: int
33
+ kind: str # file | dir | symlink | special | unsafe
34
+ compressed_size: int | None = None
35
+ readable: bool = True
36
+
37
+
38
+ class PackageSource:
39
+ kind = "folder"
40
+
41
+ def __init__(self, display: str) -> None:
42
+ self.display = display
43
+ self.entries: list[Entry] = []
44
+ self._by_path: dict[str, Entry] = {}
45
+
46
+ def index(self) -> None:
47
+ self._by_path = {e.path: e for e in self.entries if e.kind != "unsafe"}
48
+
49
+ def get(self, path: str) -> Entry | None:
50
+ return self._by_path.get(path)
51
+
52
+ def files(self) -> Iterator[Entry]:
53
+ return (e for e in self.entries if e.kind == "file" and e.readable)
54
+
55
+ def read_bytes(self, path: str, max_bytes: int) -> bytes:
56
+ raise NotImplementedError
57
+
58
+ def read_head(self, path: str, n: int) -> bytes:
59
+ return self.read_bytes_prefix(path, n)
60
+
61
+ def read_bytes_prefix(self, path: str, n: int) -> bytes:
62
+ raise NotImplementedError
63
+
64
+ def open_seekable(self, path: str, max_bytes: int) -> BinaryIO:
65
+ raise NotImplementedError
66
+
67
+ def close(self) -> None:
68
+ return None
69
+
70
+
71
+ class FolderSource(PackageSource):
72
+ kind = "folder"
73
+
74
+ def __init__(self, root: str, display: str) -> None:
75
+ super().__init__(display)
76
+ self.root = root
77
+
78
+ def _full(self, path: str) -> str:
79
+ entry = self.get(path)
80
+ if entry is None or entry.kind != "file" or not entry.readable:
81
+ raise SourceReadError(f"not a readable file entry: {path}")
82
+ return os.path.join(self.root, *path.split("/"))
83
+
84
+ def read_bytes(self, path: str, max_bytes: int) -> bytes:
85
+ full = self._full(path)
86
+ try:
87
+ with open(full, "rb") as fh:
88
+ data = fh.read(max_bytes + 1)
89
+ except OSError as exc:
90
+ raise SourceReadError(str(exc)) from exc
91
+ if len(data) > max_bytes:
92
+ raise EntryTooLarge(path)
93
+ return data
94
+
95
+ def read_bytes_prefix(self, path: str, n: int) -> bytes:
96
+ full = self._full(path)
97
+ try:
98
+ with open(full, "rb") as fh:
99
+ return fh.read(n)
100
+ except OSError as exc:
101
+ raise SourceReadError(str(exc)) from exc
102
+
103
+ def open_seekable(self, path: str, max_bytes: int) -> BinaryIO:
104
+ full = self._full(path)
105
+ try:
106
+ return open(full, "rb") # noqa: SIM115 - the caller owns and closes the handle
107
+ except OSError as exc:
108
+ raise SourceReadError(str(exc)) from exc
109
+
110
+
111
+ class SingleFileSource(FolderSource):
112
+ kind = "file"
113
+
114
+ def __init__(self, file_path: str) -> None:
115
+ name = os.path.basename(file_path)
116
+ super().__init__(os.path.dirname(os.path.abspath(file_path)), name)
117
+ size = os.stat(file_path).st_size
118
+ self.entries = [Entry(name, size, "file")]
119
+ self.index()
120
+
121
+
122
+ def _is_link(entry: os.DirEntry[str]) -> bool:
123
+ try:
124
+ if entry.is_symlink():
125
+ return True
126
+ attrs = getattr(entry.stat(follow_symlinks=False), "st_file_attributes", 0)
127
+ except OSError:
128
+ return False
129
+ return bool(attrs & _REPARSE_POINT)
130
+
131
+
132
+ def build_folder_source(
133
+ root: str, display: str, limits: ScanLimits, deadline: Deadline, report: Report
134
+ ) -> FolderSource:
135
+ """Walk a folder without following links and record every entry."""
136
+ src = FolderSource(root, display)
137
+ report.capability("folder.walk")
138
+ stack: list[tuple[str, str]] = [(root, "")]
139
+ links = 0
140
+ while stack:
141
+ deadline.check()
142
+ directory, prefix = stack.pop()
143
+ try:
144
+ with os.scandir(directory) as it:
145
+ children = sorted(it, key=lambda e: e.name)
146
+ except OSError as exc:
147
+ report.error("MM-SCN-003", f"could not list '{prefix or '.'}': {exc.strerror or exc}")
148
+ continue
149
+ for child in children:
150
+ rel = f"{prefix}{child.name}"
151
+ if len(src.entries) >= limits.max_entries:
152
+ report.add(
153
+ Finding(
154
+ "MM-ARC-007",
155
+ Status.ERROR,
156
+ f"more than {limits.max_entries} entries; the scan stopped early",
157
+ details={"limit": limits.max_entries},
158
+ )
159
+ )
160
+ report.error("MM-SCN-004", "entry count limit reached; results are partial")
161
+ src.index()
162
+ return src
163
+ if _is_link(child):
164
+ links += 1
165
+ src.entries.append(Entry(rel, 0, "symlink"))
166
+ report.add(
167
+ Finding(
168
+ "MM-ARC-020",
169
+ Status.WARNING,
170
+ "link or reparse point recorded and not followed",
171
+ source=rel,
172
+ )
173
+ )
174
+ continue
175
+ try:
176
+ st = child.stat(follow_symlinks=False)
177
+ except OSError as exc:
178
+ report.error("MM-SCN-003", f"could not stat '{rel}': {exc.strerror or exc}")
179
+ continue
180
+ if stat.S_ISDIR(st.st_mode):
181
+ src.entries.append(Entry(rel, 0, "dir"))
182
+ stack.append((child.path, rel + "/"))
183
+ elif stat.S_ISREG(st.st_mode):
184
+ src.entries.append(Entry(rel, st.st_size, "file"))
185
+ else:
186
+ src.entries.append(Entry(rel, 0, "special", readable=False))
187
+ report.add(
188
+ Finding(
189
+ "MM-ARC-006",
190
+ Status.WARNING,
191
+ "special file recorded and never opened",
192
+ source=rel,
193
+ )
194
+ )
195
+ src.index()
196
+ return src
@@ -0,0 +1,302 @@
1
+ """Defensive, read-only ZIP inspection. Nothing is extracted to disk."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import re
7
+ import stat
8
+ import zipfile
9
+ import zlib
10
+ from typing import BinaryIO
11
+
12
+ from modmanifest.archive.limits import Deadline, ScanLimits
13
+ from modmanifest.archive.sources import (
14
+ CorruptEntry,
15
+ Entry,
16
+ EntryTooLarge,
17
+ PackageSource,
18
+ SourceReadError,
19
+ )
20
+ from modmanifest.models import Finding, Report, Status
21
+
22
+ NESTED_ARCHIVE_EXTENSIONS = (".zip", ".7z", ".rar", ".gz", ".tgz", ".tar", ".bz2", ".xz")
23
+ _DRIVE = re.compile(r"^[A-Za-z]:")
24
+ _CHUNK = 64 * 1024
25
+
26
+
27
+ class ZipSource(PackageSource):
28
+ kind = "zip"
29
+
30
+ def __init__(self, target: str | BinaryIO, display: str) -> None:
31
+ super().__init__(display)
32
+ self._zf = zipfile.ZipFile(target)
33
+ self._infos: dict[str, zipfile.ZipInfo] = {}
34
+
35
+ def close(self) -> None:
36
+ self._zf.close()
37
+
38
+ def infolist(self) -> list[zipfile.ZipInfo]:
39
+ return self._zf.infolist()
40
+
41
+ def _read(self, path: str, limit: int) -> bytes:
42
+ entry = self.get(path)
43
+ info = self._infos.get(path)
44
+ if entry is None or info is None or entry.kind != "file" or not entry.readable:
45
+ raise SourceReadError(f"not a readable file entry: {path}")
46
+ chunks: list[bytes] = []
47
+ total = 0
48
+ try:
49
+ with self._zf.open(info) as fh:
50
+ while total <= limit:
51
+ block = fh.read(min(_CHUNK, limit + 1 - total))
52
+ if not block:
53
+ break
54
+ chunks.append(block)
55
+ total += len(block)
56
+ except (zipfile.BadZipFile, zlib.error, EOFError) as exc:
57
+ raise CorruptEntry(str(exc)) from exc
58
+ except (RuntimeError, NotImplementedError) as exc:
59
+ raise SourceReadError(str(exc)) from exc
60
+ return b"".join(chunks)
61
+
62
+ def read_bytes(self, path: str, max_bytes: int) -> bytes:
63
+ data = self._read(path, max_bytes)
64
+ if len(data) > max_bytes:
65
+ raise EntryTooLarge(path)
66
+ return data
67
+
68
+ def read_bytes_prefix(self, path: str, n: int) -> bytes:
69
+ try:
70
+ return self._read(path, n)[:n]
71
+ except CorruptEntry:
72
+ return b""
73
+
74
+ def open_seekable(self, path: str, max_bytes: int) -> BinaryIO:
75
+ return io.BytesIO(self.read_bytes(path, max_bytes))
76
+
77
+
78
+ def _path_problem(raw: str) -> str | None:
79
+ name = raw.replace("\\", "/")
80
+ if "\x00" in raw:
81
+ return "contains a NUL byte"
82
+ if name.startswith("/"):
83
+ return "absolute path"
84
+ if _DRIVE.match(name):
85
+ return "drive-letter path"
86
+ parts = [p for p in name.split("/") if p not in ("", ".")]
87
+ if any(p == ".." for p in parts):
88
+ return "path traversal component"
89
+ if any(":" in p for p in parts):
90
+ return "colon in path component (drive or alternate data stream)"
91
+ return None
92
+
93
+
94
+ def open_zip(
95
+ target: str | BinaryIO,
96
+ display: str,
97
+ limits: ScanLimits,
98
+ deadline: Deadline,
99
+ report: Report,
100
+ *,
101
+ scope: str = "",
102
+ archive_size: int | None = None,
103
+ ) -> ZipSource | None:
104
+ """Open and analyze a ZIP, recording findings. Returns None when it cannot be inspected."""
105
+ label = scope or None
106
+ if archive_size is not None and archive_size > limits.max_archive_bytes:
107
+ report.add(
108
+ Finding(
109
+ "MM-ARC-016",
110
+ Status.ERROR,
111
+ f"archive is {archive_size} bytes; the limit is {limits.max_archive_bytes}",
112
+ source=label,
113
+ )
114
+ )
115
+ report.error("MM-SCN-004", "archive size limit reached; the archive was not opened")
116
+ return None
117
+ try:
118
+ src = ZipSource(target, display)
119
+ except zipfile.BadZipFile as exc:
120
+ report.add(
121
+ Finding("MM-ARC-001", Status.ERROR, f"not a readable ZIP archive: {exc}", source=label)
122
+ )
123
+ return None
124
+ except OSError as exc:
125
+ report.error("MM-SCN-002", f"could not open archive: {exc.strerror or exc}")
126
+ return None
127
+ report.capability("archive.zip.structure")
128
+ infos = src.infolist()
129
+ if len(infos) > limits.max_entries:
130
+ report.add(
131
+ Finding(
132
+ "MM-ARC-007",
133
+ Status.ERROR,
134
+ f"{len(infos)} entries exceed the limit of {limits.max_entries}",
135
+ source=label,
136
+ details={"limit": limits.max_entries, "entries": len(infos)},
137
+ )
138
+ )
139
+ report.error("MM-SCN-004", "entry count limit reached; the archive was not inspected")
140
+ src.close()
141
+ return None
142
+
143
+ def rel(p: str) -> str:
144
+ return f"{scope}!/{p}" if scope else p
145
+
146
+ errors_before = sum(1 for f in report.findings if f.status is Status.ERROR)
147
+ seen: dict[str, int] = {}
148
+ folded: dict[str, str] = {}
149
+ total = 0
150
+ total_flagged = False
151
+ for i, info in enumerate(infos):
152
+ if i % 500 == 0:
153
+ deadline.check()
154
+ raw = info.orig_filename
155
+ if "\\" in raw:
156
+ report.add(
157
+ Finding(
158
+ "MM-ARC-015",
159
+ Status.WARNING,
160
+ "entry name uses a backslash separator; treated as '/'",
161
+ source=rel(raw),
162
+ )
163
+ )
164
+ norm = raw.replace("\\", "/")
165
+ problem = _path_problem(raw)
166
+ if problem:
167
+ report.add(
168
+ Finding(
169
+ "MM-ARC-002", Status.ERROR, f"unsafe entry path: {problem}", source=rel(raw)
170
+ )
171
+ )
172
+ src.entries.append(Entry(norm, info.file_size, "unsafe", readable=False))
173
+ continue
174
+ is_dir = info.is_dir()
175
+ path = norm.rstrip("/") if is_dir else norm
176
+ kind = "dir" if is_dir else "file"
177
+ readable = True
178
+ mode = info.external_attr >> 16
179
+ if info.create_system == 3 and not is_dir and mode:
180
+ if stat.S_ISLNK(mode):
181
+ kind, readable = "symlink", False
182
+ report.add(
183
+ Finding(
184
+ "MM-ARC-005",
185
+ Status.WARNING,
186
+ "symbolic link entry recorded and never resolved",
187
+ source=rel(path),
188
+ )
189
+ )
190
+ elif (
191
+ stat.S_ISCHR(mode)
192
+ or stat.S_ISBLK(mode)
193
+ or stat.S_ISFIFO(mode)
194
+ or stat.S_ISSOCK(mode)
195
+ ):
196
+ kind, readable = "special", False
197
+ report.add(
198
+ Finding(
199
+ "MM-ARC-006",
200
+ Status.ERROR,
201
+ "special file entry (device, pipe or socket)",
202
+ source=rel(path),
203
+ )
204
+ )
205
+ seen[path] = seen.get(path, 0) + 1
206
+ if seen[path] == 2:
207
+ report.add(
208
+ Finding(
209
+ "MM-ARC-003",
210
+ Status.ERROR,
211
+ "entry name appears more than once; the entry is ambiguous and was not read",
212
+ source=rel(path),
213
+ )
214
+ )
215
+ key = path.casefold()
216
+ if key in folded and folded[key] != path:
217
+ report.add(
218
+ Finding(
219
+ "MM-ARC-004",
220
+ Status.WARNING,
221
+ f"differs only by case from '{folded[key]}'",
222
+ source=rel(path),
223
+ )
224
+ )
225
+ folded.setdefault(key, path)
226
+ if kind == "file":
227
+ total += info.file_size
228
+ if info.flag_bits & 0x1:
229
+ readable = False
230
+ report.add(
231
+ Finding(
232
+ "MM-ARC-011",
233
+ Status.UNSUPPORTED,
234
+ "encrypted entry; contents were not read",
235
+ source=rel(path),
236
+ )
237
+ )
238
+ if info.file_size > limits.max_entry_bytes:
239
+ readable = False
240
+ report.add(
241
+ Finding(
242
+ "MM-ARC-009",
243
+ Status.ERROR,
244
+ f"declared size {info.file_size} exceeds the entry limit",
245
+ source=rel(path),
246
+ details={"limit": limits.max_entry_bytes},
247
+ )
248
+ )
249
+ if (
250
+ info.file_size >= limits.ratio_min_bytes
251
+ and info.file_size / max(info.compress_size, 1) > limits.max_ratio
252
+ ):
253
+ report.add(
254
+ Finding(
255
+ "MM-ARC-010",
256
+ Status.WARNING,
257
+ f"compression ratio above {limits.max_ratio:g}:1",
258
+ source=rel(path),
259
+ details={"size": info.file_size, "compressed": info.compress_size},
260
+ )
261
+ )
262
+ if path.lower().endswith(NESTED_ARCHIVE_EXTENSIONS):
263
+ report.add(
264
+ Finding(
265
+ "MM-ARC-012",
266
+ Status.UNSUPPORTED,
267
+ "nested archive listed but not opened",
268
+ source=rel(path),
269
+ )
270
+ )
271
+ if total > limits.max_total_bytes and not total_flagged:
272
+ total_flagged = True
273
+ report.add(
274
+ Finding(
275
+ "MM-ARC-008",
276
+ Status.ERROR,
277
+ f"declared expanded size exceeds {limits.max_total_bytes} bytes",
278
+ source=label,
279
+ details={"limit": limits.max_total_bytes},
280
+ )
281
+ )
282
+ src.entries.append(Entry(path, info.file_size, kind, info.compress_size, readable))
283
+ src._infos.setdefault(path, info)
284
+ ambiguous = {p for p, n in seen.items() if n > 1}
285
+ for entry in src.entries:
286
+ if entry.path in ambiguous:
287
+ entry.readable = False
288
+ if total_flagged:
289
+ for entry in src.entries:
290
+ entry.readable = False
291
+ src.index()
292
+ errors_after = sum(1 for f in report.findings if f.status is Status.ERROR)
293
+ if errors_after == errors_before:
294
+ report.add(
295
+ Finding(
296
+ "MM-ARC-014",
297
+ Status.PASS,
298
+ f"{len(infos)} entries checked for path safety, duplicates, links, sizes, ratios",
299
+ source=label,
300
+ )
301
+ )
302
+ return src