contextlock 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentbom/__init__.py +0 -0
- agentbom/__main__.py +3 -0
- agentbom/cli.py +221 -0
- agentbom/discover.py +337 -0
- agentbom/gitindex.py +119 -0
- agentbom/lockfile.py +261 -0
- agentbom/models.py +180 -0
- agentbom/parsers/__init__.py +53 -0
- agentbom/parsers/_shape.py +96 -0
- agentbom/parsers/hooks.py +146 -0
- agentbom/parsers/jsonc.py +35 -0
- agentbom/parsers/markdown.py +111 -0
- agentbom/parsers/mcp.py +94 -0
- agentbom/parsers/script.py +15 -0
- agentbom/parsers/settings.py +47 -0
- agentbom/policy.py +178 -0
- agentbom/py.typed +0 -0
- agentbom/registry.py +120 -0
- agentbom/report.py +237 -0
- agentbom/rules/__init__.py +51 -0
- agentbom/rules/_prose.py +109 -0
- agentbom/rules/_surfaces.py +92 -0
- agentbom/rules/abom_001_hidden_unicode.py +232 -0
- agentbom/rules/abom_002_remote_exec.py +148 -0
- agentbom/rules/abom_003_credential_paths.py +200 -0
- agentbom/rules/abom_004_outbound_transfer.py +219 -0
- agentbom/rules/abom_005_unpinned.py +399 -0
- agentbom/rules/abom_006_drift.py +52 -0
- agentbom/rules/abom_007_permissions.py +192 -0
- agentbom/rules/abom_008_auto_run_hooks.py +160 -0
- agentbom/rules/abom_009_instruction_override.py +166 -0
- agentbom/rules/abom_010_secrets.py +203 -0
- agentbom/sarif.py +142 -0
- agentbom/scan.py +35 -0
- contextlock-0.1.0.dist-info/METADATA +238 -0
- contextlock-0.1.0.dist-info/RECORD +39 -0
- contextlock-0.1.0.dist-info/WHEEL +4 -0
- contextlock-0.1.0.dist-info/entry_points.txt +3 -0
- contextlock-0.1.0.dist-info/licenses/LICENSE +201 -0
agentbom/__init__.py
ADDED
|
File without changes
|
agentbom/__main__.py
ADDED
agentbom/cli.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import stat
|
|
3
|
+
import sys
|
|
4
|
+
import tempfile
|
|
5
|
+
from datetime import date
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated, NoReturn
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
|
|
13
|
+
from agentbom.discover import read_source
|
|
14
|
+
from agentbom.lockfile import (
|
|
15
|
+
LOCKFILE_NAME,
|
|
16
|
+
LockError,
|
|
17
|
+
Lockfile,
|
|
18
|
+
build_lock,
|
|
19
|
+
parse_lock,
|
|
20
|
+
render_lock,
|
|
21
|
+
)
|
|
22
|
+
from agentbom.models import SEVERITY_RANK, ScanError, ScanResult, Severity
|
|
23
|
+
from agentbom.policy import POLICY_FILENAME, Policy, PolicyError, apply_policy, parse_policy
|
|
24
|
+
from agentbom.report import print_errors, print_terminal, render_json, render_markdown
|
|
25
|
+
from agentbom.sarif import render_sarif
|
|
26
|
+
from agentbom.scan import scan
|
|
27
|
+
|
|
28
|
+
app = typer.Typer(add_completion=False, no_args_is_help=True)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class OutputFormat(StrEnum):
|
|
32
|
+
terminal = "terminal"
|
|
33
|
+
json = "json"
|
|
34
|
+
sarif = "sarif"
|
|
35
|
+
markdown = "markdown"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class FailOn(StrEnum):
|
|
39
|
+
high = "high"
|
|
40
|
+
medium = "medium"
|
|
41
|
+
low = "low"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
_THRESHOLDS: dict[FailOn, Severity] = {
|
|
45
|
+
FailOn.high: "high",
|
|
46
|
+
FailOn.medium: "medium",
|
|
47
|
+
FailOn.low: "low",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
PathArgument = Annotated[Path, typer.Argument(help="Repository root to scan.")]
|
|
51
|
+
FormatOption = Annotated[OutputFormat, typer.Option("--format", help="Output format.")]
|
|
52
|
+
FailOnOption = Annotated[
|
|
53
|
+
FailOn | None,
|
|
54
|
+
typer.Option(
|
|
55
|
+
"--fail-on",
|
|
56
|
+
help="Exit 1 when a finding has this severity or higher. "
|
|
57
|
+
"Overrides fail_on in the policy file; the default is high.",
|
|
58
|
+
),
|
|
59
|
+
]
|
|
60
|
+
PolicyOption = Annotated[
|
|
61
|
+
Path | None,
|
|
62
|
+
typer.Option(
|
|
63
|
+
"--policy",
|
|
64
|
+
help=f"Policy file to apply. The default is PATH/{POLICY_FILENAME} when it exists.",
|
|
65
|
+
),
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@app.callback()
|
|
70
|
+
def main() -> None:
|
|
71
|
+
"""Inventory the agent context a repository loads and flag risky patterns in it."""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@app.command("scan")
|
|
75
|
+
def scan_command(
|
|
76
|
+
path: PathArgument = Path(),
|
|
77
|
+
output_format: FormatOption = OutputFormat.terminal,
|
|
78
|
+
fail_on: FailOnOption = None,
|
|
79
|
+
policy_file: PolicyOption = None,
|
|
80
|
+
) -> None:
|
|
81
|
+
"""List the agent context in a repository and report findings."""
|
|
82
|
+
_require_directory(path)
|
|
83
|
+
_run("scan", path, None, output_format, fail_on, _load_policy(path, policy_file))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@app.command("lock")
|
|
87
|
+
def lock_command(path: PathArgument = Path()) -> None:
|
|
88
|
+
"""Record the definition of every artifact in agentbom.lock."""
|
|
89
|
+
_require_directory(path)
|
|
90
|
+
result = scan(path)
|
|
91
|
+
if result.errors:
|
|
92
|
+
print_errors(result, Console(stderr=True))
|
|
93
|
+
_fail(f"{LOCKFILE_NAME} not written, because a partial scan cannot be trusted")
|
|
94
|
+
target = path / LOCKFILE_NAME
|
|
95
|
+
mode = _lstat_mode(target)
|
|
96
|
+
if mode is not None and not stat.S_ISREG(mode):
|
|
97
|
+
_fail(f"{LOCKFILE_NAME} is not a regular file")
|
|
98
|
+
try:
|
|
99
|
+
lock = build_lock(result.artifacts, result.file_digests)
|
|
100
|
+
except ValueError as error:
|
|
101
|
+
_fail(f"{LOCKFILE_NAME} not written: {error}")
|
|
102
|
+
_replace_file(target, render_lock(lock))
|
|
103
|
+
count = len(result.artifacts)
|
|
104
|
+
typer.echo(f"Locked {count} artifact{'' if count == 1 else 's'} in {LOCKFILE_NAME}.")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@app.command("check")
|
|
108
|
+
def check_command(
|
|
109
|
+
path: PathArgument = Path(),
|
|
110
|
+
output_format: FormatOption = OutputFormat.terminal,
|
|
111
|
+
fail_on: FailOnOption = None,
|
|
112
|
+
policy_file: PolicyOption = None,
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Report findings and any drift from agentbom.lock, which always fails the check."""
|
|
115
|
+
_require_directory(path)
|
|
116
|
+
policy = _load_policy(path, policy_file)
|
|
117
|
+
_run("check", path, _read_lock(path), output_format, fail_on, policy)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _run(
|
|
121
|
+
command: str,
|
|
122
|
+
path: Path,
|
|
123
|
+
lock: Lockfile | None,
|
|
124
|
+
output_format: OutputFormat,
|
|
125
|
+
fail_on: FailOn | None,
|
|
126
|
+
policy: Policy | None,
|
|
127
|
+
) -> NoReturn:
|
|
128
|
+
result = scan(path, lock)
|
|
129
|
+
if policy is not None:
|
|
130
|
+
result = apply_policy(policy, result, date.today())
|
|
131
|
+
_report(result, output_format, command=command, drift=lock is not None)
|
|
132
|
+
if fail_on is not None:
|
|
133
|
+
threshold = _THRESHOLDS[fail_on]
|
|
134
|
+
else:
|
|
135
|
+
threshold = "high" if policy is None else policy.fail_on
|
|
136
|
+
raise typer.Exit(exit_code(result, threshold))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _require_directory(path: Path) -> None:
|
|
140
|
+
if not path.is_dir():
|
|
141
|
+
_fail(f"{path} is not a directory")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _read_lock(root: Path) -> Lockfile:
|
|
145
|
+
file = root / LOCKFILE_NAME
|
|
146
|
+
if _lstat_mode(file) is None:
|
|
147
|
+
_fail(f"no {LOCKFILE_NAME} in {root}; run `agentbom lock` first")
|
|
148
|
+
text = read_source(file, LOCKFILE_NAME)
|
|
149
|
+
if isinstance(text, ScanError):
|
|
150
|
+
_fail(f"{text.path}: {text.message}")
|
|
151
|
+
try:
|
|
152
|
+
return parse_lock(text)
|
|
153
|
+
except LockError as error:
|
|
154
|
+
_fail(f"{LOCKFILE_NAME}: {error}")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _load_policy(root: Path, explicit: Path | None) -> Policy | None:
|
|
158
|
+
file = root / POLICY_FILENAME if explicit is None else explicit
|
|
159
|
+
if explicit is None and _lstat_mode(file) is None:
|
|
160
|
+
return None
|
|
161
|
+
text = read_source(file, file.name)
|
|
162
|
+
if isinstance(text, ScanError):
|
|
163
|
+
_fail(f"{text.path}: {text.message}")
|
|
164
|
+
try:
|
|
165
|
+
return parse_policy(text)
|
|
166
|
+
except PolicyError as error:
|
|
167
|
+
_fail(f"{file.name}: {error}")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _report(result: ScanResult, output_format: OutputFormat, *, command: str, drift: bool) -> None:
|
|
171
|
+
match output_format:
|
|
172
|
+
case OutputFormat.terminal:
|
|
173
|
+
print_terminal(result, Console(), Console(stderr=True), drift=drift)
|
|
174
|
+
return
|
|
175
|
+
case OutputFormat.json:
|
|
176
|
+
text = render_json(result)
|
|
177
|
+
case OutputFormat.sarif:
|
|
178
|
+
text = render_sarif(result)
|
|
179
|
+
case OutputFormat.markdown:
|
|
180
|
+
text = render_markdown(result, command=command, drift=drift)
|
|
181
|
+
sys.stdout.buffer.write(text.encode("utf-8"))
|
|
182
|
+
sys.stdout.flush()
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _replace_file(target: Path, text: str) -> None:
|
|
186
|
+
# A new file replaces the old one, so a crash never leaves half a lockfile and the
|
|
187
|
+
# write never goes through a link.
|
|
188
|
+
temporary: Path | None = None
|
|
189
|
+
try:
|
|
190
|
+
handle, name = tempfile.mkstemp(dir=target.parent, prefix=f".{target.name}.")
|
|
191
|
+
temporary = Path(name)
|
|
192
|
+
with os.fdopen(handle, "wb") as file:
|
|
193
|
+
file.write(text.encode("utf-8"))
|
|
194
|
+
temporary.chmod(0o644)
|
|
195
|
+
temporary.replace(target)
|
|
196
|
+
except OSError as error:
|
|
197
|
+
if temporary is not None:
|
|
198
|
+
temporary.unlink(missing_ok=True)
|
|
199
|
+
_fail(f"cannot write {target.name}: {error.strerror or type(error).__name__}")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _lstat_mode(file: Path) -> int | None:
|
|
203
|
+
try:
|
|
204
|
+
return file.lstat().st_mode
|
|
205
|
+
except FileNotFoundError:
|
|
206
|
+
return None
|
|
207
|
+
except OSError as error:
|
|
208
|
+
_fail(f"cannot read {file.name}: {error.strerror or type(error).__name__}")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _fail(message: str) -> NoReturn:
|
|
212
|
+
typer.echo(f"error: {message}", err=True)
|
|
213
|
+
raise typer.Exit(2)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def exit_code(result: ScanResult, fail_on: Severity) -> int:
|
|
217
|
+
# A partial scan cannot be trusted, so a tool error outranks any finding.
|
|
218
|
+
if result.errors:
|
|
219
|
+
return 2
|
|
220
|
+
threshold = SEVERITY_RANK[fail_on]
|
|
221
|
+
return 1 if any(SEVERITY_RANK[f.severity] >= threshold for f in result.findings) else 0
|
agentbom/discover.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""Find the agent files in a repository: the files git tracks, or every file outside git.
|
|
2
|
+
|
|
3
|
+
In a git checkout (a `.git` at the root), the files come from `.git/index`, so a file is
|
|
4
|
+
scanned exactly when it is tracked, whatever `.gitignore` or `.git/info/exclude` say.
|
|
5
|
+
Outside git, every file under the root is scanned.
|
|
6
|
+
|
|
7
|
+
A symlink is followed through the tracked paths, never on disk, and the files it leads to
|
|
8
|
+
are scanned at their real path. A symlink that cannot be followed (it dangles, is
|
|
9
|
+
absolute, leaves the repository or enters a submodule) is a scan error when its own path
|
|
10
|
+
is an agent file. Otherwise it may be a directory hiding agent files, so its target is
|
|
11
|
+
locked: adding or retargeting it is drift.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import bisect
|
|
15
|
+
import codecs
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import stat
|
|
19
|
+
import sys
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from pathlib import Path, PurePath
|
|
22
|
+
|
|
23
|
+
from agentbom.gitindex import (
|
|
24
|
+
DIRECTORY,
|
|
25
|
+
GITLINK,
|
|
26
|
+
REGULAR,
|
|
27
|
+
SYMLINK,
|
|
28
|
+
GitIndexError,
|
|
29
|
+
hash_size,
|
|
30
|
+
object_type,
|
|
31
|
+
parse_index,
|
|
32
|
+
)
|
|
33
|
+
from agentbom.models import ScanError
|
|
34
|
+
from agentbom.registry import SourcePattern, match_source
|
|
35
|
+
|
|
36
|
+
MAX_FILE_BYTES = 2 * 1024 * 1024
|
|
37
|
+
_MAX_LINK_HOPS = 40
|
|
38
|
+
_MAX_LINK_DEPTH = 8
|
|
39
|
+
_MAX_LINKED_FILES = 100_000
|
|
40
|
+
_ABSOLUTE = re.compile(r"^(/|[A-Za-z]:)")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class SourceFile:
|
|
45
|
+
path: str
|
|
46
|
+
text: str
|
|
47
|
+
pattern: SourcePattern
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class Discovery:
|
|
52
|
+
sources: list[SourceFile]
|
|
53
|
+
# Each symlink that leads an agent to a scanned file, with its target.
|
|
54
|
+
links: dict[str, str]
|
|
55
|
+
errors: list[ScanError]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class _Unresolved(Exception):
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class _Tree:
|
|
64
|
+
"""The paths in a repository: regular files, symlinks and their targets, submodules."""
|
|
65
|
+
|
|
66
|
+
files: set[str] = field(default_factory=set)
|
|
67
|
+
links: dict[str, str] = field(default_factory=dict)
|
|
68
|
+
submodules: set[str] = field(default_factory=set)
|
|
69
|
+
_sorted: list[str] = field(default_factory=list)
|
|
70
|
+
|
|
71
|
+
def freeze(self) -> None:
|
|
72
|
+
self._sorted = sorted(self.files | self.links.keys() | self.submodules)
|
|
73
|
+
|
|
74
|
+
def under(self, directory: str) -> list[str]:
|
|
75
|
+
if not directory:
|
|
76
|
+
return self._sorted
|
|
77
|
+
prefix = directory + "/"
|
|
78
|
+
start = bisect.bisect_left(self._sorted, prefix)
|
|
79
|
+
end = start
|
|
80
|
+
while end < len(self._sorted) and self._sorted[end].startswith(prefix):
|
|
81
|
+
end += 1
|
|
82
|
+
return self._sorted[start:end]
|
|
83
|
+
|
|
84
|
+
def follow(self, path: str) -> str:
|
|
85
|
+
"""The path with every symlink in it replaced by its target, as the OS would."""
|
|
86
|
+
resolved: list[str] = []
|
|
87
|
+
pending = list(reversed(path.split("/")))
|
|
88
|
+
hops = 0
|
|
89
|
+
while pending:
|
|
90
|
+
part = pending.pop()
|
|
91
|
+
if part in ("", "."):
|
|
92
|
+
continue
|
|
93
|
+
if part == "..":
|
|
94
|
+
if not resolved:
|
|
95
|
+
raise _Unresolved("points outside the repository")
|
|
96
|
+
resolved.pop()
|
|
97
|
+
continue
|
|
98
|
+
resolved.append(part)
|
|
99
|
+
current = "/".join(resolved)
|
|
100
|
+
if current in self.submodules:
|
|
101
|
+
raise _Unresolved("points into a submodule, which is not scanned")
|
|
102
|
+
target = self.links.get(current)
|
|
103
|
+
if target is None:
|
|
104
|
+
continue
|
|
105
|
+
hops += 1
|
|
106
|
+
if hops > _MAX_LINK_HOPS:
|
|
107
|
+
raise _Unresolved("goes through too many links")
|
|
108
|
+
if _ABSOLUTE.match(target):
|
|
109
|
+
raise _Unresolved("has an absolute target")
|
|
110
|
+
resolved.pop()
|
|
111
|
+
pending.extend(reversed(target.split("/")))
|
|
112
|
+
return "/".join(resolved)
|
|
113
|
+
|
|
114
|
+
def expand(self, link: str, virtual: str, depth: int, out: list[tuple[str, str]]) -> None:
|
|
115
|
+
"""Add (path an agent sees, real file) for every file reached through `link`."""
|
|
116
|
+
if depth > _MAX_LINK_DEPTH:
|
|
117
|
+
raise _Unresolved("nests links too deeply")
|
|
118
|
+
real = self.follow(link)
|
|
119
|
+
if real in self.files:
|
|
120
|
+
out.append((virtual, real))
|
|
121
|
+
return
|
|
122
|
+
children = self.under(real)
|
|
123
|
+
if not children:
|
|
124
|
+
raise _Unresolved("points to nothing in the repository")
|
|
125
|
+
if not real or link.startswith(real + "/"):
|
|
126
|
+
raise _Unresolved("points to a directory that contains it")
|
|
127
|
+
for child in children:
|
|
128
|
+
seen = f"{virtual}/{child[len(real) + 1 :]}"
|
|
129
|
+
if child in self.files:
|
|
130
|
+
out.append((seen, child))
|
|
131
|
+
elif child in self.links:
|
|
132
|
+
self.expand(child, seen, depth + 1, out)
|
|
133
|
+
if len(out) > _MAX_LINKED_FILES:
|
|
134
|
+
raise _Unresolved(f"leads to more than {_MAX_LINKED_FILES} files")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def discover(root: Path) -> Discovery:
|
|
138
|
+
if not root.is_dir():
|
|
139
|
+
raise NotADirectoryError(root)
|
|
140
|
+
errors: list[ScanError] = []
|
|
141
|
+
tree = _git_tree(root, errors) if _lexists(root / ".git") else _walk_tree(root, errors)
|
|
142
|
+
tree.freeze()
|
|
143
|
+
patterns: dict[str, SourcePattern] = {}
|
|
144
|
+
for path in sorted(tree.files):
|
|
145
|
+
pattern = match_source(path)
|
|
146
|
+
if pattern is not None:
|
|
147
|
+
patterns[path] = pattern
|
|
148
|
+
links: dict[str, str] = {}
|
|
149
|
+
for link in sorted(tree.links):
|
|
150
|
+
reached: list[tuple[str, str]] = []
|
|
151
|
+
try:
|
|
152
|
+
tree.expand(link, link, 0, reached)
|
|
153
|
+
except _Unresolved as problem:
|
|
154
|
+
if match_source(link) is not None:
|
|
155
|
+
errors.append(ScanError(path=link, message=f"symlink {problem}"))
|
|
156
|
+
else:
|
|
157
|
+
links[link] = tree.links[link]
|
|
158
|
+
continue
|
|
159
|
+
for seen, real in sorted(reached):
|
|
160
|
+
pattern = match_source(seen)
|
|
161
|
+
if pattern is not None:
|
|
162
|
+
links[link] = tree.links[link]
|
|
163
|
+
patterns.setdefault(real, pattern)
|
|
164
|
+
sources: list[SourceFile] = []
|
|
165
|
+
on_disk = _DiskCheck(root)
|
|
166
|
+
for path, pattern in sorted(patterns.items()):
|
|
167
|
+
text = on_disk.read(path)
|
|
168
|
+
if isinstance(text, ScanError):
|
|
169
|
+
errors.append(text)
|
|
170
|
+
else:
|
|
171
|
+
sources.append(SourceFile(path=path, text=text, pattern=pattern))
|
|
172
|
+
return Discovery(sources=sources, links=links, errors=errors)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _git_tree(root: Path, errors: list[ScanError]) -> _Tree:
|
|
176
|
+
tree = _Tree()
|
|
177
|
+
try:
|
|
178
|
+
git_dir = _git_dir(root)
|
|
179
|
+
common = git_dir
|
|
180
|
+
if (git_dir / "commondir").is_file():
|
|
181
|
+
common = git_dir / (git_dir / "commondir").read_text(encoding="utf-8").strip()
|
|
182
|
+
config = common / "config"
|
|
183
|
+
size = hash_size(config.read_text(encoding="utf-8")) if config.is_file() else 20
|
|
184
|
+
index = git_dir / "index"
|
|
185
|
+
entries = parse_index(index.read_bytes(), size) if index.is_file() else {}
|
|
186
|
+
except (OSError, UnicodeDecodeError, GitIndexError) as error:
|
|
187
|
+
message = _os_message(error) if isinstance(error, OSError) else str(error)
|
|
188
|
+
errors.append(ScanError(path=".git", message=message))
|
|
189
|
+
return tree
|
|
190
|
+
for path, mode in entries.items():
|
|
191
|
+
kind = object_type(mode)
|
|
192
|
+
if kind == REGULAR:
|
|
193
|
+
tree.files.add(path)
|
|
194
|
+
elif kind == SYMLINK:
|
|
195
|
+
target = _link_target(root, path, errors)
|
|
196
|
+
if target is not None:
|
|
197
|
+
tree.links[path] = target
|
|
198
|
+
elif kind == GITLINK:
|
|
199
|
+
tree.submodules.add(path)
|
|
200
|
+
elif kind == DIRECTORY:
|
|
201
|
+
errors.append(ScanError(path=path, message="a sparse directory is not checked out"))
|
|
202
|
+
return tree
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _git_dir(root: Path) -> Path:
|
|
206
|
+
dot_git = root / ".git"
|
|
207
|
+
mode = dot_git.lstat().st_mode
|
|
208
|
+
if stat.S_ISDIR(mode):
|
|
209
|
+
return dot_git
|
|
210
|
+
if not stat.S_ISREG(mode):
|
|
211
|
+
raise GitIndexError(".git is neither a directory nor a file")
|
|
212
|
+
# A worktree or submodule checkout has a `.git` file naming its git directory.
|
|
213
|
+
pointer = dot_git.read_text(encoding="utf-8").strip()
|
|
214
|
+
if not pointer.startswith("gitdir:"):
|
|
215
|
+
raise GitIndexError(".git is a file without a gitdir line")
|
|
216
|
+
return root / pointer.removeprefix("gitdir:").strip()
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _link_target(root: Path, path: str, errors: list[ScanError]) -> str | None:
|
|
220
|
+
# git stores a symlink's target as its content; with core.symlinks off the checkout
|
|
221
|
+
# writes it as a plain file, and otherwise the OS returns it.
|
|
222
|
+
file = root / path
|
|
223
|
+
try:
|
|
224
|
+
if stat.S_ISLNK(file.lstat().st_mode):
|
|
225
|
+
return file.readlink().as_posix()
|
|
226
|
+
return file.read_bytes().decode("utf-8")
|
|
227
|
+
except (OSError, UnicodeDecodeError) as error:
|
|
228
|
+
message = _os_message(error) if isinstance(error, OSError) else "target is not UTF-8"
|
|
229
|
+
errors.append(ScanError(path=path, message=f"symlink {message}"))
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _walk_tree(root: Path, errors: list[ScanError]) -> _Tree:
|
|
234
|
+
tree = _Tree()
|
|
235
|
+
|
|
236
|
+
def relative(path: str) -> str:
|
|
237
|
+
return PurePath(os.path.relpath(path, root)).as_posix()
|
|
238
|
+
|
|
239
|
+
def walk_error(error: OSError) -> None:
|
|
240
|
+
where = relative(error.filename) if error.filename else "."
|
|
241
|
+
errors.append(ScanError(path=where, message=_os_message(error)))
|
|
242
|
+
|
|
243
|
+
for current, dirnames, filenames in os.walk(root, onerror=walk_error, followlinks=False):
|
|
244
|
+
here = Path(current)
|
|
245
|
+
directory = "" if here == root else relative(current)
|
|
246
|
+
directories: list[str] = []
|
|
247
|
+
for name in dirnames:
|
|
248
|
+
if name == ".git":
|
|
249
|
+
continue
|
|
250
|
+
if _is_link(here / name):
|
|
251
|
+
_add_disk_link(tree, here / name, _join(directory, name), errors)
|
|
252
|
+
else:
|
|
253
|
+
directories.append(name)
|
|
254
|
+
dirnames[:] = sorted(directories)
|
|
255
|
+
for name in filenames:
|
|
256
|
+
path = _join(directory, name)
|
|
257
|
+
if _is_link(here / name):
|
|
258
|
+
_add_disk_link(tree, here / name, path, errors)
|
|
259
|
+
else:
|
|
260
|
+
tree.files.add(path)
|
|
261
|
+
return tree
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _add_disk_link(tree: _Tree, file: Path, path: str, errors: list[ScanError]) -> None:
|
|
265
|
+
try:
|
|
266
|
+
tree.links[path] = file.readlink().as_posix()
|
|
267
|
+
except OSError as error:
|
|
268
|
+
errors.append(ScanError(path=path, message=f"symlink {_os_message(error)}"))
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class _DiskCheck:
|
|
272
|
+
"""Reads tracked files, refusing any whose path on disk goes through a link."""
|
|
273
|
+
|
|
274
|
+
def __init__(self, root: Path) -> None:
|
|
275
|
+
self._root = root
|
|
276
|
+
self._linked: dict[str, bool] = {}
|
|
277
|
+
|
|
278
|
+
def read(self, path: str) -> str | ScanError:
|
|
279
|
+
parts = path.split("/")
|
|
280
|
+
for depth in range(1, len(parts)):
|
|
281
|
+
directory = "/".join(parts[:depth])
|
|
282
|
+
if directory not in self._linked:
|
|
283
|
+
self._linked[directory] = _is_link(self._root / directory)
|
|
284
|
+
if self._linked[directory]:
|
|
285
|
+
return ScanError(path=path, message=f"{directory} is a link on disk")
|
|
286
|
+
return read_source(self._root / path, path)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _decode_text(data: bytes) -> str:
|
|
290
|
+
text = data.removeprefix(codecs.BOM_UTF8).decode("utf-8")
|
|
291
|
+
return text.replace("\r\n", "\n").replace("\r", "\n")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def read_source(file: Path, path: str) -> str | ScanError:
|
|
295
|
+
try:
|
|
296
|
+
if not stat.S_ISREG(file.lstat().st_mode):
|
|
297
|
+
return ScanError(path=path, message="not a regular file")
|
|
298
|
+
with file.open("rb") as handle:
|
|
299
|
+
data = handle.read(MAX_FILE_BYTES + 1)
|
|
300
|
+
except OSError as error:
|
|
301
|
+
return ScanError(path=path, message=_os_message(error))
|
|
302
|
+
if len(data) > MAX_FILE_BYTES:
|
|
303
|
+
return ScanError(path=path, message="file is larger than the 2 MiB limit")
|
|
304
|
+
try:
|
|
305
|
+
return _decode_text(data)
|
|
306
|
+
except UnicodeDecodeError as error:
|
|
307
|
+
return ScanError(path=path, message=f"not valid UTF-8 (byte {error.start})")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _os_message(error: OSError) -> str:
|
|
311
|
+
# str(error) embeds the absolute filename, and output must stay repo-relative.
|
|
312
|
+
return f"cannot read: {error.strerror or type(error).__name__}"
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _lexists(path: Path) -> bool:
|
|
316
|
+
try:
|
|
317
|
+
path.lstat()
|
|
318
|
+
except FileNotFoundError:
|
|
319
|
+
return False
|
|
320
|
+
return True
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _is_link(path: Path) -> bool:
|
|
324
|
+
try:
|
|
325
|
+
info = path.lstat()
|
|
326
|
+
except OSError:
|
|
327
|
+
return False
|
|
328
|
+
if stat.S_ISLNK(info.st_mode):
|
|
329
|
+
return True
|
|
330
|
+
if sys.platform == "win32":
|
|
331
|
+
# Windows junctions are not symlinks to Python 3.11, yet os.walk follows them.
|
|
332
|
+
return info.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT
|
|
333
|
+
return False
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _join(directory: str, name: str) -> str:
|
|
337
|
+
return f"{directory}/{name}" if directory else name
|
agentbom/gitindex.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The paths and modes a git index records, read from its bytes without running git."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import struct
|
|
5
|
+
|
|
6
|
+
# The object type in the top four bits of an index entry's mode.
|
|
7
|
+
_TYPE_SHIFT = 12
|
|
8
|
+
REGULAR = 0o10
|
|
9
|
+
SYMLINK = 0o12
|
|
10
|
+
GITLINK = 0o16
|
|
11
|
+
DIRECTORY = 0o04
|
|
12
|
+
|
|
13
|
+
_SHA1_SIZE = 20
|
|
14
|
+
_SHA256_SIZE = 32
|
|
15
|
+
_HEADER = struct.Struct(">4sII")
|
|
16
|
+
_EXTENSION = struct.Struct(">4sI")
|
|
17
|
+
_EXTENDED_FLAG = 0x4000
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class GitIndexError(ValueError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def object_type(mode: int) -> int:
|
|
25
|
+
return mode >> _TYPE_SHIFT
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def hash_size(config: str) -> int:
|
|
29
|
+
"""The object hash size in bytes, from the text of the repository's `config` file."""
|
|
30
|
+
section = ""
|
|
31
|
+
for raw in config.split("\n"):
|
|
32
|
+
line = raw.strip()
|
|
33
|
+
if line.startswith("["):
|
|
34
|
+
section = line[1 : line.find("]")].strip().lower()
|
|
35
|
+
continue
|
|
36
|
+
key, _, value = line.partition("=")
|
|
37
|
+
if section == "extensions" and key.strip().lower() == "objectformat":
|
|
38
|
+
return _SHA256_SIZE if value.strip().lower() == "sha256" else _SHA1_SIZE
|
|
39
|
+
return _SHA1_SIZE
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_index(data: bytes, hash_bytes: int) -> dict[str, int]:
|
|
43
|
+
"""Each path in the index with its mode. Versions 2, 3 and 4 are supported."""
|
|
44
|
+
try:
|
|
45
|
+
return _parse(data, hash_bytes)
|
|
46
|
+
except (IndexError, struct.error) as error:
|
|
47
|
+
raise GitIndexError("the index is truncated") from error
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _parse(data: bytes, hash_bytes: int) -> dict[str, int]:
|
|
51
|
+
signature, version, count = _HEADER.unpack_from(data)
|
|
52
|
+
if signature != b"DIRC":
|
|
53
|
+
raise GitIndexError("not a git index")
|
|
54
|
+
if version not in (2, 3, 4):
|
|
55
|
+
raise GitIndexError(f"index version {version} is not supported")
|
|
56
|
+
body, trailer = data[:-hash_bytes], data[-hash_bytes:]
|
|
57
|
+
digest = hashlib.sha256 if hash_bytes == _SHA256_SIZE else hashlib.sha1
|
|
58
|
+
# index.skipHash writes a trailer of zeros instead of the checksum.
|
|
59
|
+
if any(trailer) and digest(body).digest() != trailer:
|
|
60
|
+
raise GitIndexError("the index checksum does not match")
|
|
61
|
+
fixed = 40 + hash_bytes + 2
|
|
62
|
+
entries: dict[str, int] = {}
|
|
63
|
+
offset = _HEADER.size
|
|
64
|
+
previous = b""
|
|
65
|
+
for _ in range(count):
|
|
66
|
+
mode = struct.unpack_from(">I", body, offset + 24)[0]
|
|
67
|
+
flags = struct.unpack_from(">H", body, offset + 40 + hash_bytes)[0]
|
|
68
|
+
cursor = offset + fixed
|
|
69
|
+
if flags & _EXTENDED_FLAG:
|
|
70
|
+
if version < 3:
|
|
71
|
+
raise GitIndexError("an extended entry in a version 2 index")
|
|
72
|
+
cursor += 2
|
|
73
|
+
if version == 4:
|
|
74
|
+
strip, cursor = _varint(body, cursor)
|
|
75
|
+
if strip > len(previous):
|
|
76
|
+
raise GitIndexError("an entry strips more of the previous path than it has")
|
|
77
|
+
end = _terminator(body, cursor)
|
|
78
|
+
name = previous[: len(previous) - strip] + body[cursor:end]
|
|
79
|
+
offset = end + 1
|
|
80
|
+
else:
|
|
81
|
+
end = _terminator(body, cursor)
|
|
82
|
+
name = body[cursor:end]
|
|
83
|
+
# Versions 2 and 3 pad each entry with 1 to 8 NULs to a multiple of 8 bytes.
|
|
84
|
+
offset += ((end - offset) // 8 + 1) * 8
|
|
85
|
+
previous = name
|
|
86
|
+
entries[name.decode("utf-8", "replace")] = mode
|
|
87
|
+
_check_extensions(body, offset)
|
|
88
|
+
return entries
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _varint(data: bytes, position: int) -> tuple[int, int]:
|
|
92
|
+
byte = data[position]
|
|
93
|
+
position += 1
|
|
94
|
+
value = byte & 0x7F
|
|
95
|
+
while byte & 0x80:
|
|
96
|
+
byte = data[position]
|
|
97
|
+
position += 1
|
|
98
|
+
value = ((value + 1) << 7) | (byte & 0x7F)
|
|
99
|
+
return value, position
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _terminator(data: bytes, start: int) -> int:
|
|
103
|
+
end = data.find(b"\0", start)
|
|
104
|
+
if end < 0:
|
|
105
|
+
raise GitIndexError("the index is truncated")
|
|
106
|
+
return end
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _check_extensions(body: bytes, offset: int) -> None:
|
|
110
|
+
while offset < len(body):
|
|
111
|
+
name, size = _EXTENSION.unpack_from(body, offset)
|
|
112
|
+
# An extension whose name starts with a capital letter is optional. Any other, such
|
|
113
|
+
# as `link` (split index) or `sdir` (sparse index), changes which entries exist.
|
|
114
|
+
if not b"A" <= name[:1] <= b"Z":
|
|
115
|
+
label = name.decode("ascii", "replace")
|
|
116
|
+
raise GitIndexError(f"index extension {label!r} is not supported")
|
|
117
|
+
offset += _EXTENSION.size + size
|
|
118
|
+
if offset != len(body):
|
|
119
|
+
raise GitIndexError("the index is truncated")
|