ithrl 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ithrl/__init__.py +1 -0
- ithrl/__main__.py +6 -0
- ithrl/adapters/__init__.py +49 -0
- ithrl/adapters/_jsonl.py +266 -0
- ithrl/adapters/codex/__init__.py +5 -0
- ithrl/adapters/codex/context.py +204 -0
- ithrl/adapters/codex/decisions.py +224 -0
- ithrl/adapters/codex/discovery.py +60 -0
- ithrl/adapters/codex/parser.py +191 -0
- ithrl/adapters/contracts.py +106 -0
- ithrl/adapters/copilot/__init__.py +5 -0
- ithrl/adapters/copilot/context.py +182 -0
- ithrl/adapters/copilot/decisions.py +294 -0
- ithrl/adapters/copilot/discovery.py +28 -0
- ithrl/adapters/copilot/parser.py +283 -0
- ithrl/adapters/discovery.py +220 -0
- ithrl/adapters/errors.py +52 -0
- ithrl/adapters/registry.py +433 -0
- ithrl/behavior.py +138 -0
- ithrl/calibration.py +434 -0
- ithrl/case/__init__.py +33 -0
- ithrl/case/contracts.py +88 -0
- ithrl/case/evidence.py +169 -0
- ithrl/case/filesystem.py +340 -0
- ithrl/case/source.py +208 -0
- ithrl/case/source_context.py +119 -0
- ithrl/case/source_storage.py +234 -0
- ithrl/case/storage.py +377 -0
- ithrl/cli/__init__.py +968 -0
- ithrl/cli/extract.py +478 -0
- ithrl/cli/native.py +484 -0
- ithrl/cli/output.py +786 -0
- ithrl/cli/paths.py +24 -0
- ithrl/cli/sampled.py +485 -0
- ithrl/cli/sampled_output.py +472 -0
- ithrl/cli/sampled_reduction.py +1036 -0
- ithrl/cli/sampled_session.py +342 -0
- ithrl/cli/shrink.py +523 -0
- ithrl/context.py +465 -0
- ithrl/decisions.py +271 -0
- ithrl/examples/synthetic-trace.json +46 -0
- ithrl/execution/__init__.py +68 -0
- ithrl/execution/budget.py +146 -0
- ithrl/execution/contracts.py +426 -0
- ithrl/execution/preflight.py +346 -0
- ithrl/experiment.py +453 -0
- ithrl/extraction.py +206 -0
- ithrl/measurement.py +315 -0
- ithrl/observations.py +1148 -0
- ithrl/path_safety.py +100 -0
- ithrl/reduce/__init__.py +53 -0
- ithrl/reduce/contracts.py +508 -0
- ithrl/reduce/engine.py +901 -0
- ithrl/reduce/search.py +178 -0
- ithrl/results/__init__.py +49 -0
- ithrl/results/models.py +294 -0
- ithrl/results/storage.py +485 -0
- ithrl/results/summary.py +420 -0
- ithrl/results/validation.py +611 -0
- ithrl/run_state.py +1135 -0
- ithrl/runner/__init__.py +44 -0
- ithrl/runner/codex/__init__.py +17 -0
- ithrl/runner/codex/capability.py +708 -0
- ithrl/runner/codex/configuration.py +529 -0
- ithrl/runner/codex/isolation.py +396 -0
- ithrl/runner/codex/materialize.py +358 -0
- ithrl/runner/codex/policy.py +67 -0
- ithrl/runner/codex/protocol.py +649 -0
- ithrl/runner/codex/protocol_contracts.py +171 -0
- ithrl/runner/codex/runner.py +533 -0
- ithrl/runner/codex/runtime_configuration.py +264 -0
- ithrl/runner/codex/schema.py +527 -0
- ithrl/runner/codex/transcript.py +130 -0
- ithrl/runner/codex/worker.py +175 -0
- ithrl/runner/contracts.py +419 -0
- ithrl/runner/copilot/__init__.py +18 -0
- ithrl/runner/copilot/capability.py +825 -0
- ithrl/runner/copilot/configuration.py +201 -0
- ithrl/runner/copilot/isolation.py +374 -0
- ithrl/runner/copilot/materialize.py +484 -0
- ithrl/runner/copilot/policy.py +110 -0
- ithrl/runner/copilot/protocol/__init__.py +18 -0
- ithrl/runner/copilot/protocol/contracts.py +74 -0
- ithrl/runner/copilot/protocol/decoding.py +93 -0
- ithrl/runner/copilot/protocol/parser.py +418 -0
- ithrl/runner/copilot/protocol/state.py +130 -0
- ithrl/runner/copilot/runner.py +425 -0
- ithrl/runner/copilot/worker.py +351 -0
- ithrl/runner/evidence.py +345 -0
- ithrl/runner/fake.py +107 -0
- ithrl/runner/registry.py +90 -0
- ithrl/runner/runtime/__init__.py +50 -0
- ithrl/runner/runtime/contracts.py +157 -0
- ithrl/runner/runtime/output.py +184 -0
- ithrl/runner/runtime/process.py +954 -0
- ithrl/runner/runtime/protection.py +562 -0
- ithrl/runner/runtime/state.py +282 -0
- ithrl/runner/validation.py +376 -0
- ithrl/sampling.py +412 -0
- ithrl/statistics.py +276 -0
- ithrl/synthetic.py +511 -0
- ithrl/timestamps.py +26 -0
- ithrl/workflow.py +570 -0
- ithrl-0.1.0.dist-info/METADATA +118 -0
- ithrl-0.1.0.dist-info/RECORD +108 -0
- ithrl-0.1.0.dist-info/WHEEL +4 -0
- ithrl-0.1.0.dist-info/entry_points.txt +2 -0
- ithrl-0.1.0.dist-info/licenses/LICENSE +201 -0
ithrl/case/source.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Opaque source artifacts retained by a case."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
from collections.abc import Iterable
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from itertools import islice, pairwise
|
|
11
|
+
from pathlib import PurePosixPath
|
|
12
|
+
from typing import Annotated, Literal
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
15
|
+
|
|
16
|
+
MAX_SOURCE_ARTIFACTS = 64
|
|
17
|
+
MAX_SOURCE_ARTIFACT_BYTES = 64 * 1024 * 1024
|
|
18
|
+
MAX_SOURCE_BYTES = 256 * 1024 * 1024
|
|
19
|
+
MAX_SOURCE_PATH_BYTES = 1024
|
|
20
|
+
MAX_SOURCE_COMPONENT_BYTES = 255
|
|
21
|
+
MAX_SOURCE_MANIFEST_BYTES = 1024 * 1024
|
|
22
|
+
SOURCE_MANIFEST_PATH = "source/manifest.json"
|
|
23
|
+
|
|
24
|
+
_MEDIA_TYPE_PATTERN = re.compile(r"^[A-Za-z0-9!#$&^_.+-]+/[A-Za-z0-9!#$&^_.+-]+$")
|
|
25
|
+
_WINDOWS_RESERVED_NAMES = {
|
|
26
|
+
"aux",
|
|
27
|
+
"con",
|
|
28
|
+
"nul",
|
|
29
|
+
"prn",
|
|
30
|
+
*(f"com{index}" for index in range(1, 10)),
|
|
31
|
+
*(f"lpt{index}" for index in range(1, 10)),
|
|
32
|
+
}
|
|
33
|
+
_WINDOWS_RESERVED_CHARACTERS = frozenset('<>:"|?*')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class SourceArtifactIdentity(BaseModel):
|
|
37
|
+
"""Stable identity of one opaque source artifact."""
|
|
38
|
+
|
|
39
|
+
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
|
40
|
+
|
|
41
|
+
schema_version: Literal[1] = 1
|
|
42
|
+
path: Annotated[str, Field(min_length=1, max_length=MAX_SOURCE_PATH_BYTES)]
|
|
43
|
+
media_type: Annotated[str, Field(min_length=3, max_length=255)]
|
|
44
|
+
size_bytes: Annotated[int, Field(ge=1, le=MAX_SOURCE_ARTIFACT_BYTES)]
|
|
45
|
+
sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
|
46
|
+
|
|
47
|
+
@field_validator("path")
|
|
48
|
+
@classmethod
|
|
49
|
+
def validate_path(cls, value: str) -> str:
|
|
50
|
+
return validate_source_path(value)
|
|
51
|
+
|
|
52
|
+
@field_validator("media_type")
|
|
53
|
+
@classmethod
|
|
54
|
+
def validate_media_type(cls, value: str) -> str:
|
|
55
|
+
if not _MEDIA_TYPE_PATTERN.fullmatch(value):
|
|
56
|
+
raise ValueError("media_type must be an unparameterized Internet media type")
|
|
57
|
+
return value.lower()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class SourceManifest(BaseModel):
|
|
61
|
+
"""Versioned index of every opaque file within the source boundary."""
|
|
62
|
+
|
|
63
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
64
|
+
|
|
65
|
+
schema_version: Literal[1] = 1
|
|
66
|
+
artifacts: Annotated[
|
|
67
|
+
tuple[SourceArtifactIdentity, ...], Field(min_length=1, max_length=MAX_SOURCE_ARTIFACTS)
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
@model_validator(mode="after")
|
|
71
|
+
def require_canonical_artifact_order(self) -> SourceManifest:
|
|
72
|
+
paths = tuple(artifact.path for artifact in self.artifacts)
|
|
73
|
+
if len(paths) != len(set(paths)):
|
|
74
|
+
raise ValueError("source manifest contains duplicate artifact paths")
|
|
75
|
+
if paths != tuple(sorted(paths)):
|
|
76
|
+
raise ValueError("source manifest artifacts must be ordered by path")
|
|
77
|
+
_validate_non_overlapping_paths(paths)
|
|
78
|
+
if sum(artifact.size_bytes for artifact in self.artifacts) > MAX_SOURCE_BYTES:
|
|
79
|
+
raise ValueError("source manifest exceeds the total byte limit")
|
|
80
|
+
return self
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True, slots=True)
|
|
84
|
+
class SourceArtifact:
|
|
85
|
+
"""One opaque source file with exact immutable content."""
|
|
86
|
+
|
|
87
|
+
path: str
|
|
88
|
+
media_type: str
|
|
89
|
+
content: bytes = field(repr=False)
|
|
90
|
+
_identity: SourceArtifactIdentity = field(init=False, repr=False, compare=False)
|
|
91
|
+
|
|
92
|
+
def __post_init__(self) -> None:
|
|
93
|
+
if type(self.content) is not bytes:
|
|
94
|
+
raise TypeError("source artifact content must be bytes")
|
|
95
|
+
size_bytes = len(self.content)
|
|
96
|
+
if size_bytes == 0:
|
|
97
|
+
raise ValueError("source artifact content must not be empty")
|
|
98
|
+
if size_bytes > MAX_SOURCE_ARTIFACT_BYTES:
|
|
99
|
+
raise ValueError("source artifact exceeds the per-file byte limit")
|
|
100
|
+
identity = SourceArtifactIdentity(
|
|
101
|
+
path=self.path,
|
|
102
|
+
media_type=self.media_type,
|
|
103
|
+
size_bytes=size_bytes,
|
|
104
|
+
sha256=hashlib.sha256(self.content).hexdigest(),
|
|
105
|
+
)
|
|
106
|
+
object.__setattr__(self, "path", identity.path)
|
|
107
|
+
object.__setattr__(self, "media_type", identity.media_type)
|
|
108
|
+
object.__setattr__(self, "_identity", identity)
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def identity(self) -> SourceArtifactIdentity:
|
|
112
|
+
return self._identity
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def size_bytes(self) -> int:
|
|
116
|
+
return self._identity.size_bytes
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def sha256(self) -> str:
|
|
120
|
+
return self._identity.sha256
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def prepare_source_artifacts(
|
|
124
|
+
artifacts: Iterable[SourceArtifact],
|
|
125
|
+
) -> tuple[SourceArtifact, ...]:
|
|
126
|
+
"""Validate and order source artifacts for deterministic persistence."""
|
|
127
|
+
|
|
128
|
+
collected = tuple(islice(artifacts, MAX_SOURCE_ARTIFACTS + 1))
|
|
129
|
+
if len(collected) > MAX_SOURCE_ARTIFACTS:
|
|
130
|
+
raise ValueError("case exceeds the source artifact count limit")
|
|
131
|
+
if any(not isinstance(artifact, SourceArtifact) for artifact in collected):
|
|
132
|
+
raise TypeError("source artifacts must use SourceArtifact")
|
|
133
|
+
ordered = tuple(sorted(collected, key=_artifact_path))
|
|
134
|
+
paths = tuple(artifact.path for artifact in ordered)
|
|
135
|
+
if len(paths) != len(set(paths)):
|
|
136
|
+
raise ValueError("case contains duplicate source artifact paths")
|
|
137
|
+
_validate_non_overlapping_paths(paths)
|
|
138
|
+
if sum(artifact.identity.size_bytes for artifact in ordered) > MAX_SOURCE_BYTES:
|
|
139
|
+
raise ValueError("case exceeds the total source byte limit")
|
|
140
|
+
return ordered
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _artifact_path(artifact: SourceArtifact) -> str:
|
|
144
|
+
return artifact.path
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _validate_non_overlapping_paths(paths: tuple[str, ...]) -> None:
|
|
148
|
+
portable_paths = tuple(sorted(_portable_path_key(path) for path in paths))
|
|
149
|
+
if len(portable_paths) != len(set(portable_paths)):
|
|
150
|
+
raise ValueError("source artifact paths must not alias across supported platforms")
|
|
151
|
+
for parent, child in pairwise(portable_paths):
|
|
152
|
+
if child.startswith(f"{parent}/"):
|
|
153
|
+
raise ValueError("source artifact paths must not overlap")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def validate_source_path(value: str) -> str:
|
|
157
|
+
"""Require a portable relative path contained by the source directory."""
|
|
158
|
+
|
|
159
|
+
parts = value.split("/")
|
|
160
|
+
_validate_path_characters(value)
|
|
161
|
+
_validate_path_structure(value, parts)
|
|
162
|
+
_validate_path_lengths(value, parts)
|
|
163
|
+
_validate_portable_components(parts)
|
|
164
|
+
if _portable_path_key(value) == _portable_path_key(SOURCE_MANIFEST_PATH):
|
|
165
|
+
raise ValueError("source artifact path is reserved")
|
|
166
|
+
return value
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _validate_path_characters(value: str) -> None:
|
|
170
|
+
if "\\" in value or any(ord(character) < 32 or ord(character) == 127 for character in value):
|
|
171
|
+
raise ValueError("source artifact path contains unsupported characters")
|
|
172
|
+
if unicodedata.normalize("NFC", value) != value:
|
|
173
|
+
raise ValueError("source artifact path must use canonical Unicode")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _validate_path_structure(value: str, parts: list[str]) -> None:
|
|
177
|
+
if (
|
|
178
|
+
PurePosixPath(value).is_absolute()
|
|
179
|
+
or len(parts) < 2
|
|
180
|
+
or parts[0] != "source"
|
|
181
|
+
or any(part in {"", ".", ".."} for part in parts)
|
|
182
|
+
):
|
|
183
|
+
raise ValueError("source artifact path must be a safe path under 'source/'")
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _validate_path_lengths(value: str, parts: list[str]) -> None:
|
|
187
|
+
if len(value.encode("utf-8")) > MAX_SOURCE_PATH_BYTES:
|
|
188
|
+
raise ValueError("source artifact path is too long")
|
|
189
|
+
if any(len(part.encode("utf-8")) > MAX_SOURCE_COMPONENT_BYTES for part in parts):
|
|
190
|
+
raise ValueError("source artifact path component is too long")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _validate_portable_components(parts: list[str]) -> None:
|
|
194
|
+
if any(not _is_portable_component(part) for part in parts):
|
|
195
|
+
raise ValueError("source artifact path is not portable across supported platforms")
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _is_portable_component(value: str) -> bool:
|
|
199
|
+
stem = value.split(".", maxsplit=1)[0].casefold()
|
|
200
|
+
return (
|
|
201
|
+
not any(character in _WINDOWS_RESERVED_CHARACTERS for character in value)
|
|
202
|
+
and not value.endswith((" ", "."))
|
|
203
|
+
and stem not in _WINDOWS_RESERVED_NAMES
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _portable_path_key(value: str) -> str:
|
|
208
|
+
return "/".join(part.casefold() for part in value.split("/"))
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Verify source-backed context and decision selectors against opaque evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
|
|
9
|
+
from ithrl.case.contracts import CaseFormatError
|
|
10
|
+
from ithrl.case.source import SourceArtifact
|
|
11
|
+
from ithrl.context import ContextGraph, JsonObject, SourceRecordSelector, content_digest
|
|
12
|
+
from ithrl.decisions import DecisionEvidence
|
|
13
|
+
|
|
14
|
+
_MAX_SOURCE_RECORD_BYTES = 8 * 1024 * 1024
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def validate_source_references(
|
|
18
|
+
context: ContextGraph,
|
|
19
|
+
decision: DecisionEvidence | None,
|
|
20
|
+
artifacts: tuple[SourceArtifact, ...],
|
|
21
|
+
) -> None:
|
|
22
|
+
"""Verify every native record selector and its canonical record digest."""
|
|
23
|
+
|
|
24
|
+
if decision is not None:
|
|
25
|
+
_validate_decision_boundary(context, decision)
|
|
26
|
+
expected: dict[str, list[tuple[SourceRecordSelector, str]]] = defaultdict(list)
|
|
27
|
+
for node in context.nodes:
|
|
28
|
+
if node.selector is not None:
|
|
29
|
+
expected[node.source].append((node.selector, node.digest))
|
|
30
|
+
if decision is not None:
|
|
31
|
+
expected[decision.selector.source].append(
|
|
32
|
+
(decision.selector.record, decision.selector.record_digest)
|
|
33
|
+
)
|
|
34
|
+
if not expected:
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
by_path = {artifact.path: artifact for artifact in artifacts}
|
|
38
|
+
missing = sorted(expected.keys() - by_path.keys())
|
|
39
|
+
if missing:
|
|
40
|
+
raise CaseFormatError("source selector refers to unavailable evidence")
|
|
41
|
+
for path, references in expected.items():
|
|
42
|
+
_validate_artifact_records(by_path[path], references)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _validate_decision_boundary(
|
|
46
|
+
context: ContextGraph,
|
|
47
|
+
decision: DecisionEvidence,
|
|
48
|
+
) -> None:
|
|
49
|
+
for node in context.nodes:
|
|
50
|
+
if node.selector is None:
|
|
51
|
+
continue
|
|
52
|
+
if node.source != decision.selector.source:
|
|
53
|
+
raise CaseFormatError("source-backed decision context must use one source artifact")
|
|
54
|
+
if node.selector.line >= decision.selector.context_before_line:
|
|
55
|
+
raise CaseFormatError("source-backed context must precede the selected decision")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _validate_artifact_records(
|
|
59
|
+
artifact: SourceArtifact,
|
|
60
|
+
references: list[tuple[SourceRecordSelector, str]],
|
|
61
|
+
) -> None:
|
|
62
|
+
by_line: dict[int, list[tuple[SourceRecordSelector, str]]] = defaultdict(list)
|
|
63
|
+
for selector, digest in references:
|
|
64
|
+
by_line[selector.line].append((selector, digest))
|
|
65
|
+
remaining = set(by_line)
|
|
66
|
+
for line_number, raw_line in enumerate(io.BytesIO(artifact.content), start=1):
|
|
67
|
+
selected_records = by_line.get(line_number)
|
|
68
|
+
if selected_records is None:
|
|
69
|
+
continue
|
|
70
|
+
record = _load_record(raw_line)
|
|
71
|
+
digest = content_digest(record)
|
|
72
|
+
for selector, expected_digest in selected_records:
|
|
73
|
+
_validate_record_identity(record, selector)
|
|
74
|
+
if digest != expected_digest:
|
|
75
|
+
raise CaseFormatError("source record digest does not match case metadata")
|
|
76
|
+
remaining.remove(line_number)
|
|
77
|
+
if not remaining:
|
|
78
|
+
return
|
|
79
|
+
raise CaseFormatError("source selector line is unavailable")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _load_record(raw: bytes) -> JsonObject:
|
|
83
|
+
record = raw.removesuffix(b"\n").removesuffix(b"\r")
|
|
84
|
+
if len(record) > _MAX_SOURCE_RECORD_BYTES:
|
|
85
|
+
raise CaseFormatError("selected source record exceeds the byte limit")
|
|
86
|
+
|
|
87
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
|
88
|
+
result: dict[str, object] = {}
|
|
89
|
+
for key, value in pairs:
|
|
90
|
+
if key in result:
|
|
91
|
+
raise CaseFormatError("selected source record contains a duplicate key")
|
|
92
|
+
result[key] = value
|
|
93
|
+
return result
|
|
94
|
+
|
|
95
|
+
def reject_constant(_value: str) -> None:
|
|
96
|
+
raise CaseFormatError("selected source record contains an invalid number")
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
value = json.loads(
|
|
100
|
+
record.decode("utf-8"),
|
|
101
|
+
object_pairs_hook=reject_duplicate_keys,
|
|
102
|
+
parse_constant=reject_constant,
|
|
103
|
+
)
|
|
104
|
+
except CaseFormatError:
|
|
105
|
+
raise
|
|
106
|
+
except (json.JSONDecodeError, RecursionError, UnicodeError, ValueError) as error:
|
|
107
|
+
raise CaseFormatError("selected source record is not valid JSON") from error
|
|
108
|
+
if not isinstance(value, dict):
|
|
109
|
+
raise CaseFormatError("selected source record must be a JSON object")
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _validate_record_identity(record: JsonObject, selector: SourceRecordSelector) -> None:
|
|
114
|
+
if selector.identity_key is None:
|
|
115
|
+
return
|
|
116
|
+
actual = record.get(selector.identity_key)
|
|
117
|
+
expected = selector.identity_value
|
|
118
|
+
if type(actual) is not type(expected) or actual != expected:
|
|
119
|
+
raise CaseFormatError("source record identity does not match its selector")
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Persist and verify opaque files within a case's source boundary."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import stat
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from pathlib import Path, PurePosixPath
|
|
10
|
+
|
|
11
|
+
from pydantic import ValidationError
|
|
12
|
+
|
|
13
|
+
from ithrl.case.contracts import CaseFormatError, UnsafeCasePathError
|
|
14
|
+
from ithrl.case.filesystem import (
|
|
15
|
+
PublishedFile,
|
|
16
|
+
atomic_write,
|
|
17
|
+
atomic_write_bytes,
|
|
18
|
+
create_private_directory,
|
|
19
|
+
read_safe_bytes,
|
|
20
|
+
read_safe_text,
|
|
21
|
+
require_private_directory,
|
|
22
|
+
)
|
|
23
|
+
from ithrl.case.source import (
|
|
24
|
+
MAX_SOURCE_ARTIFACT_BYTES,
|
|
25
|
+
MAX_SOURCE_BYTES,
|
|
26
|
+
MAX_SOURCE_MANIFEST_BYTES,
|
|
27
|
+
SOURCE_MANIFEST_PATH,
|
|
28
|
+
SourceArtifact,
|
|
29
|
+
SourceManifest,
|
|
30
|
+
)
|
|
31
|
+
from ithrl.context import JsonValue, validate_json_limits
|
|
32
|
+
|
|
33
|
+
SOURCE_ROOT = Path("source")
|
|
34
|
+
SOURCE_MANIFEST = Path(SOURCE_MANIFEST_PATH)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def write_sources(
|
|
38
|
+
root: Path,
|
|
39
|
+
sources: tuple[SourceArtifact, ...],
|
|
40
|
+
published: list[PublishedFile],
|
|
41
|
+
) -> None:
|
|
42
|
+
if not sources:
|
|
43
|
+
return
|
|
44
|
+
source_root = create_private_directory(root / SOURCE_ROOT)
|
|
45
|
+
directories = sorted(
|
|
46
|
+
{
|
|
47
|
+
root / parent
|
|
48
|
+
for artifact in sources
|
|
49
|
+
for parent in source_parent_paths(artifact.path)
|
|
50
|
+
if parent != SOURCE_ROOT
|
|
51
|
+
},
|
|
52
|
+
key=_path_depth,
|
|
53
|
+
)
|
|
54
|
+
for directory in directories:
|
|
55
|
+
create_private_directory(directory)
|
|
56
|
+
for artifact in sources:
|
|
57
|
+
published.append(atomic_write_bytes(root / PurePosixPath(artifact.path), artifact.content))
|
|
58
|
+
manifest = SourceManifest(artifacts=tuple(artifact.identity for artifact in sources))
|
|
59
|
+
rendered = _render_manifest(manifest)
|
|
60
|
+
if len(rendered.encode("utf-8")) > MAX_SOURCE_MANIFEST_BYTES:
|
|
61
|
+
raise CaseFormatError("source manifest exceeds the byte limit")
|
|
62
|
+
published.append(atomic_write(source_root / "manifest.json", rendered))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def load_sources(root: Path) -> tuple[SourceArtifact, ...]:
|
|
66
|
+
source_root = root / SOURCE_ROOT
|
|
67
|
+
if not os.path.lexists(source_root):
|
|
68
|
+
return ()
|
|
69
|
+
require_private_directory(source_root, label="source")
|
|
70
|
+
manifest_text = read_safe_text(
|
|
71
|
+
root / SOURCE_MANIFEST,
|
|
72
|
+
max_bytes=MAX_SOURCE_MANIFEST_BYTES,
|
|
73
|
+
label="source manifest",
|
|
74
|
+
)
|
|
75
|
+
manifest = _parse_manifest(manifest_text)
|
|
76
|
+
validate_source_tree(root, manifest)
|
|
77
|
+
return _read_artifacts(root, manifest)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _parse_manifest(value: str) -> SourceManifest:
|
|
81
|
+
raw = _load_json(value)
|
|
82
|
+
try:
|
|
83
|
+
validate_json_limits(raw)
|
|
84
|
+
manifest = SourceManifest.model_validate(raw)
|
|
85
|
+
except (ValidationError, ValueError) as error:
|
|
86
|
+
raise CaseFormatError("invalid source manifest") from error
|
|
87
|
+
if value != _render_manifest(manifest):
|
|
88
|
+
raise CaseFormatError("source manifest is not in canonical form")
|
|
89
|
+
return manifest
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _read_artifacts(root: Path, manifest: SourceManifest) -> tuple[SourceArtifact, ...]:
|
|
93
|
+
sources: list[SourceArtifact] = []
|
|
94
|
+
total_bytes = 0
|
|
95
|
+
for expected in manifest.artifacts:
|
|
96
|
+
content = read_safe_bytes(
|
|
97
|
+
root / PurePosixPath(expected.path),
|
|
98
|
+
max_bytes=MAX_SOURCE_ARTIFACT_BYTES,
|
|
99
|
+
label="source artifact",
|
|
100
|
+
)
|
|
101
|
+
total_bytes += len(content)
|
|
102
|
+
if total_bytes > MAX_SOURCE_BYTES:
|
|
103
|
+
raise CaseFormatError("source evidence exceeds the total byte limit")
|
|
104
|
+
artifact = SourceArtifact(
|
|
105
|
+
path=expected.path,
|
|
106
|
+
media_type=expected.media_type,
|
|
107
|
+
content=content,
|
|
108
|
+
)
|
|
109
|
+
if artifact.identity != expected:
|
|
110
|
+
raise CaseFormatError("source artifact identity does not match its manifest")
|
|
111
|
+
sources.append(artifact)
|
|
112
|
+
return tuple(sources)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def validate_source_tree(root: Path, manifest: SourceManifest) -> None:
|
|
116
|
+
expected_files, expected_directories = _expected_entries(manifest)
|
|
117
|
+
seen_files, seen_directories = _scan_source_tree(root, expected_files, expected_directories)
|
|
118
|
+
if seen_files != expected_files or seen_directories != expected_directories:
|
|
119
|
+
raise CaseFormatError("source evidence is incomplete")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _expected_entries(manifest: SourceManifest) -> tuple[set[str], set[str]]:
|
|
123
|
+
files = {SOURCE_MANIFEST_PATH, *(artifact.path for artifact in manifest.artifacts)}
|
|
124
|
+
directories = {SOURCE_ROOT.as_posix()}
|
|
125
|
+
for artifact in manifest.artifacts:
|
|
126
|
+
directories.update(path.as_posix() for path in source_parent_paths(artifact.path))
|
|
127
|
+
return files, directories
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _scan_source_tree(
|
|
131
|
+
root: Path,
|
|
132
|
+
expected_files: set[str],
|
|
133
|
+
expected_directories: set[str],
|
|
134
|
+
) -> tuple[set[str], set[str]]:
|
|
135
|
+
pending = [root / SOURCE_ROOT]
|
|
136
|
+
seen_files: set[str] = set()
|
|
137
|
+
seen_directories: set[str] = set()
|
|
138
|
+
while pending:
|
|
139
|
+
directory = pending.pop()
|
|
140
|
+
seen_directories.add(directory.relative_to(root).as_posix())
|
|
141
|
+
for entry in _directory_entries(directory):
|
|
142
|
+
relative = Path(entry.path).relative_to(root).as_posix()
|
|
143
|
+
metadata = _entry_metadata(entry)
|
|
144
|
+
if stat.S_ISLNK(metadata.st_mode):
|
|
145
|
+
raise UnsafeCasePathError("source evidence must not contain symbolic links")
|
|
146
|
+
if stat.S_ISDIR(metadata.st_mode):
|
|
147
|
+
_accept_directory(entry, relative, expected_directories, pending)
|
|
148
|
+
elif stat.S_ISREG(metadata.st_mode):
|
|
149
|
+
_accept_file(relative, expected_files, seen_files)
|
|
150
|
+
else:
|
|
151
|
+
raise UnsafeCasePathError("source evidence entries must be regular files")
|
|
152
|
+
return seen_files, seen_directories
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _directory_entries(directory: Path) -> Iterator[os.DirEntry[str]]:
|
|
156
|
+
try:
|
|
157
|
+
with os.scandir(directory) as iterator:
|
|
158
|
+
yield from iterator
|
|
159
|
+
except OSError as error:
|
|
160
|
+
raise CaseFormatError("cannot inspect source evidence") from error
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _entry_metadata(entry: os.DirEntry[str]) -> os.stat_result:
|
|
164
|
+
try:
|
|
165
|
+
return entry.stat(follow_symlinks=False)
|
|
166
|
+
except OSError as error:
|
|
167
|
+
raise CaseFormatError("cannot inspect source evidence") from error
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _accept_directory(
|
|
171
|
+
entry: os.DirEntry[str],
|
|
172
|
+
relative: str,
|
|
173
|
+
expected: set[str],
|
|
174
|
+
pending: list[Path],
|
|
175
|
+
) -> None:
|
|
176
|
+
if relative not in expected:
|
|
177
|
+
raise CaseFormatError("source directory contains an unexpected entry")
|
|
178
|
+
path = Path(entry.path)
|
|
179
|
+
require_private_directory(path, label="source")
|
|
180
|
+
pending.append(path)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _accept_file(relative: str, expected: set[str], seen: set[str]) -> None:
|
|
184
|
+
if relative not in expected:
|
|
185
|
+
raise CaseFormatError("source directory contains an unexpected entry")
|
|
186
|
+
seen.add(relative)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def source_parent_paths(path: str) -> tuple[Path, ...]:
|
|
190
|
+
parent = Path(*PurePosixPath(path).parts).parent
|
|
191
|
+
parents: list[Path] = []
|
|
192
|
+
while parent != Path():
|
|
193
|
+
parents.append(parent)
|
|
194
|
+
parent = parent.parent
|
|
195
|
+
return tuple(reversed(parents))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _render_manifest(manifest: SourceManifest) -> str:
|
|
199
|
+
return _render_json(manifest.model_dump(mode="json"))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _render_json(value: JsonValue) -> str:
|
|
203
|
+
try:
|
|
204
|
+
return (
|
|
205
|
+
json.dumps(
|
|
206
|
+
value,
|
|
207
|
+
allow_nan=False,
|
|
208
|
+
ensure_ascii=False,
|
|
209
|
+
indent=2,
|
|
210
|
+
sort_keys=True,
|
|
211
|
+
)
|
|
212
|
+
+ "\n"
|
|
213
|
+
)
|
|
214
|
+
except (TypeError, ValueError) as error:
|
|
215
|
+
raise CaseFormatError("value is not valid JSON") from error
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _load_json(source: str) -> object:
|
|
219
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
|
220
|
+
result: dict[str, object] = {}
|
|
221
|
+
for key, value in pairs:
|
|
222
|
+
if key in result:
|
|
223
|
+
raise CaseFormatError("source manifest contains a duplicate key")
|
|
224
|
+
result[key] = value
|
|
225
|
+
return result
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
return json.loads(source, object_pairs_hook=reject_duplicate_keys)
|
|
229
|
+
except (json.JSONDecodeError, RecursionError) as error:
|
|
230
|
+
raise CaseFormatError("invalid source manifest") from error
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _path_depth(path: Path) -> int:
|
|
234
|
+
return len(path.parts)
|