promptmigrate-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- promptmigrate/__init__.py +21 -0
- promptmigrate/__main__.py +3 -0
- promptmigrate/apply.py +116 -0
- promptmigrate/cli.py +123 -0
- promptmigrate/commands/__init__.py +5 -0
- promptmigrate/commands/check.py +65 -0
- promptmigrate/commands/current.py +25 -0
- promptmigrate/commands/diff.py +19 -0
- promptmigrate/commands/downgrade.py +36 -0
- promptmigrate/commands/env.py +35 -0
- promptmigrate/commands/get.py +48 -0
- promptmigrate/commands/history.py +41 -0
- promptmigrate/commands/init.py +43 -0
- promptmigrate/commands/plan.py +51 -0
- promptmigrate/commands/restamp.py +47 -0
- promptmigrate/commands/revision.py +97 -0
- promptmigrate/commands/upgrade.py +36 -0
- promptmigrate/commands/validate.py +96 -0
- promptmigrate/core.py +30 -0
- promptmigrate/draftdiff.py +41 -0
- promptmigrate/errors.py +91 -0
- promptmigrate/output.py +122 -0
- promptmigrate/plan.py +112 -0
- promptmigrate/project.py +160 -0
- promptmigrate/protocol/__init__.py +9 -0
- promptmigrate/protocol/equality.py +15 -0
- promptmigrate/protocol/jsonloader.py +92 -0
- promptmigrate/protocol/limits.py +9 -0
- promptmigrate/protocol/rules.py +80 -0
- promptmigrate/revisions.py +181 -0
- promptmigrate/secretscan.py +195 -0
- promptmigrate/store/__init__.py +13 -0
- promptmigrate/store/environments.py +181 -0
- promptmigrate/store/events.py +70 -0
- promptmigrate/store/lock.py +66 -0
- promptmigrate/store/sqlite.py +147 -0
- promptmigrate_cli-0.1.0.dist-info/METADATA +247 -0
- promptmigrate_cli-0.1.0.dist-info/RECORD +42 -0
- promptmigrate_cli-0.1.0.dist-info/WHEEL +5 -0
- promptmigrate_cli-0.1.0.dist-info/entry_points.txt +2 -0
- promptmigrate_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- promptmigrate_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Heuristic secret detection, the fingerprinted allowlist, and output redaction. Pure functions; no I/O.
|
|
2
|
+
|
|
3
|
+
Detection reduces the chance of a secret entering a new revision or PromptMigrate output. It can miss secrets, and it
|
|
4
|
+
does nothing for secrets already in Git history, which must be rotated.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterable, Iterator
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from .errors import SchemaError
|
|
17
|
+
|
|
18
|
+
# The one warning text printed whenever display output had values redacted.
|
|
19
|
+
_REDACTED = "{count} secret-like value(s) redacted from this output; detection is heuristic and can miss secrets"
|
|
20
|
+
|
|
21
|
+
VALUE_RULES: dict[str, re.Pattern[str]] = {
|
|
22
|
+
"openai-anthropic-key": re.compile(r"sk-[A-Za-z0-9_-]{20,}"),
|
|
23
|
+
"aws-access-key-id": re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
24
|
+
"github-token": re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}"),
|
|
25
|
+
"slack-token": re.compile(r"xox[abposr]-[A-Za-z0-9-]{10,}"),
|
|
26
|
+
"private-key-pem": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
|
27
|
+
"bearer-token": re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{20,}"),
|
|
28
|
+
"jwt": re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"),
|
|
29
|
+
}
|
|
30
|
+
KEY_RULE = "secret-key-name"
|
|
31
|
+
_SECRET_KEY = re.compile(
|
|
32
|
+
r"(?i)^(api[_-]?key|apikey|password|passwd|secret|client[_-]?secret|access[_-]?token|auth[_-]?token|token"
|
|
33
|
+
r"|private[_-]?key)$"
|
|
34
|
+
)
|
|
35
|
+
RULE_IDS: tuple[str, ...] = (*VALUE_RULES, KEY_RULE)
|
|
36
|
+
|
|
37
|
+
_FIELDS = ("name", "path", "rule", "fingerprint", "reason")
|
|
38
|
+
_FINGERPRINT = re.compile(r"sha256:[0-9a-f]{64}")
|
|
39
|
+
_SHOWABLE = re.compile(r"[a-z0-9_-]{1,40}")
|
|
40
|
+
_JWT_START = re.compile(r"eyJ[A-Za-z0-9_-]*")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Finding:
|
|
45
|
+
"""A secret-like value at `path` (a JSON pointer) inside config `name`. Never carries the value."""
|
|
46
|
+
|
|
47
|
+
name: str
|
|
48
|
+
path: str
|
|
49
|
+
rule: str
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class AllowEntry:
|
|
54
|
+
name: str
|
|
55
|
+
path: str
|
|
56
|
+
rule: str
|
|
57
|
+
fingerprint: str
|
|
58
|
+
reason: str
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def fingerprint(text: str) -> str:
|
|
62
|
+
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def pointer(parts: tuple[str | int, ...]) -> str:
|
|
66
|
+
return "".join("/" + str(part).replace("~", "~0").replace("/", "~1") for part in parts)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _strings(config: Any) -> Iterator[tuple[tuple[str | int, ...], str | None, str]]:
|
|
70
|
+
"""Each string value in document order, with its path parts and its object key (None inside a list).
|
|
71
|
+
|
|
72
|
+
Iterative, so deeply nested legacy configs can't overflow the stack.
|
|
73
|
+
"""
|
|
74
|
+
stack: list[tuple[tuple[str | int, ...], str | None, Any]] = [((), None, config)]
|
|
75
|
+
while stack:
|
|
76
|
+
parts, key, value = stack.pop()
|
|
77
|
+
if isinstance(value, dict):
|
|
78
|
+
stack.extend(((*parts, k), k, v) for k, v in reversed(list(value.items())))
|
|
79
|
+
elif isinstance(value, list):
|
|
80
|
+
stack.extend(((*parts, i), None, v) for i, v in reversed(list(enumerate(value))))
|
|
81
|
+
elif isinstance(value, str):
|
|
82
|
+
yield parts, key, value
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _jwt_matches(text: str) -> Iterator[str]:
|
|
86
|
+
"""VALUE_RULES["jwt"] matches, found in linear time.
|
|
87
|
+
|
|
88
|
+
A later "eyJ" inside the same [A-Za-z0-9_-] run shares the run's tail and is shorter, so if the full pattern fails
|
|
89
|
+
at the run's first "eyJ" it fails at every later one: skip to the end of the run.
|
|
90
|
+
"""
|
|
91
|
+
pattern, pos = VALUE_RULES["jwt"], 0
|
|
92
|
+
while (start := _JWT_START.search(text, pos)) is not None:
|
|
93
|
+
full = pattern.match(text, start.start())
|
|
94
|
+
if full is not None:
|
|
95
|
+
yield full.group(0)
|
|
96
|
+
pos = full.end()
|
|
97
|
+
else:
|
|
98
|
+
pos = start.end()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _matches(config: Any) -> Iterator[tuple[tuple[str | int, ...], str, str]]:
|
|
102
|
+
"""(path parts, rule, matched substring): value rules in table order, then the key rule."""
|
|
103
|
+
for parts, key, text in _strings(config):
|
|
104
|
+
for rule, pattern in VALUE_RULES.items():
|
|
105
|
+
found = _jwt_matches(text) if rule == "jwt" else (m.group(0) for m in pattern.finditer(text))
|
|
106
|
+
for matched in found:
|
|
107
|
+
yield parts, rule, matched
|
|
108
|
+
if key is not None and _SECRET_KEY.search(key) and text and not text.startswith("secret://"):
|
|
109
|
+
yield parts, KEY_RULE, text
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _findings(name: str, config: Any, allow: Iterable[AllowEntry], used: set[AllowEntry] | None):
|
|
113
|
+
allowed = {(e.name, e.path, e.rule, e.fingerprint): e for e in allow}
|
|
114
|
+
found: dict[Finding, tuple[str | int, ...]] = {}
|
|
115
|
+
for parts, rule, matched in _matches(config):
|
|
116
|
+
path = pointer(parts)
|
|
117
|
+
entry = allowed.get((name, path, rule, fingerprint(matched)))
|
|
118
|
+
if entry is not None:
|
|
119
|
+
if used is not None:
|
|
120
|
+
used.add(entry)
|
|
121
|
+
continue
|
|
122
|
+
found.setdefault(Finding(name, path, rule), parts)
|
|
123
|
+
return found
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def scan(name: str, config: Any, allow: Iterable[AllowEntry] = (), used: set[AllowEntry] | None = None) -> list[Finding]:
|
|
127
|
+
"""Non-allowlisted findings in `config`, one per (path, rule), in document order.
|
|
128
|
+
|
|
129
|
+
An allowlist entry suppresses a finding only when name, path, rule and the fingerprint of the exact matched
|
|
130
|
+
substring all match (for the key rule, the matched substring is the whole value). Entries that suppressed
|
|
131
|
+
something are added to `used`.
|
|
132
|
+
"""
|
|
133
|
+
return list(_findings(name, config, allow, used))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def redact(name: str, config: Any, allow: Iterable[AllowEntry] = ()) -> tuple[Any, list[str]]:
|
|
137
|
+
"""A copy of `config` with each flagged string replaced by "<redacted:RULE>", plus the redacted paths.
|
|
138
|
+
|
|
139
|
+
The first finding at a path decides the rule. The input is never mutated; without findings it is returned as is.
|
|
140
|
+
"""
|
|
141
|
+
found = _findings(name, config, allow, None)
|
|
142
|
+
if not found:
|
|
143
|
+
return config, []
|
|
144
|
+
copy = json.loads(json.dumps(config))
|
|
145
|
+
paths: list[str] = []
|
|
146
|
+
for finding, parts in found.items():
|
|
147
|
+
if finding.path in paths:
|
|
148
|
+
continue
|
|
149
|
+
paths.append(finding.path)
|
|
150
|
+
if not parts:
|
|
151
|
+
copy = f"<redacted:{finding.rule}>"
|
|
152
|
+
continue
|
|
153
|
+
target = copy
|
|
154
|
+
for part in parts[:-1]:
|
|
155
|
+
target = target[part]
|
|
156
|
+
target[parts[-1]] = f"<redacted:{finding.rule}>"
|
|
157
|
+
return copy, paths
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _showable(text: Any) -> str:
|
|
161
|
+
"""`repr(text)` when it is safe to echo in an error, else a placeholder. Never echoes a fingerprint or value."""
|
|
162
|
+
return repr(text) if isinstance(text, str) and _SHOWABLE.fullmatch(text) else "(not shown)"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def parse_allowlist(value: Any, filename: str) -> list[AllowEntry]:
|
|
166
|
+
"""Validate a parsed secret-allowlist.json. Errors name entries and rules, never fingerprints."""
|
|
167
|
+
prefix = f"Invalid secret allowlist {filename}"
|
|
168
|
+
if not isinstance(value, dict) or set(value) != {"allow"} or not isinstance(value["allow"], list):
|
|
169
|
+
raise SchemaError(f'{prefix}: the top level must be {{"allow": [...]}}')
|
|
170
|
+
entries = []
|
|
171
|
+
seen: dict[tuple[str, str, str, str], int] = {}
|
|
172
|
+
for index, item in enumerate(value["allow"]):
|
|
173
|
+
where = f"{prefix}: entry {index}"
|
|
174
|
+
if not isinstance(item, dict):
|
|
175
|
+
raise SchemaError(f"{where} must be an object")
|
|
176
|
+
unknown = sorted(set(item) - set(_FIELDS))
|
|
177
|
+
if unknown:
|
|
178
|
+
raise SchemaError(f"{where}: unknown key {_showable(unknown[0])}")
|
|
179
|
+
missing = [field for field in _FIELDS if field not in item]
|
|
180
|
+
if missing:
|
|
181
|
+
raise SchemaError(f"{where}: missing {missing[0]}")
|
|
182
|
+
if not all(isinstance(item[field], str) for field in _FIELDS):
|
|
183
|
+
raise SchemaError(f"{where}: every field must be a string")
|
|
184
|
+
if item["rule"] not in RULE_IDS:
|
|
185
|
+
raise SchemaError(f"{where}: unknown rule {_showable(item['rule'])}")
|
|
186
|
+
if not _FINGERPRINT.fullmatch(item["fingerprint"]):
|
|
187
|
+
raise SchemaError(f"{where}: fingerprint must be 'sha256:' followed by 64 lowercase hex digits")
|
|
188
|
+
if not item["reason"].strip():
|
|
189
|
+
raise SchemaError(f"{where}: reason must not be empty")
|
|
190
|
+
key = (item["name"], item["path"], item["rule"], item["fingerprint"])
|
|
191
|
+
if key in seen:
|
|
192
|
+
raise SchemaError(f"{where}: duplicates entry {seen[key]}")
|
|
193
|
+
seen[key] = index
|
|
194
|
+
entries.append(AllowEntry(*(item[field] for field in _FIELDS)))
|
|
195
|
+
return entries
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Declared local environments and SQLite connection lifecycle.
|
|
2
|
+
|
|
3
|
+
Names are defined in the submodules and re-exported here. Patch a function on the submodule that defines it.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from ..project import require_init, root_dir
|
|
7
|
+
from .environments import (
|
|
8
|
+
DEFAULT_ENVIRONMENTS, _infer_environments, _existing_mode, _resolve_declared, _validate_name, _warn_adoption,
|
|
9
|
+
_warn_production, _write_environments, add_environment, adopt_environments, canonical_name, display_name,
|
|
10
|
+
list_environments, read_environments, resolve_environment,
|
|
11
|
+
)
|
|
12
|
+
from .lock import _project_lock
|
|
13
|
+
from .sqlite import _db_path, _is_linklike, _sqlite_uri, _states_dir, _wal_mode, connect_readonly, connect_write
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Declared local environments: the environments.json manifest, legacy inference and adoption, name resolution.
|
|
2
|
+
|
|
3
|
+
Callers resolve environment names before opening connections (see store.sqlite).
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import stat
|
|
11
|
+
from typing import Any
|
|
12
|
+
import warnings
|
|
13
|
+
|
|
14
|
+
from .. import project, protocol
|
|
15
|
+
from ..errors import (
|
|
16
|
+
EnvironmentAdoptionWarning, EnvironmentNotDeclaredError, ProductionNameWarning, SchemaError, StorageError,
|
|
17
|
+
)
|
|
18
|
+
from . import lock, sqlite
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Environment-name syntax lives in protocol.rules, so store.sqlite can validate names without importing this module.
|
|
22
|
+
_NAME = protocol.rules._ENV_NAME
|
|
23
|
+
_validate_name = protocol.rules.validate_environment_name
|
|
24
|
+
DEFAULT_ENVIRONMENTS = ("dev", "staging")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_environments(project_dir: Path) -> tuple[dict[str, dict[str, Any]], bool]:
|
|
28
|
+
"""Read declarations, or infer legacy declarations without writing any files."""
|
|
29
|
+
root = project.require_init(project_dir)
|
|
30
|
+
path = root / "environments.json"
|
|
31
|
+
if path.exists() or path.is_symlink():
|
|
32
|
+
value, _raw, _rules = protocol.load_json(path, kind="environments", root=root)
|
|
33
|
+
if not isinstance(value, dict) or set(value) != {"environments"} or not isinstance(value["environments"], dict):
|
|
34
|
+
raise SchemaError("Invalid environments file: expected exactly {'environments': object}")
|
|
35
|
+
environments = value["environments"]
|
|
36
|
+
existed = True
|
|
37
|
+
else:
|
|
38
|
+
environments = _infer_environments(project_dir)
|
|
39
|
+
existed = False
|
|
40
|
+
for name, metadata in environments.items():
|
|
41
|
+
_validate_name(name)
|
|
42
|
+
if not isinstance(metadata, dict):
|
|
43
|
+
raise SchemaError(f"Invalid environments file: metadata for {name!r} must be an object")
|
|
44
|
+
if protocol.config_depth(metadata) > protocol.limits.MAX_CONFIG_DEPTH:
|
|
45
|
+
raise SchemaError(f"Invalid environments file: metadata for {name!r} exceeds depth limit")
|
|
46
|
+
collisions = protocol.case_collisions(environments)
|
|
47
|
+
if collisions:
|
|
48
|
+
raise SchemaError(f"Invalid environments file: case-collision {collisions[0]}")
|
|
49
|
+
return environments, existed
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _infer_environments(project_dir: Path) -> dict[str, dict[str, Any]]:
|
|
53
|
+
"""Legacy declarations: the defaults plus every state file a pre-manifest release could have created.
|
|
54
|
+
|
|
55
|
+
Those releases validated names with the same syntax, so a stem that fails it was never PromptMigrate state and is
|
|
56
|
+
ignored. Reserved and case-colliding stems stay in and fail closed later, because they may be real state.
|
|
57
|
+
"""
|
|
58
|
+
stems = [p.stem for p in sqlite._states_dir(project_dir).glob("*.sqlite3") if p.is_file() and _NAME.fullmatch(p.stem)]
|
|
59
|
+
taken = {stem.lower() for stem in stems}
|
|
60
|
+
names = stems + [name for name in DEFAULT_ENVIRONMENTS if name not in taken]
|
|
61
|
+
return {name: {} for name in sorted(names, key=str.lower)}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def canonical_name(environments: dict[str, Any], env: str) -> str | None:
|
|
65
|
+
"""Return the declared spelling that matches env case-insensitively, or None."""
|
|
66
|
+
lowered = env.lower()
|
|
67
|
+
return next((name for name in environments if name.lower() == lowered), None)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def display_name(project_dir: Path, env: str) -> str:
|
|
71
|
+
"""Return env's declared spelling for status labels, or env unchanged when it is not declared."""
|
|
72
|
+
_validate_name(env)
|
|
73
|
+
environments, _existed = read_environments(project_dir)
|
|
74
|
+
return canonical_name(environments, env) or env
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _existing_mode(path: Path) -> int | None:
|
|
78
|
+
"""Return an existing file's permission bits, or None for a new file.
|
|
79
|
+
|
|
80
|
+
Never reads the process umask: `os.umask` is process-wide, so reading it means setting it, which races with
|
|
81
|
+
other threads in an embedding app. A new file gets its mode from `os.open(..., 0o666)`, which applies the umask.
|
|
82
|
+
"""
|
|
83
|
+
try:
|
|
84
|
+
return stat.S_IMODE(path.stat().st_mode)
|
|
85
|
+
except FileNotFoundError:
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _write_environments(project_dir: Path, environments: dict[str, dict[str, Any]]) -> Path:
|
|
90
|
+
path = project.require_init(project_dir) / "environments.json"
|
|
91
|
+
text = json.dumps({"environments": environments}, ensure_ascii=False, indent=2) + "\n"
|
|
92
|
+
if len(text.encode("utf-8")) > protocol.limits.MAX_CONTROL_FILE_BYTES:
|
|
93
|
+
raise SchemaError(f"Invalid environments file: output exceeds the {protocol.limits.MAX_CONTROL_FILE_BYTES}-byte limit")
|
|
94
|
+
mode = _existing_mode(path)
|
|
95
|
+
temporary = None
|
|
96
|
+
try:
|
|
97
|
+
# Not tempfile: it creates 0600 files, and fixing that with chmod ignores the umask. _create_temp opens with
|
|
98
|
+
# 0o666, so the OS applies the umask; only an existing manifest's own mode is restored.
|
|
99
|
+
descriptor, temporary = project._create_temp(path.parent)
|
|
100
|
+
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as file:
|
|
101
|
+
file.write(text)
|
|
102
|
+
if mode is not None:
|
|
103
|
+
os.chmod(temporary, mode)
|
|
104
|
+
os.replace(temporary, path)
|
|
105
|
+
finally:
|
|
106
|
+
if temporary is not None:
|
|
107
|
+
temporary.unlink(missing_ok=True)
|
|
108
|
+
return path
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _warn_adoption() -> None:
|
|
112
|
+
warnings.warn("Adopted legacy environments into environments.json.", EnvironmentAdoptionWarning, stacklevel=3)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _warn_production(name: str) -> None:
|
|
116
|
+
if name.lower() in {"prod", "production", "live"}:
|
|
117
|
+
warnings.warn(f"'{name}' is local SQLite state only; PromptMigrate does not deploy anything.",
|
|
118
|
+
ProductionNameWarning, stacklevel=3)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def resolve_environment(project_dir: Path, env: str, *, mutating: bool = False) -> str:
|
|
122
|
+
"""Resolve case-insensitively; with mutating=True, also persist inferred legacy declarations."""
|
|
123
|
+
_validate_name(env)
|
|
124
|
+
if mutating:
|
|
125
|
+
with lock._project_lock(project_dir):
|
|
126
|
+
name = _resolve_declared(project_dir, env, adopt=True)
|
|
127
|
+
else:
|
|
128
|
+
name = _resolve_declared(project_dir, env, adopt=False)
|
|
129
|
+
_warn_production(name)
|
|
130
|
+
return name
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _resolve_declared(project_dir: Path, env: str, *, adopt: bool) -> str:
|
|
134
|
+
environments, existed = read_environments(project_dir)
|
|
135
|
+
name = canonical_name(environments, env)
|
|
136
|
+
if name is None:
|
|
137
|
+
raise EnvironmentNotDeclaredError(env)
|
|
138
|
+
if adopt and not existed:
|
|
139
|
+
_write_environments(project_dir, environments)
|
|
140
|
+
_warn_adoption()
|
|
141
|
+
return name
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def adopt_environments(project_dir: Path) -> None:
|
|
145
|
+
"""Persist inferred legacy declarations once; a no-op when the manifest already exists.
|
|
146
|
+
|
|
147
|
+
Migration commands call this only after their transaction commits, so a rejected command writes nothing.
|
|
148
|
+
"""
|
|
149
|
+
with lock._project_lock(project_dir):
|
|
150
|
+
environments, existed = read_environments(project_dir)
|
|
151
|
+
if not existed:
|
|
152
|
+
_write_environments(project_dir, environments)
|
|
153
|
+
_warn_adoption()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def add_environment(project_dir: Path, name: str) -> Path:
|
|
157
|
+
"""Atomically declare a new name, retaining all existing metadata."""
|
|
158
|
+
_validate_name(name)
|
|
159
|
+
with lock._project_lock(project_dir):
|
|
160
|
+
environments, existed = read_environments(project_dir)
|
|
161
|
+
if canonical_name(environments, name) is not None:
|
|
162
|
+
raise SchemaError(f"Environment {name!r} is already declared")
|
|
163
|
+
environments[name] = {}
|
|
164
|
+
path = _write_environments(project_dir, environments)
|
|
165
|
+
if not existed:
|
|
166
|
+
_warn_adoption()
|
|
167
|
+
_warn_production(name)
|
|
168
|
+
return path
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def list_environments(project_dir: Path) -> list[tuple[str, bool]]:
|
|
172
|
+
"""List canonical declarations and state-file presence without opening SQLite."""
|
|
173
|
+
environments, _existed = read_environments(project_dir)
|
|
174
|
+
states = sqlite._states_dir(project_dir)
|
|
175
|
+
result = []
|
|
176
|
+
for name in sorted(environments, key=str.lower):
|
|
177
|
+
path = states / f"{name}.sqlite3"
|
|
178
|
+
if sqlite._is_linklike(path):
|
|
179
|
+
raise StorageError(f"Invalid environment state {name!r}: database path is a symbolic link")
|
|
180
|
+
result.append((name, path.is_file()))
|
|
181
|
+
return result
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Append-only `events` table of an environment state DB (ADR-029): schema, triggers and the row writer.
|
|
2
|
+
|
|
3
|
+
Callers own the connection and the transaction; nothing here commits.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
import sqlite3
|
|
9
|
+
|
|
10
|
+
from ..errors import StorageError
|
|
11
|
+
|
|
12
|
+
# (name, declared type, NOT NULL flag, primary-key flag): PRAGMA table_info's (cid, name, type, notnull, dflt_value,
|
|
13
|
+
# pk) columns 1, 2, 3 and 5, as it reports them for _TABLE.
|
|
14
|
+
_SHAPE = (
|
|
15
|
+
("seq", "INTEGER", 0, 1), ("occurred_at", "TEXT", 1, 0), ("command", "TEXT", 1, 0),
|
|
16
|
+
("revision", "TEXT", 1, 0), ("checksum", "TEXT", 1, 0), ("previous_checksum", "TEXT", 0, 0),
|
|
17
|
+
)
|
|
18
|
+
COLUMNS = tuple(column[0] for column in _SHAPE)
|
|
19
|
+
|
|
20
|
+
_TABLE = """
|
|
21
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
22
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
23
|
+
occurred_at TEXT NOT NULL,
|
|
24
|
+
command TEXT NOT NULL CHECK (command IN ('upgrade', 'downgrade', 'restamp')),
|
|
25
|
+
revision TEXT NOT NULL,
|
|
26
|
+
checksum TEXT NOT NULL,
|
|
27
|
+
previous_checksum TEXT
|
|
28
|
+
)
|
|
29
|
+
"""
|
|
30
|
+
_TRIGGERS = (
|
|
31
|
+
"CREATE TRIGGER IF NOT EXISTS events_no_update BEFORE UPDATE ON events "
|
|
32
|
+
"BEGIN SELECT RAISE(ABORT, 'events is append-only'); END",
|
|
33
|
+
"CREATE TRIGGER IF NOT EXISTS events_no_delete BEFORE DELETE ON events "
|
|
34
|
+
"BEGIN SELECT RAISE(ABORT, 'events is append-only'); END",
|
|
35
|
+
# INSERT OR REPLACE on an existing seq deletes the old row without firing the DELETE trigger (SQLite only does so
|
|
36
|
+
# with recursive_triggers on, a per-connection setting), so an insert that reuses a seq is rejected here.
|
|
37
|
+
"CREATE TRIGGER IF NOT EXISTS events_no_replace BEFORE INSERT ON events "
|
|
38
|
+
"WHEN EXISTS (SELECT 1 FROM events WHERE seq = NEW.seq) "
|
|
39
|
+
"BEGIN SELECT RAISE(ABORT, 'events is append-only'); END",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
_TRIGGER_NAMES = {"events_no_update", "events_no_delete", "events_no_replace"}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def ensure(db: sqlite3.Connection) -> None:
|
|
47
|
+
"""Create the table and its append-only triggers if absent; fail closed on any other shape."""
|
|
48
|
+
try:
|
|
49
|
+
db.execute(_TABLE)
|
|
50
|
+
shape = tuple((row[1], row[2].upper(), row[3], row[5]) for row in db.execute("PRAGMA table_info(events)"))
|
|
51
|
+
if shape != _SHAPE:
|
|
52
|
+
raise StorageError("Invalid environment state: events table has unexpected columns")
|
|
53
|
+
for trigger in _TRIGGERS: # only on a table of the expected shape
|
|
54
|
+
db.execute(trigger)
|
|
55
|
+
# IF NOT EXISTS skips a trigger name already used on another table, which would leave `events` unguarded.
|
|
56
|
+
bound = {row[0] for row in db.execute(
|
|
57
|
+
"SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name='events'")}
|
|
58
|
+
if not _TRIGGER_NAMES <= bound:
|
|
59
|
+
raise StorageError("Invalid environment state: events triggers are missing or bound to another table")
|
|
60
|
+
except sqlite3.Error as exc: # e.g. a view named `events`, or a same-named trigger on another table
|
|
61
|
+
raise StorageError(f"Invalid environment state: cannot create the events table: {exc}") from exc
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def record(db: sqlite3.Connection, command: str, revision: str, checksum: str,
|
|
65
|
+
previous_checksum: str | None = None) -> None:
|
|
66
|
+
"""Append one event row; the table must exist (see `ensure`)."""
|
|
67
|
+
db.execute(
|
|
68
|
+
"INSERT INTO events(occurred_at, command, revision, checksum, previous_checksum) VALUES(?, ?, ?, ?, ?)",
|
|
69
|
+
(datetime.now(timezone.utc).isoformat(), command, revision, checksum, previous_checksum),
|
|
70
|
+
)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Project-wide lock serializing manifest changes, revision writes, and initial environment database setup."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import sys
|
|
10
|
+
import threading
|
|
11
|
+
|
|
12
|
+
from .. import project
|
|
13
|
+
from ..errors import StorageError
|
|
14
|
+
|
|
15
|
+
# Project roots whose lock this thread already holds. POSIX flock on a second descriptor blocks against the
|
|
16
|
+
# first, so a nested lock (restamp around connect_write) would deadlock; the Windows mutex is re-entrant already.
|
|
17
|
+
_held = threading.local()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@contextmanager
|
|
21
|
+
def _project_lock(project_dir: Path) -> Iterator[None]:
|
|
22
|
+
"""Serialize project metadata changes, revision writes, and initial environment database setup."""
|
|
23
|
+
root = project.require_init(project_dir)
|
|
24
|
+
# sys.platform, not os.name: mypy narrows it, so each branch is only checked for its own platform
|
|
25
|
+
if sys.platform == "win32":
|
|
26
|
+
import ctypes
|
|
27
|
+
digest = hashlib.sha256(os.fsencode(root)).hexdigest()
|
|
28
|
+
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
29
|
+
kernel.CreateMutexW.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_wchar_p)
|
|
30
|
+
kernel.CreateMutexW.restype = ctypes.c_void_p
|
|
31
|
+
kernel.WaitForSingleObject.argtypes = (ctypes.c_void_p, ctypes.c_uint32)
|
|
32
|
+
kernel.WaitForSingleObject.restype = ctypes.c_uint32
|
|
33
|
+
kernel.ReleaseMutex.argtypes = (ctypes.c_void_p,)
|
|
34
|
+
kernel.CloseHandle.argtypes = (ctypes.c_void_p,)
|
|
35
|
+
handle = kernel.CreateMutexW(None, False, f"Global\\PromptMigrate-{digest}")
|
|
36
|
+
if not handle:
|
|
37
|
+
raise StorageError(
|
|
38
|
+
f"Could not create environment manifest mutex: Windows error {ctypes.get_last_error()}"
|
|
39
|
+
) from None
|
|
40
|
+
result = kernel.WaitForSingleObject(handle, 0xFFFFFFFF)
|
|
41
|
+
if result not in (0, 0x80):
|
|
42
|
+
error = ctypes.get_last_error() # read before CloseHandle, which can overwrite the thread's last error
|
|
43
|
+
kernel.CloseHandle(handle)
|
|
44
|
+
raise StorageError(f"Could not acquire environment manifest mutex: Windows error {error}") from None
|
|
45
|
+
try:
|
|
46
|
+
yield
|
|
47
|
+
finally:
|
|
48
|
+
kernel.ReleaseMutex(handle)
|
|
49
|
+
kernel.CloseHandle(handle)
|
|
50
|
+
else:
|
|
51
|
+
import fcntl
|
|
52
|
+
held = _held.__dict__.setdefault("roots", set())
|
|
53
|
+
if root in held:
|
|
54
|
+
yield
|
|
55
|
+
return
|
|
56
|
+
descriptor = os.open(root, os.O_RDONLY)
|
|
57
|
+
try:
|
|
58
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
|
59
|
+
held.add(root)
|
|
60
|
+
try:
|
|
61
|
+
yield
|
|
62
|
+
finally:
|
|
63
|
+
held.discard(root)
|
|
64
|
+
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
65
|
+
finally:
|
|
66
|
+
os.close(descriptor)
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Environment SQLite state: database paths and the read-only and read/write connection lifecycles.
|
|
2
|
+
|
|
3
|
+
Connection helpers accept the canonical environment name; the migration layer owns BEGIN IMMEDIATE and replay.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from collections.abc import Iterator
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import sqlite3
|
|
12
|
+
import sys
|
|
13
|
+
from urllib.parse import quote
|
|
14
|
+
|
|
15
|
+
from .. import project, protocol
|
|
16
|
+
from ..errors import StorageError
|
|
17
|
+
from . import lock
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _is_linklike(path: Path) -> bool:
|
|
21
|
+
if path.is_symlink():
|
|
22
|
+
return True
|
|
23
|
+
if sys.platform == "win32": # not os.name: mypy only narrows sys.platform
|
|
24
|
+
try:
|
|
25
|
+
return bool(path.stat(follow_symlinks=False).st_file_attributes & 0x400)
|
|
26
|
+
except FileNotFoundError:
|
|
27
|
+
return False
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _states_dir(project_dir: Path) -> Path:
|
|
32
|
+
path = project.require_init(project_dir) / "states"
|
|
33
|
+
if _is_linklike(path):
|
|
34
|
+
raise StorageError("Invalid environment state: states directory is a symbolic link or junction")
|
|
35
|
+
if path.exists() and not path.is_dir():
|
|
36
|
+
raise StorageError("Invalid environment state: states path is not a directory")
|
|
37
|
+
return path
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _sqlite_uri(path: Path) -> str:
|
|
41
|
+
"""Build a read-only SQLite URI without treating a Windows UNC host as authority."""
|
|
42
|
+
if path.drive.startswith("\\\\"):
|
|
43
|
+
return "file://" + quote(path.as_posix(), safe="/") + "?mode=ro"
|
|
44
|
+
return path.as_uri() + "?mode=ro"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _wal_mode(path: Path) -> bool:
|
|
48
|
+
try:
|
|
49
|
+
with path.open("rb") as database:
|
|
50
|
+
header = database.read(20)
|
|
51
|
+
except OSError as exc:
|
|
52
|
+
raise StorageError(f"Cannot inspect environment state: {exc}") from exc
|
|
53
|
+
return header.startswith(b"SQLite format 3\0") and len(header) >= 20 and 2 in header[18:20]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _db_path(project_dir: Path, env: str) -> Path:
|
|
57
|
+
protocol.rules.validate_environment_name(env)
|
|
58
|
+
path = _states_dir(project_dir) / f"{env}.sqlite3"
|
|
59
|
+
if _is_linklike(path):
|
|
60
|
+
raise StorageError(f"Invalid environment state {env!r}: database path is a symbolic link")
|
|
61
|
+
return path
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@contextmanager
|
|
65
|
+
def connect_readonly(project_dir: Path, env: str) -> Iterator[sqlite3.Connection | None]:
|
|
66
|
+
"""Yield base state for an absent DB, otherwise a checked, read-only connection."""
|
|
67
|
+
path = _db_path(project_dir, env)
|
|
68
|
+
if not path.exists():
|
|
69
|
+
if _is_linklike(path):
|
|
70
|
+
raise StorageError(f"Invalid environment state {env!r}: database path is a dangling symbolic link")
|
|
71
|
+
yield None
|
|
72
|
+
return
|
|
73
|
+
connection = None
|
|
74
|
+
try:
|
|
75
|
+
try:
|
|
76
|
+
if _wal_mode(path):
|
|
77
|
+
raise StorageError(f"Invalid environment state {env!r}: WAL journal mode is unsupported for read-only inspection")
|
|
78
|
+
connection = sqlite3.connect(_sqlite_uri(path), uri=True, timeout=10)
|
|
79
|
+
# One read transaction for the whole inspection: the ledger and prompts must come from the same snapshot,
|
|
80
|
+
# or a concurrent upgrade committing between the two reads would look like drift.
|
|
81
|
+
connection.execute("BEGIN")
|
|
82
|
+
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
83
|
+
if not {"applied_revisions", "prompts"} <= tables:
|
|
84
|
+
raise StorageError(f"Invalid environment state {env!r}: missing required tables")
|
|
85
|
+
except sqlite3.Error as exc:
|
|
86
|
+
raise StorageError(f"Cannot read environment state {env!r}: {exc}") from exc
|
|
87
|
+
yield connection
|
|
88
|
+
finally:
|
|
89
|
+
if connection is not None:
|
|
90
|
+
connection.close()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@contextmanager
|
|
94
|
+
def connect_write(project_dir: Path, env: str) -> Iterator[sqlite3.Connection]:
|
|
95
|
+
"""Initialize writable storage, commit on success, roll back on error, always close."""
|
|
96
|
+
connection = None
|
|
97
|
+
with lock._project_lock(project_dir):
|
|
98
|
+
path = _db_path(project_dir, env)
|
|
99
|
+
path.parent.mkdir(exist_ok=True)
|
|
100
|
+
existed = path.exists()
|
|
101
|
+
connection = None
|
|
102
|
+
try:
|
|
103
|
+
connection = sqlite3.connect(path, timeout=10)
|
|
104
|
+
connection.execute("PRAGMA busy_timeout=10000")
|
|
105
|
+
connection.execute("PRAGMA synchronous=FULL")
|
|
106
|
+
connection.execute("PRAGMA foreign_keys=ON")
|
|
107
|
+
if existed:
|
|
108
|
+
tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
109
|
+
if not {"applied_revisions", "prompts"} <= tables:
|
|
110
|
+
raise StorageError(f"Invalid environment state {env!r}: missing required tables")
|
|
111
|
+
else:
|
|
112
|
+
connection.execute("""
|
|
113
|
+
CREATE TABLE IF NOT EXISTS applied_revisions (
|
|
114
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
115
|
+
revision TEXT UNIQUE NOT NULL,
|
|
116
|
+
checksum TEXT NOT NULL,
|
|
117
|
+
applied_at TEXT NOT NULL
|
|
118
|
+
)
|
|
119
|
+
""")
|
|
120
|
+
connection.execute("""
|
|
121
|
+
CREATE TABLE IF NOT EXISTS prompts (
|
|
122
|
+
name TEXT PRIMARY KEY,
|
|
123
|
+
config TEXT NOT NULL
|
|
124
|
+
)
|
|
125
|
+
""")
|
|
126
|
+
connection.commit()
|
|
127
|
+
except BaseException:
|
|
128
|
+
try:
|
|
129
|
+
if connection is not None:
|
|
130
|
+
try:
|
|
131
|
+
connection.rollback()
|
|
132
|
+
finally:
|
|
133
|
+
connection.close()
|
|
134
|
+
finally:
|
|
135
|
+
if not existed:
|
|
136
|
+
for suffix in ("", "-wal", "-shm"):
|
|
137
|
+
Path(f"{path}{suffix}").unlink(missing_ok=True)
|
|
138
|
+
raise
|
|
139
|
+
assert connection is not None
|
|
140
|
+
try:
|
|
141
|
+
yield connection
|
|
142
|
+
connection.commit()
|
|
143
|
+
except BaseException:
|
|
144
|
+
connection.rollback()
|
|
145
|
+
raise
|
|
146
|
+
finally:
|
|
147
|
+
connection.close()
|