pi-longhaul 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/CONTEXT.md +6 -0
- package/README.md +10 -1
- package/docs/security.md +58 -0
- package/package.json +2 -2
- package/src/credential-scrub.ts +319 -0
- package/src/learned-secrets.ts +1 -1
- package/src/ledger/handoff-commands.ts +2 -0
- package/src/ledger/handoff-errors.ts +19 -5
- package/src/narrative/named-values.ts +14 -4
- package/src/narrative/prompt.ts +6 -4
- package/src/narrative/turns/conversation.ts +3 -2
- package/src/pass.ts +24 -6
- package/src/remember-tool.ts +17 -3
- package/src/span-credentials.ts +194 -0
- package/src/store/project.ts +13 -4
- package/src/store/recall.ts +8 -1
- package/src/tool-context.ts +16 -0
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,38 @@ entries and log event types. Rendered state text, prompt wording and section lay
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [v1.0.4] - 2026-10-22
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- The summarizer copied credentials into the state it wrote: a password from a script's output into Resume, an API
|
|
16
|
+
key or Bearer value into Now, decisions and the narrative. The state is sent with every later request, so the
|
|
17
|
+
value went out again at every turn, and states on real sessions held live ones. What the summarizer writes is now
|
|
18
|
+
scrubbed before it is recorded: each credential value becomes `<redacted>` and the rest of the line stays. A value
|
|
19
|
+
is one when the span showed it as a credential (a credential key's value, symbols and all, a command's password
|
|
20
|
+
argument, a value a secret store printed, `the password is …`), wherever the summarizer put it and however it
|
|
21
|
+
wrapped it; when the text names it
|
|
22
|
+
(`password=…`, `"api_key": "…"`, `Bearer …`, a `user:pass@` login, or a random-looking value just after a
|
|
23
|
+
credential word, so `password (64-hex): <hex>` and `the gateway key (<uuid>)`); or when its shape is one wherever
|
|
24
|
+
it stands (a JWT, a Vault token, AWS, GitHub, OpenAI-style, Slack and age keys). The span's credentials include
|
|
25
|
+
what the previous summary and a switchover's earlier messages showed. A digest (`sha256:…`) is never one, and
|
|
26
|
+
neither is a path, a count, a version or a date the text gives a key, nor, among the span's credentials, a name
|
|
27
|
+
(`app-token`) under a token or secret key, an id or hash its label names (`Evaluation ID:`, `plan_hash`,
|
|
28
|
+
`commit`), the rest of a command after a key whose value it computes (`TOKEN=$(vault kv get … | python3 …)`), a test
|
|
29
|
+
name after `--- PASS:`, or a value after a credential word that is part of a file name (`~/.vault-token`): on real
|
|
30
|
+
sessions the scrub took out the credentials and left versions, hashes, ids and names alone. A state written before this shows redacted from its session's next compaction
|
|
31
|
+
on (the summary, the summarizer's prompt, `longhaul_recall`, `npm run state`), by every rule but the span's own
|
|
32
|
+
credentials, which only the pass that wrote it had. Until then pi keeps sending the summary it stored, and a
|
|
33
|
+
transcript keeps every summary and record it already holds, so a credential found in a state must be rotated. The compaction log records how many values a pass redacted
|
|
34
|
+
(`redacted`), never the values. Both summarizer prompts also tell the model never to copy a credential's value
|
|
35
|
+
and to say where it lives instead, and `longhaul_remember` refuses a decision or dead end that carries one.
|
|
36
|
+
- An error line naming a credential before its value (`ops-control password (64-hex): <value>`) was recorded as an
|
|
37
|
+
open error and shown in every state; no command-style shape matched it. Such a line is now recorded with the value
|
|
38
|
+
redacted, so the failure stays open, and one already carried is redacted at the next compaction.
|
|
39
|
+
- The credential rules found too little: a password key glued to its prefix (`PGPASSWORD=`, `MYSQL_PWD=`), short
|
|
40
|
+
keys (`pw=`, `pass=`), `Authorization: Basic`, a private key's body past its first line, and GitLab, Google API,
|
|
41
|
+
Stripe, npm and Hugging Face tokens, now in both the scrub and the command ledger's credential shapes.
|
|
42
|
+
|
|
11
43
|
## [v1.0.3] - 2026-10-22
|
|
12
44
|
|
|
13
45
|
### Changed
|
package/CONTEXT.md
CHANGED
|
@@ -76,6 +76,12 @@ Values a later turn may need again, verbatim, with where each came from: those t
|
|
|
76
76
|
that the session used again (a hash, a version, a long number).
|
|
77
77
|
_Avoid_: facts (the scorer's measure of a related but different set)
|
|
78
78
|
|
|
79
|
+
**Credential scrub**:
|
|
80
|
+
Taking each credential value out of the text the summarizer writes into the state (the narrative, Now,
|
|
81
|
+
decisions, dead ends), leaving `<redacted>` and the words around it. The value is gone; what it was and where it
|
|
82
|
+
lives can stay.
|
|
83
|
+
_Avoid_: sanitize (one-line.ts: control characters and newlines), redact (the ledger's word for dropping a whole command)
|
|
84
|
+
|
|
79
85
|
## Configuration
|
|
80
86
|
|
|
81
87
|
**Agent dir**:
|
package/README.md
CHANGED
|
@@ -349,7 +349,16 @@ compaction than one that can.
|
|
|
349
349
|
never from a line that looks like it holds a credential. A value the summarizer names is kept only when it
|
|
350
350
|
stands in the turn it cites and no part of it stood on a credential line anywhere in the span; its name is
|
|
351
351
|
shown as the summarizer's label.
|
|
352
|
-
|
|
352
|
+
- What the summarizer writes into the state (the narrative, Now, decisions and dead ends) has each credential
|
|
353
|
+
value replaced by `<redacted>` before it is recorded: a whole value the span showed as a credential (a credential
|
|
354
|
+
key's value, a command's password argument, a secret store's output, `the password is …`), however it is wrapped, a value after a credential key,
|
|
355
|
+
`Bearer` or a credential word, a `user:pass@` password, and token shapes (JWTs, Vault, AWS, GitHub, OpenAI-style,
|
|
356
|
+
Slack, age). States written before this are shown the same way from their next compaction on, by everything but
|
|
357
|
+
the span's own credentials; a stored summary and the transcript keep what they hold, so rotate a credential found
|
|
358
|
+
in a state.
|
|
359
|
+
The prompt also tells the summarizer never to copy one, and `longhaul_remember` refuses a decision or dead end
|
|
360
|
+
that carries one. Error lines are recorded with the value redacted.
|
|
361
|
+
These filters work on patterns: a bare token with no key or secret marker near it can still be learned and
|
|
353
362
|
carried. Treat a session's transcript as you would its tool output.
|
|
354
363
|
|
|
355
364
|
The threat model is in [docs/security.md](docs/security.md).
|
package/docs/security.md
CHANGED
|
@@ -217,6 +217,64 @@ the session (a web page, a README, another agent's message) could try to plant a
|
|
|
217
217
|
replacement. A credential token anywhere in those turns refuses a value of the span as well, and so does one in
|
|
218
218
|
the original words of an edited message, in a message another extension injected, or on a credential line of the
|
|
219
219
|
summary, which is never a source itself.
|
|
220
|
+
- What the summarizer writes into the state (the narrative, every Now item and Resume, decisions, dead ends) is
|
|
221
|
+
scrubbed before the span record is written (`credential-scrub.ts`): each credential value is replaced by
|
|
222
|
+
`<redacted>` and the words around it stay. A value counts when its shape is one wherever it stands (a JWT, a Vault
|
|
223
|
+
token, a private key's whole body, AWS, GitHub, GitLab, Google API, Stripe, npm, Hugging Face, OpenAI-style, Slack
|
|
224
|
+
and age keys); when the text names it, as a credential key's value (`DB_PASSWORD=`, `PGPASSWORD=`, `MYSQL_PWD=`,
|
|
225
|
+
`"api_key":`, escaped quotes too, symbols kept in the value up to a space or a quote; a password word may be glued
|
|
226
|
+
to its prefix, any other only after `_` or `-`, so `compass:` is no key; a value-like or opaque one of twelve
|
|
227
|
+
characters or more, never a path, and a number, count, version or date only as six digits and more under a key
|
|
228
|
+
that says password in full, so `token: missing`, `tests pass: 120/120` and `e2e_pass: 1234567` stay), a value-like
|
|
229
|
+
`Bearer` or `Basic` value, the password of a `user:pass@` URL (to the word's last `@`), or a random-looking value
|
|
230
|
+
(a UUID, hex from 20 characters, a long mixed-case run with digits) starting within 40 characters after a
|
|
231
|
+
credential word; or when it is one of the span's credentials (`span-credentials.ts`): a whole value the span
|
|
232
|
+
itself showed as a credential, found however the summarizer wrapped it (quoted, bold, between dashes or curly
|
|
233
|
+
quotes, after `=`, glued to a flag, in a login with or without a scheme: `app/x@db`, `app:x@db:5432`). The span's
|
|
234
|
+
credentials come from everything the summarizer was shown: the span, the previous summary, a switchover's earlier
|
|
235
|
+
user messages and, in warm mode, the session's last request. They are built from strong evidence only, because each
|
|
236
|
+
one is taken out everywhere: what the text rules above name; the value of a key named for one credential,
|
|
237
|
+
ending in its name or a short part after it (singular, so `tokens: 1200` and a list of `secrets` are not one, and
|
|
238
|
+
no key about one: `password_policy`, `token_env`), any shape under a password key
|
|
239
|
+
(`Summer2024.x`, `unmade-tremor-oxidize`, `48291375`), not a name under a token, secret or key key (`app-token`,
|
|
240
|
+
`APP_TOKEN`, a file or host name); a command's password argument (`mysql -pX`, `sshpass -p X`, `redis-cli -a
|
|
241
|
+
X`, `curl -u user:X`, `htpasswd -b FILE USER X`, `--db-password X`, `openssl -pass pass:X`, `keytool -storepass
|
|
242
|
+
X`); a random-looking word on another credential line; a value a secret store printed alone (or a random-looking
|
|
243
|
+
one among its other output); and `the password is …`, `the db pass is …`. Never a number (but a password's),
|
|
244
|
+
version, date or path, nor one plain lowercase word. A key holds its quoted string or its first word (after Go's
|
|
245
|
+
`:=` or a type, `db_password: str = "…"`; a YAML password's plain value runs to the line's end), never the rest of
|
|
246
|
+
a command line, and nothing when its value is computed (`TOKEN=$(vault kv get … | python3 -c "import sys,json")`
|
|
247
|
+
once made `python3` a credential); a random-looking word the word before it on its line labels as an id or a hash
|
|
248
|
+
(`Evaluation ID:`, `plan_hash`, `commit`) is that, not a secret, unless the label names a credential
|
|
249
|
+
(`secret_id`); and an upper-case `--- PASS: TestName` is a test runner's status, while `pass: 'x'` and `pass=x`
|
|
250
|
+
are keys. A credential word inside a path or file name (`/tmp/admin-bearer.txt`,
|
|
251
|
+
`~/.vault-token`) says where one lives, so the value after it is not taken for one. The broader set that refuses a named value
|
|
252
|
+
(narrative/named-values.ts, with the words after "pin", "card" or "login" and a secret store's whole output)
|
|
253
|
+
redacted versions, commit hashes and names in about a third of real fields, so the scrub does not use it. A
|
|
254
|
+
`sha256:` digest is never one, and a variable or placeholder (`$KEY`, `<from vault>`) stays. The projection applies
|
|
255
|
+
the text rules only, since only the pass that wrote a record had its span, so a record written before the scrub
|
|
256
|
+
shows redacted wherever the state is read from its session's next compaction on: the summary, the summarizer's
|
|
257
|
+
prompt (the open span's remembered items too), `longhaul_recall` (every hit, error lines and the span's narrative
|
|
258
|
+
included) and the state CLI. Nothing rewrites what is stored: pi re-sends the latest compaction's summary string
|
|
259
|
+
until the next compaction (on an older branch, whenever someone returns to it), and the transcript keeps the old
|
|
260
|
+
summaries, `details.state` checkpoints, span records, remember entries and the turns trace's model-written
|
|
261
|
+
reasons. A credential found in a state was sent with every request since; rotate it. `longhaul_remember` refuses
|
|
262
|
+
a decision or dead end that the scrub would change, with the credentials of every message still in context (the
|
|
263
|
+
kept window included). Measured on 24 real transcripts: the text rules redact 22 values over 6612 model-written
|
|
264
|
+
fields, at least one of them not a secret (an Ed25519 public key after "key"); the span's credentials, each span
|
|
265
|
+
against its own messages and the summary before it, redact 7 more over 2981 fields in 224 spans, every one a
|
|
266
|
+
credential or the visible part of one (a Wi-Fi passphrase, two admin passwords, a token's printed prefix, a rotated
|
|
267
|
+
secret's prefix). The summarizer prompts say never to
|
|
268
|
+
copy a credential's value; the scrub is what holds when a model does. An error signature with a command-style
|
|
269
|
+
shape is never recorded; any other is recorded as the scrub leaves it (`password (64-hex): <redacted>`), carried
|
|
270
|
+
ones too, so an ordinary failure that mentions a secret by id stays open. Every pattern is linear on hostile text
|
|
271
|
+
(T4); a private key without an END line is cut 16 KiB on. Not found: a credential the span never showed as one,
|
|
272
|
+
written in prose with no credential word near it; a token or secret key's value shaped as a name; without the
|
|
273
|
+
span, a value on the line after its key (`password:` then the value), a key in smart quotes or before a full-width
|
|
274
|
+
colon; a password given to `docker login -p` or piped to a `--password-stdin` command; a passphrase with spaces
|
|
275
|
+
(`correct horse battery staple`); a key about a password rather than holding one (`password_policy:`) gives no
|
|
276
|
+
value, by design; and a symbol at a value's edge (`Summer2024!` leaves `<redacted>!`). The user's own words (objective,
|
|
277
|
+
done-when and correction quotes) are verbatim by design and not scrubbed.
|
|
220
278
|
- The ledger redacts commands that carry a secret, by the same patterns (a one-time-password seed passed to
|
|
221
279
|
`oathtool -b` (or `-bv`, `--base32`, or a hex or base32 key of 16 or more characters), `ykman oath accounts add`,
|
|
222
280
|
`ykman ... --key`, `sshpass -p`, `htpasswd -b`, `openssl -pass pass:`, `keytool -storepass`, `redis-cli -a`,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-longhaul",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"description": "Keep pi sessions working all night. A compactor that carries your session forward instead of rewriting it: your rules kept in your own words, the state written by code, old reasoning left behind, and about 5x faster than pi's own compaction on a local model.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"extensions": [
|
|
33
33
|
"./src/index.ts"
|
|
34
34
|
],
|
|
35
|
-
"image": "https://raw.githubusercontent.com/lobo235/pi-longhaul/v1.0.
|
|
35
|
+
"image": "https://raw.githubusercontent.com/lobo235/pi-longhaul/v1.0.4/.github/assets/infographic.png"
|
|
36
36
|
},
|
|
37
37
|
"files": [
|
|
38
38
|
"src/**/*.ts",
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Credential scrub (CONTEXT.md *Credential scrub*; security.md T3): text a model wrote into the state (the narrative,
|
|
3
|
+
* Now, decisions, dead ends) and the open-error lines, with each credential value replaced by REDACTED_VALUE. The
|
|
4
|
+
* state is rendered into every later request, so a password the summarizer copied out of a tool's output was sent
|
|
5
|
+
* again at every turn. Only the value goes: the words around it stay, so a Resume line still says what to do next.
|
|
6
|
+
*
|
|
7
|
+
* A value is a credential when
|
|
8
|
+
* - its shape is one wherever it stands: a JWT, a Vault token, an AWS, GitHub, OpenAI-style, Slack or age key;
|
|
9
|
+
* - the text names it: the value of a credential key (`password=x`, `"api_key": "x"`), a Bearer value, the password
|
|
10
|
+
* of a `user:pass@` URL, or a random-looking value within CONTEXT_REACH characters after a credential word
|
|
11
|
+
* (`password (64-hex): <hex>`, `the gateway key (<uuid>)`);
|
|
12
|
+
* - or it is one of the span's credentials (span-credentials.ts): a whole value the span itself showed as one
|
|
13
|
+
* (`DB_PASSWORD=Ab3$x;9Kq2`, symbols and all), wherever the model put it and however it wrapped it (quoted, bold,
|
|
14
|
+
* between dashes, glued to a flag or a login).
|
|
15
|
+
* A digest (`sha256:<hex>`) is never one. Every pattern is linear: a summarizer's answer is untrusted (security.md T4).
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { StateEvent } from "./store/events.ts";
|
|
19
|
+
|
|
20
|
+
/** What a redacted value is replaced with. Holds no token character, so a second scrub leaves it alone. */
|
|
21
|
+
export const REDACTED_VALUE = "<redacted>";
|
|
22
|
+
|
|
23
|
+
/** The summarizer prompts' rule (narrative/prompt.ts, narrative/turns/conversation.ts); the scrub is what holds when a model ignores it. */
|
|
24
|
+
export const CREDENTIAL_RULE = 'Never copy a password, key, token or other credential\'s value into the state, in any field: say what it is and where it lives ("the gateway key from its Vault secret"), never the value itself.';
|
|
25
|
+
|
|
26
|
+
/** The span's credentials: whole values, each one judged a credential where the span showed it (span-credentials.ts). */
|
|
27
|
+
export type SpanCredentials = ReadonlySet<string>;
|
|
28
|
+
|
|
29
|
+
export interface Scrubbed {
|
|
30
|
+
text: string;
|
|
31
|
+
/** How many values were replaced; logged as a count, never the values. */
|
|
32
|
+
redacted: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A token: what a value is cut on, as named-values.ts TOKEN_SPLIT cuts a credential line. */
|
|
36
|
+
const TOKEN = /[^\s=:"'`,;{}[\]()<>]+/g;
|
|
37
|
+
/** A word: what a whole value is cut on. Space, and the dashes and quotes prose puts around a value (`—x—`, `“x”`). */
|
|
38
|
+
export const WORD = /[^\s\u2013\u2014\u2018\u2019\u201c\u201d\u00ab\u00bb]+/g;
|
|
39
|
+
/** How far after a credential word a random-looking value counts as its value. */
|
|
40
|
+
const CONTEXT_REACH = 40;
|
|
41
|
+
|
|
42
|
+
/** Shapes that are credentials wherever they stand (ledger/handoff-commands.ts SECRET_PATTERNS, and learned-secrets.ts TOKEN_SHAPES). */
|
|
43
|
+
const CREDENTIAL_SHAPES = [
|
|
44
|
+
/(?<![\w-])eyJ[\w-]{4,}\.[\w-]{4,}\.[\w-]*/g,
|
|
45
|
+
/(?<![\w-])hv[sbr]\.[\w-]{8,}/g,
|
|
46
|
+
/\bAGE-SECRET-KEY-1[0-9A-Z]{20,}/g,
|
|
47
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
48
|
+
/\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[abprs]-[A-Za-z0-9-]{10,})/g,
|
|
49
|
+
// GitLab, Google API, Stripe, npm and Hugging Face tokens; as ledger/handoff-commands.ts SECRET_PATTERNS, kept in step.
|
|
50
|
+
/\b(?:glpat-[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35}|[sr]k_(?:live|test)_[A-Za-z0-9]{16,}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z]{30,})/g,
|
|
51
|
+
];
|
|
52
|
+
const PEM_BEGIN = /-----BEGIN [A-Z ]{0,64}PRIVATE KEY(?: BLOCK)?-----/g;
|
|
53
|
+
const PEM_END = "-----END ";
|
|
54
|
+
/** A private key's body without an END line is cut here. */
|
|
55
|
+
const PEM_CHARS = 16_384;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The key part names a credential. A password word may be glued to its prefix (`PGPASSWORD`, `MYSQL_PWD`: a lazy
|
|
59
|
+
* bounded prefix from a word start, at most 48 tries per start, linear, security.md T4); any other word only after
|
|
60
|
+
* `_` or `-` (`DB_PASS`, `client-secret`), so `compass` and `bypass` are no key. No plural `tokens` (a count). A bare
|
|
61
|
+
* `pass` is one only before `=` or a quoted value (`pass=`, `pass: 'x'`): `--- PASS: TestName` is a test runner's status.
|
|
62
|
+
*/
|
|
63
|
+
const CREDENTIAL_KEY = String.raw`(?<![A-Za-z0-9])(?:[A-Za-z0-9_-]{0,48}?(?:passw(?:or)?ds?|passwd|pwd)|(?:[A-Za-z0-9]{1,32}[_-]){1,3}pass|pass(?=(?:\\{0,3}["'])?[ \t]{0,8}(?:=|:[ \t]{0,8}\\{0,3}["']))|(?:[A-Za-z0-9]{1,32}[_-]){0,3}(?:pw|passphrases?|passcodes?|secrets?|token|api[_-]?keys?|access[_-]?keys?|private[_-]?keys?|credentials?))`;
|
|
64
|
+
/**
|
|
65
|
+
* A key that ends in a password word in full, whose value may be all digits (`ROOT_PASSWORD=48291375`); not `pass`
|
|
66
|
+
* or `pw` (`e2e_pass: 1234567`), nor a key about passwords (`password_min_length: 123456`).
|
|
67
|
+
*/
|
|
68
|
+
const NUMERIC_PASSWORD_KEY = /(?:passw(?:or)?d|passwd|pwd|passphrase|passcode)s?$/i;
|
|
69
|
+
/** Whether `key` (a key part, its quotes and separator included or not) says password in full at its end. */
|
|
70
|
+
export const isNumericPasswordKey = (key: string) => NUMERIC_PASSWORD_KEY.test(withoutTrailing(key, "\"' \t=:\\"));
|
|
71
|
+
/** `key = value`, `key: value`, quoted as JSON or not, escaped quotes included; the value is group 2, up to a space or a quote, so symbols in it stay in it. */
|
|
72
|
+
const KEYED_VALUE = new RegExp(String.raw`(${CREDENTIAL_KEY}(?:\\{0,3}["'])?[ \t]{0,8}[=:][ \t]{0,8}(?:\\{0,3}["'])?)([^\s"'\x60\\]{3,})`, "gi");
|
|
73
|
+
const BEARER_VALUE = /(\b(?:bearer|basic)\s+)([^\s"'`,;{}[\]()<>]{8,})/gi;
|
|
74
|
+
/** A URL login's prefix, `scheme://user:` (the user may be empty); its password runs to the word's last `@` (urlPasswordSpans). */
|
|
75
|
+
const URL_LOGIN = /[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,512}:/gi;
|
|
76
|
+
const URL_PASSWORD_CHARS = 512;
|
|
77
|
+
/** A credential named in prose: a random-looking value after it is its value. */
|
|
78
|
+
const CREDENTIAL_WORD = /(?<![A-Za-z])(?:passw(?:or)?ds?|passwd|pwd|passphrases?|passcodes?|secrets?|tokens?|api[_ -]?keys?|keys?|bearer|credentials?|auth)(?![A-Za-z])/gi;
|
|
79
|
+
|
|
80
|
+
/** A value the text points at rather than holds: a variable, a placeholder, a mask. */
|
|
81
|
+
const REFERENCE = /^(?:\$|%|\*|\{|<|\.\.\.)/;
|
|
82
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
83
|
+
/** As named-values.ts RANDOM_LOOKING: long hex with a digit and a letter, a long mixed-case run with digits, 13+ digits. */
|
|
84
|
+
const RANDOM_LOOKING = [/^\d{13,}$/, /^(?=[0-9a-f]*\d)(?=[0-9a-f]*[a-f])[0-9a-f]{13,}$/i, /^(?=[A-Za-z0-9_+/-]*[a-z])(?=[A-Za-z0-9_+/-]*[A-Z])(?=[A-Za-z0-9_+/-]*\d)[A-Za-z0-9_+/-]{16,}={0,2}$/];
|
|
85
|
+
const DIGEST_PREFIX = /sha(?:1|224|256|384|512):$/i;
|
|
86
|
+
|
|
87
|
+
export const isRandomLooking = (token: string) => UUID.test(token) || RANDOM_LOOKING.some((shape) => shape.test(token));
|
|
88
|
+
/** A token a credential could be: random-looking, or six characters or more with a digit. */
|
|
89
|
+
export const isValueLike = (token: string) => isRandomLooking(token) || (token.length >= 6 && /\d/.test(token));
|
|
90
|
+
/** A value a credential key holds: value-like, or opaque (twelve characters or more), never a path (`PWD=/home/app`); a word is a status ("token: missing"). A Bearer value must be value-like: "Bearer authentication" is prose. */
|
|
91
|
+
const isHeldValue = (token: string) => (isValueLike(token) || token.length >= 12) && !/^(?:\/|~\/|\.\.?\/)/.test(token);
|
|
92
|
+
/** A number, a count (`120/120`), a version, a time or a date: no credential, unless a password key holds it. */
|
|
93
|
+
const NUMBERISH = /^(?:v?\d+(?:[.:_/-]\d+)*|\d{4}-\d{2}-\d{2}(?:[T ][\d:.]+Z?)?)$/i;
|
|
94
|
+
export const isNumberish = (token: string) => NUMBERISH.test(token);
|
|
95
|
+
/** A credential key's value (isHeldValue); a number only under a password key, so `tests pass: 120/120` stays. */
|
|
96
|
+
const isKeyedHeld = (token: string, key: string) => {
|
|
97
|
+
const bare = withoutTrailing(token, ",.;:!?)]}");
|
|
98
|
+
return isHeldValue(token) && (!isNumberish(bare) || (isNumericPasswordKey(key) && /^\d{6,}$/.test(bare)));
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* A value prose names only by a nearby credential word: random-looking, hex only from LONG_HEX characters, since a
|
|
102
|
+
* plan or object id near "secret" is 16 (measured on real states: plan hashes, short ids).
|
|
103
|
+
*/
|
|
104
|
+
const LONG_HEX = 20;
|
|
105
|
+
const isNamedByContext = (token: string) => isRandomLooking(token) && !(/^[0-9a-f]+$/i.test(token) && token.length < LONG_HEX);
|
|
106
|
+
|
|
107
|
+
/** `token` without the trailing `marks`. A loop, not `/[.!?]+$/`: that regex retries from every mark of a long run (quadratic, security.md T4). */
|
|
108
|
+
function withoutTrailing(token: string, marks: string): string {
|
|
109
|
+
let end = token.length;
|
|
110
|
+
while (end > 0 && marks.includes(token[end - 1])) end--;
|
|
111
|
+
return token.slice(0, end);
|
|
112
|
+
}
|
|
113
|
+
/** `token` without the sentence marks that end it. */
|
|
114
|
+
const withoutClosingMarks = (token: string) => withoutTrailing(token, ".!?");
|
|
115
|
+
|
|
116
|
+
interface Span { start: number; end: number }
|
|
117
|
+
|
|
118
|
+
function shapeSpans(text: string): Span[] {
|
|
119
|
+
const spans: Span[] = [];
|
|
120
|
+
for (const shape of CREDENTIAL_SHAPES) for (const match of text.matchAll(shape)) spans.push({ start: match.index, end: match.index + match[0].length });
|
|
121
|
+
return [...spans, ...pemSpans(text)];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** A private key from its BEGIN line through its END line, or PEM_CHARS on when it has none. One scan: each key starts after the last. */
|
|
125
|
+
function pemSpans(text: string): Span[] {
|
|
126
|
+
const spans: Span[] = [];
|
|
127
|
+
let from = 0;
|
|
128
|
+
for (const begin of text.matchAll(PEM_BEGIN)) {
|
|
129
|
+
if (begin.index < from) continue;
|
|
130
|
+
// END is looked for inside the key's window only: the rest of the text, once per key, is quadratic (security.md T4).
|
|
131
|
+
const window = text.slice(begin.index, begin.index + PEM_CHARS);
|
|
132
|
+
const end = window.indexOf(PEM_END, begin[0].length);
|
|
133
|
+
const close = end < 0 ? -1 : window.indexOf("-----", end + PEM_END.length);
|
|
134
|
+
const stop = begin.index + (close < 0 ? window.length : close + 5);
|
|
135
|
+
spans.push({ start: begin.index, end: stop });
|
|
136
|
+
from = stop;
|
|
137
|
+
}
|
|
138
|
+
return spans;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The value group (2) of each match of `pattern`, unless it only points at a value or `holds` refuses it (given the key part, group 1). */
|
|
142
|
+
function groupSpans(text: string, pattern: RegExp, holds: (value: string, key: string) => boolean = () => true): Span[] {
|
|
143
|
+
const spans: Span[] = [];
|
|
144
|
+
for (const match of text.matchAll(pattern)) {
|
|
145
|
+
if (REFERENCE.test(match[2]) || !holds(match[2], match[1])) continue;
|
|
146
|
+
const start = match.index + match[1].length;
|
|
147
|
+
spans.push({ start, end: start + match[2].length });
|
|
148
|
+
}
|
|
149
|
+
return spans;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Whether the credential word at [start, end) is part of a path or a file name (`/tmp/admin-bearer.txt`,
|
|
154
|
+
* `token.json`), which says where a credential lives, not that a value follows. Looks at most FILE_NAME_REACH back.
|
|
155
|
+
*/
|
|
156
|
+
const FILE_NAME_REACH = 256;
|
|
157
|
+
function inFileName(text: string, start: number, end: number): boolean {
|
|
158
|
+
const before = text.slice(Math.max(0, start - FILE_NAME_REACH), start);
|
|
159
|
+
return /\/[^\s/]*$/.test(before) || /^[\w-]*\.[A-Za-z][A-Za-z0-9]{0,4}(?![\w-])/.test(text.slice(end, end + FILE_NAME_REACH));
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Tokens named by context (isNamedByContext) that start within CONTEXT_REACH characters after a credential word, digests excepted. */
|
|
163
|
+
function contextSpans(text: string, tokens: ReadonlyArray<{ token: string; start: number }>): Span[] {
|
|
164
|
+
const spans: Span[] = [];
|
|
165
|
+
let index = 0;
|
|
166
|
+
for (const word of text.matchAll(CREDENTIAL_WORD)) {
|
|
167
|
+
const after = word.index + word[0].length;
|
|
168
|
+
if (inFileName(text, word.index, after)) continue;
|
|
169
|
+
while (index < tokens.length && tokens[index].start < after) index++;
|
|
170
|
+
for (let next = index; next < tokens.length && tokens[next].start <= after + CONTEXT_REACH; next++) {
|
|
171
|
+
const { token, start } = tokens[next];
|
|
172
|
+
const bare = withoutClosingMarks(token);
|
|
173
|
+
if (isNamedByContext(bare) && !DIGEST_PREFIX.test(text.slice(Math.max(0, start - 7), start))) spans.push({ start, end: start + bare.length });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return spans;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The password of each `scheme://user:pass@host`: up to the last `@` of the word, so a password holding one is taken
|
|
181
|
+
* whole (`app:p@ss1234@db`). A port followed by a path is no login. Each login looks at most URL_PASSWORD_CHARS on.
|
|
182
|
+
*/
|
|
183
|
+
function urlPasswordSpans(text: string): Span[] {
|
|
184
|
+
const spans: Span[] = [];
|
|
185
|
+
for (const match of text.matchAll(URL_LOGIN)) {
|
|
186
|
+
const start = match.index + match[0].length;
|
|
187
|
+
const window = text.slice(start, start + URL_PASSWORD_CHARS + 1);
|
|
188
|
+
const space = window.search(/\s/);
|
|
189
|
+
const word = space < 0 ? window : window.slice(0, space);
|
|
190
|
+
const at = word.lastIndexOf("@");
|
|
191
|
+
if (at <= 0 || /^\d{1,5}\//.test(word) || REFERENCE.test(word)) continue;
|
|
192
|
+
spans.push({ start, end: start + at });
|
|
193
|
+
}
|
|
194
|
+
return spans;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const isWordChar = (char: string) => /[\p{L}\p{N}]/u.test(char);
|
|
198
|
+
|
|
199
|
+
/** `word` without what wraps it (quotes, `**`, brackets, a closing mark), as offsets into `word`; empty when nothing is left. */
|
|
200
|
+
export function credentialCore(word: string): { start: number; end: number } {
|
|
201
|
+
let start = 0;
|
|
202
|
+
let end = word.length;
|
|
203
|
+
while (start < end && !isWordChar(word[start])) start++;
|
|
204
|
+
while (end > start && !isWordChar(word[end - 1])) end--;
|
|
205
|
+
return { start, end };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Where a value may stand in a word, widest first: the word, after its first or last `=` or `:` (`DB_PASSWORD='x'`),
|
|
210
|
+
* after a short flag (`-px`), and each part between `/` and `@` (`app/x@db`), also after that part's first `:`
|
|
211
|
+
* (`app:x@db:5432`). Linear in the word.
|
|
212
|
+
*/
|
|
213
|
+
function placesIn(word: string): Array<[start: number, end: number]> {
|
|
214
|
+
const places: Array<[number, number]> = [[0, word.length]];
|
|
215
|
+
const first = word.search(/[=:]/);
|
|
216
|
+
const last = Math.max(word.lastIndexOf("="), word.lastIndexOf(":"));
|
|
217
|
+
if (first >= 0) places.push([first + 1, word.length]);
|
|
218
|
+
if (last > first) places.push([last + 1, word.length]);
|
|
219
|
+
const flag = /^-{1,2}[A-Za-z]/.exec(word);
|
|
220
|
+
if (flag) places.push([flag[0].length, word.length]);
|
|
221
|
+
// The next `:` at or after a part's start; the pointer only moves forward, so the scan stays linear.
|
|
222
|
+
let colon = -1;
|
|
223
|
+
const part = (from: number, to: number) => {
|
|
224
|
+
places.push([from, to]);
|
|
225
|
+
if (colon < from) {
|
|
226
|
+
const next = word.indexOf(":", from);
|
|
227
|
+
colon = next < 0 ? Infinity : next;
|
|
228
|
+
}
|
|
229
|
+
if (colon < to) places.push([colon + 1, to]);
|
|
230
|
+
};
|
|
231
|
+
let from = 0;
|
|
232
|
+
for (const separator of word.matchAll(/[/@]/g)) {
|
|
233
|
+
part(from, separator.index);
|
|
234
|
+
from = separator.index + 1;
|
|
235
|
+
}
|
|
236
|
+
if (from > 0) part(from, word.length);
|
|
237
|
+
return places;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Each span value in `text`, wrapping trimmed (credentialCore), at the widest place in its word that holds it. */
|
|
241
|
+
function valueSpans(text: string, values: SpanCredentials): Span[] {
|
|
242
|
+
const spans: Span[] = [];
|
|
243
|
+
for (const match of text.matchAll(WORD)) {
|
|
244
|
+
const word = match[0];
|
|
245
|
+
for (const [from, to] of placesIn(word)) {
|
|
246
|
+
const core = credentialCore(word.slice(from, to));
|
|
247
|
+
if (core.end > core.start && values.has(word.slice(from + core.start, from + core.end))) {
|
|
248
|
+
spans.push({ start: match.index + from + core.start, end: match.index + from + core.end });
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return spans;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** `text` with every span replaced, overlapping spans as one. */
|
|
257
|
+
function replaceSpans(text: string, spans: Span[]): Scrubbed {
|
|
258
|
+
if (!spans.length) return { text, redacted: 0 };
|
|
259
|
+
spans.sort((a, b) => a.start - b.start);
|
|
260
|
+
let out = "";
|
|
261
|
+
let cursor = 0;
|
|
262
|
+
let redacted = 0;
|
|
263
|
+
for (const span of spans) {
|
|
264
|
+
if (span.end <= cursor) continue;
|
|
265
|
+
if (span.start >= cursor) {
|
|
266
|
+
out += text.slice(cursor, span.start) + REDACTED_VALUE;
|
|
267
|
+
redacted++;
|
|
268
|
+
}
|
|
269
|
+
cursor = span.end;
|
|
270
|
+
}
|
|
271
|
+
return { text: out + text.slice(cursor), redacted };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** What the text itself names as a credential: a shape, a credential key's value, a Bearer value, a URL password, a value after a credential word. */
|
|
275
|
+
function namedSpans(text: string): Span[] {
|
|
276
|
+
const tokens = [...text.matchAll(TOKEN)].map((match) => ({ token: match[0], start: match.index }));
|
|
277
|
+
return [
|
|
278
|
+
...shapeSpans(text),
|
|
279
|
+
...groupSpans(text, KEYED_VALUE, isKeyedHeld),
|
|
280
|
+
...groupSpans(text, BEARER_VALUE, isValueLike),
|
|
281
|
+
...urlPasswordSpans(text),
|
|
282
|
+
...contextSpans(text, tokens),
|
|
283
|
+
];
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** The values `text` names as credentials, wrapping trimmed: what scrubCredentials takes out of it on its own. */
|
|
287
|
+
export function namedCredentialValues(text: string): string[] {
|
|
288
|
+
return namedSpans(text).map((span) => {
|
|
289
|
+
const value = text.slice(span.start, span.end);
|
|
290
|
+
const core = credentialCore(value);
|
|
291
|
+
return value.slice(core.start, core.end);
|
|
292
|
+
}).filter(Boolean);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* `text` with its credential values replaced. `span` is the span's credentials; without them (text read back from a
|
|
297
|
+
* span record already written) only what the text itself names is found.
|
|
298
|
+
*/
|
|
299
|
+
export function scrubCredentials(text: string, span?: SpanCredentials): Scrubbed {
|
|
300
|
+
return replaceSpans(text, [...namedSpans(text), ...(span?.size ? valueSpans(text, span) : [])]);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** What a pass wrote, scrubbed: the events whose text the summarizer chose, and the span's narrative. */
|
|
304
|
+
export function scrubModelWritten(events: readonly StateEvent[], narrative: string | undefined, span: SpanCredentials): { events: StateEvent[]; narrative?: string; redacted: number } {
|
|
305
|
+
let redacted = 0;
|
|
306
|
+
const clean = (text: string) => {
|
|
307
|
+
const scrubbed = scrubCredentials(text, span);
|
|
308
|
+
redacted += scrubbed.redacted;
|
|
309
|
+
return scrubbed.text;
|
|
310
|
+
};
|
|
311
|
+
const scrubbedEvents = events.map((event): StateEvent => {
|
|
312
|
+
switch (event.kind) {
|
|
313
|
+
case "decision.add": case "deadEnd.add": return { ...event, text: clean(event.text) };
|
|
314
|
+
case "now.set": return { ...event, now: { inProgress: event.now.inProgress.map(clean), next: event.now.next.map(clean), blocked: event.now.blocked.map(clean), resume: clean(event.now.resume) } };
|
|
315
|
+
default: return event;
|
|
316
|
+
}
|
|
317
|
+
});
|
|
318
|
+
return { events: scrubbedEvents, ...(narrative === undefined ? {} : { narrative: clean(narrative) }), redacted };
|
|
319
|
+
}
|
package/src/learned-secrets.ts
CHANGED
|
@@ -27,7 +27,7 @@ const KEY_CHARS = 64;
|
|
|
27
27
|
const BLOCK_OPENER = /^(?:[|>](?:[-+]?\d?|\d[-+])|[{[])?\s*(?:#.*)?$/;
|
|
28
28
|
|
|
29
29
|
/** A line read as `key = value`, `key: value` or `key value` (aligned table), or undefined when it is not one. */
|
|
30
|
-
function keyedLine(line: string): { key: string; value: string } | undefined {
|
|
30
|
+
export function keyedLine(line: string): { key: string; value: string } | undefined {
|
|
31
31
|
const text = line.trim();
|
|
32
32
|
const separator = text.slice(0, KEY_CHARS + 1).search(/[=:]|\s{2,}|\t/);
|
|
33
33
|
if (separator <= 0) return undefined;
|
|
@@ -74,6 +74,8 @@ const SECRET_PATTERNS: RegExp[] = [
|
|
|
74
74
|
/\bAGE-SECRET-KEY-1[0-9A-Z]{20,}/,
|
|
75
75
|
/\bAKIA[0-9A-Z]{16}\b/,
|
|
76
76
|
/\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[abprs]-[A-Za-z0-9-]{10,})/,
|
|
77
|
+
// As credential-scrub.ts CREDENTIAL_SHAPES: GitLab, Google API, Stripe, npm and Hugging Face tokens.
|
|
78
|
+
/\b(?:glpat-[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35}|[sr]k_(?:live|test)_[A-Za-z0-9]{16,}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z]{30,})/,
|
|
77
79
|
];
|
|
78
80
|
|
|
79
81
|
/** Whether `text` (a command, or a tool result's failure signature) carries a credential shape. */
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import type { ErrorRecord } from "./handoff.ts";
|
|
8
8
|
import { containsSecret, normalizeCommand, redactCommand } from "./handoff-commands.ts";
|
|
9
9
|
import type { ObservedError, ObservedRun } from "./handoff-observe.ts";
|
|
10
|
+
import { scrubCredentials } from "../credential-scrub.ts";
|
|
10
11
|
|
|
11
12
|
const VOLATILE_RES: RegExp[] = [
|
|
12
13
|
/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
|
|
@@ -24,6 +25,14 @@ export function errorKey(signature: string): string {
|
|
|
24
25
|
return line.replace(/\s+/g, " ").trim();
|
|
25
26
|
}
|
|
26
27
|
|
|
28
|
+
/**
|
|
29
|
+
* The signature a failure is recorded under: none when it has a command-style credential shape (dropped, as ever),
|
|
30
|
+
* else with each value the credential scrub finds redacted (`password (64-hex): <redacted>`). Redacted, not dropped:
|
|
31
|
+
* an ordinary failure ("secret db-creds version <uuid> not found") must stay open. The scrub is deterministic, so the
|
|
32
|
+
* same failure keys the same way pass after pass.
|
|
33
|
+
*/
|
|
34
|
+
const recordedSignature = (signature: string) => containsSecret(signature) ? undefined : scrubCredentials(signature).text;
|
|
35
|
+
|
|
27
36
|
function idNumber(id: string): number {
|
|
28
37
|
const match = /^E(\d+)$/.exec(id);
|
|
29
38
|
return match ? Number(match[1]) : 0;
|
|
@@ -40,23 +49,28 @@ function succeededAfter(runs: ObservedRun[], command: string, after: number): bo
|
|
|
40
49
|
* unresolved again and last seen now; a new one gets the next id. Then the mechanical rule: a
|
|
41
50
|
* record whose source command exits 0 later in this pass is resolved. Records are copied. A
|
|
42
51
|
* signature that carries a credential shape (a failing push or curl echoes its URL or header) is
|
|
43
|
-
* never recorded, carried or new, because the exact half outlives the messages it came from;
|
|
52
|
+
* never recorded, carried or new, because the exact half outlives the messages it came from; any
|
|
53
|
+
* other is recorded with the values the credential scrub finds redacted (recordedSignature); a
|
|
44
54
|
* source command with one is stored as `<redacted>`, like its command record. Ids are never reused.
|
|
45
55
|
*/
|
|
46
56
|
export function mergeErrors(previous: ErrorRecord[], observed: ObservedError[], runs: ObservedRun[], pass: number): ErrorRecord[] {
|
|
47
57
|
let nextId = previous.reduce((max, record) => Math.max(max, idNumber(record.id)), 0) + 1;
|
|
48
|
-
const records = previous.
|
|
58
|
+
const records = previous.flatMap((record) => {
|
|
59
|
+
const signature = recordedSignature(record.signature);
|
|
60
|
+
return signature === undefined ? [] : [{ ...record, signature, sourceCommand: redactCommand(record.sourceCommand) }];
|
|
61
|
+
});
|
|
49
62
|
const lastSeenAt = new Map<string, number>();
|
|
50
63
|
for (const error of observed) {
|
|
51
|
-
const
|
|
52
|
-
|
|
64
|
+
const signature = recordedSignature(error.signature);
|
|
65
|
+
const key = signature === undefined ? "" : errorKey(signature);
|
|
66
|
+
if (signature === undefined || key === "") continue;
|
|
53
67
|
let record = records.find((candidate) => errorKey(candidate.signature) === key);
|
|
54
68
|
if (record) {
|
|
55
69
|
record.resolved = false;
|
|
56
70
|
record.lastSeenPass = pass;
|
|
57
71
|
if (record.sourceCommand === undefined && error.sourceCommand !== undefined) record.sourceCommand = redactCommand(error.sourceCommand);
|
|
58
72
|
} else {
|
|
59
|
-
record = { id: `E${nextId++}`, signature
|
|
73
|
+
record = { id: `E${nextId++}`, signature, sourceCommand: redactCommand(error.sourceCommand), resolved: false, lastSeenPass: pass };
|
|
60
74
|
records.push(record);
|
|
61
75
|
}
|
|
62
76
|
lastSeenAt.set(record.id, error.runIndex);
|
|
@@ -45,7 +45,7 @@ function textsIn(message: Rec): string[] {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
/** Everything a message carries as text, call arguments string by string (so a written file is read line by line). */
|
|
48
|
-
function everyTextIn(message: Rec): string[] {
|
|
48
|
+
export function everyTextIn(message: Rec): string[] {
|
|
49
49
|
const out: string[] = [];
|
|
50
50
|
if (typeof message.content === "string") out.push(message.content);
|
|
51
51
|
if (Array.isArray(message.content)) {
|
|
@@ -89,6 +89,18 @@ export interface SpanTexts {
|
|
|
89
89
|
holderOf?: (value: string) => string | undefined;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Every token that stood on a credential line, after a credential named in prose, in a `user:pass@` login or in a
|
|
94
|
+
* secret store's output anywhere in `messages`: what a value is refused for. Broad on purpose, since refusing a value
|
|
95
|
+
* costs little; the credential scrub uses the stricter span-credentials.ts, as it takes each one out everywhere.
|
|
96
|
+
*/
|
|
97
|
+
function credentialTokensIn(messages: readonly AgentMessage[]): Set<string> {
|
|
98
|
+
const tainted = new Set<string>();
|
|
99
|
+
for (const message of messages as unknown as Rec[]) for (const text of everyTextIn(message)) for (const token of credentialTokensOf(text)) tainted.add(token);
|
|
100
|
+
for (const output of secretStoreOutputs(messages as unknown as Rec[])) for (const token of tokensOf(output)) tainted.add(token);
|
|
101
|
+
return tainted;
|
|
102
|
+
}
|
|
103
|
+
|
|
92
104
|
/**
|
|
93
105
|
* Each turn's text, by the ordinal tag prepare.ts puts on its user message (thinking was stripped before), and the
|
|
94
106
|
* span's credential tokens. Those are taken from every message, sources or not: a password a web page showed is no
|
|
@@ -110,14 +122,12 @@ export function spanTextsOf(messages: AgentMessage[]): SpanTexts {
|
|
|
110
122
|
*/
|
|
111
123
|
export function turnTextsOf(messages: ReadonlyArray<{ ordinal?: string; message: AgentMessage; taintOnly?: boolean }>): SpanTexts {
|
|
112
124
|
const turns = new Map<string, string[]>();
|
|
113
|
-
const tainted = new Set<string>();
|
|
114
125
|
let current: string[] | undefined;
|
|
115
126
|
for (const { ordinal, message, taintOnly } of messages as unknown as ReadonlyArray<{ ordinal?: string; message: Rec; taintOnly?: boolean }>) {
|
|
116
|
-
for (const text of everyTextIn(message)) for (const token of credentialTokensOf(text)) tainted.add(token);
|
|
117
127
|
if (ordinal) turns.set(ordinal, (current = []));
|
|
118
128
|
if (!taintOnly) current?.push(...textsIn(message));
|
|
119
129
|
}
|
|
120
|
-
|
|
130
|
+
const tainted = credentialTokensIn(messages.map((entry) => entry.message));
|
|
121
131
|
const joined = new Map([...turns].map(([ordinal, texts]) => [ordinal, texts.join("\n")]));
|
|
122
132
|
// Each turn's clean text once, newest first: a value that misses is looked up again on every attempt.
|
|
123
133
|
let clean: Array<[string, string]> | undefined;
|
package/src/narrative/prompt.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
8
8
|
|
|
9
|
+
import { CREDENTIAL_RULE, scrubCredentials } from "../credential-scrub.ts";
|
|
9
10
|
import { normalizeWs } from "../intent.ts";
|
|
10
11
|
import { quoteText } from "../render.ts";
|
|
11
12
|
import { DEFAULT_CAPS, type Now, type Quote, type StateEvent } from "../store/events.ts";
|
|
@@ -35,7 +36,7 @@ import { LABEL_FORMS, LIFT_GUIDANCE, RULE_DEFINITION, RULE_WHOLE_CHARS } from ".
|
|
|
35
36
|
* state-16: outside a /goal, the session's opening request is the goal line once the Request has moved from it
|
|
36
37
|
* (store/project.ts goalLine), so the project's purpose survives every compaction; never quoted as the objective.
|
|
37
38
|
*/
|
|
38
|
-
export const NARRATIVE_PROMPT_REVISION = "state-
|
|
39
|
+
export const NARRATIVE_PROMPT_REVISION = "state-17";
|
|
39
40
|
|
|
40
41
|
export const NARRATIVE_SYSTEM_PROMPT = `You maintain the SESSION STATE of a coding session that is being compacted. You are given the state so far, the user's messages with their ordinals (u1, u2, ...), and the conversation being compacted. Answer with ONE fenced json block and nothing else:
|
|
41
42
|
|
|
@@ -52,7 +53,7 @@ export const NARRATIVE_SYSTEM_PROMPT = `You maintain the SESSION STATE of a codi
|
|
|
52
53
|
}
|
|
53
54
|
\`\`\`
|
|
54
55
|
|
|
55
|
-
Rules. "objective" is the request the work is answering now. It is optional: include it only when none is set or the user moved to a different request in this span, and quote the user's own message verbatim. The goal line, when present, is the active /goal or the session's opening request: code keeps it, so never quote it as the objective; under a goal the objective is the specific ask. "labels": Label EVERY message in <user-messages> with one of ${LABEL_FORMS}. ${RULE_DEFINITION} ${LIFT_GUIDANCE} The target is R<n> from the state, or u<n> for a rule given earlier in this span. "not a rule" is a request, question, answer, approval or one-off. Code copies the user's words into the state, so a short message labeled a rule needs no quote; a message longer than ${RULE_WHOLE_CHARS} characters labeled a rule must come with the rule's exact words: {"u7": {"rule": "<exact words>"}}. "corrections" is only for a standing rule that an earlier summary records and no listed message gives; quote the user's exact words. "values": values a later turn may need again that no file will give back: a measurement, a total, a port or a count a turn reported, a hash it named, a file-name format the user chose ("<name>-YYYYMMDD.tar.gz"). Copy each exactly as it appears in the turn "ref" names (the user's message, your replies and the tool output up to the next user message); a value is a number with or without its unit ("166 ms"), a version, a short commit hash, a path or file name (with <placeholders> if the format has them), a host:port or a date; nothing else is kept. "name" is a short label (at most 40 characters, no colon) saying what it is. At most 12, never a password, key or token. "decisions" and "deadEnds" are your own one-sentence records with evidence: a command id C<n>, an error id E<n>, a message u<n>, or the words "last command" / "last error". "now" replaces the previous Now entirely. Never write "done" unless the state's Done gate line says met. Empty arrays are fine. No prose outside the fence.`;
|
|
56
|
+
Rules. "objective" is the request the work is answering now. It is optional: include it only when none is set or the user moved to a different request in this span, and quote the user's own message verbatim. The goal line, when present, is the active /goal or the session's opening request: code keeps it, so never quote it as the objective; under a goal the objective is the specific ask. "labels": Label EVERY message in <user-messages> with one of ${LABEL_FORMS}. ${RULE_DEFINITION} ${LIFT_GUIDANCE} The target is R<n> from the state, or u<n> for a rule given earlier in this span. "not a rule" is a request, question, answer, approval or one-off. Code copies the user's words into the state, so a short message labeled a rule needs no quote; a message longer than ${RULE_WHOLE_CHARS} characters labeled a rule must come with the rule's exact words: {"u7": {"rule": "<exact words>"}}. "corrections" is only for a standing rule that an earlier summary records and no listed message gives; quote the user's exact words. "values": values a later turn may need again that no file will give back: a measurement, a total, a port or a count a turn reported, a hash it named, a file-name format the user chose ("<name>-YYYYMMDD.tar.gz"). Copy each exactly as it appears in the turn "ref" names (the user's message, your replies and the tool output up to the next user message); a value is a number with or without its unit ("166 ms"), a version, a short commit hash, a path or file name (with <placeholders> if the format has them), a host:port or a date; nothing else is kept. "name" is a short label (at most 40 characters, no colon) saying what it is. At most 12, never a password, key or token. "decisions" and "deadEnds" are your own one-sentence records with evidence: a command id C<n>, an error id E<n>, a message u<n>, or the words "last command" / "last error". "now" replaces the previous Now entirely. Never write "done" unless the state's Done gate line says met. ${CREDENTIAL_RULE} Empty arrays are fine. No prose outside the fence.`;
|
|
56
57
|
|
|
57
58
|
export interface PromptState {
|
|
58
59
|
/** The Goal line (store/project.ts goalLine), shown so the summarizer never re-picks it as the Request. */
|
|
@@ -110,8 +111,9 @@ const rememberedText = (event: StateEvent): { kind: string; text: string } | und
|
|
|
110
111
|
switch (event.kind) {
|
|
111
112
|
case "correction.add": return { kind: "correction", text: `${event.id} [${event.quote.ordinal}] ${event.quote.text}` };
|
|
112
113
|
case "correction.lift": return { kind: "lift", text: `${event.target} lifted by ${event.by.ordinal}` };
|
|
113
|
-
|
|
114
|
-
case "
|
|
114
|
+
// Written by a tool call this span: the projection has not seen it, so it is scrubbed here (credential-scrub.ts).
|
|
115
|
+
case "decision.add": return { kind: "decision", text: `${event.id} ${scrubCredentials(event.text).text}` };
|
|
116
|
+
case "deadEnd.add": return { kind: "dead-end", text: `${event.id} ${scrubCredentials(event.text).text}` };
|
|
115
117
|
case "objective.set": return { kind: "objective", text: `${event.quote.ordinal} ${event.quote.text}` };
|
|
116
118
|
case "objective.doneWhen": return { kind: "done-when", text: `${event.quote.ordinal} ${event.quote.text}` };
|
|
117
119
|
default: return undefined;
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { Type } from "typebox";
|
|
10
10
|
import type { AssistantMessage, Context, Message, Model, ModelThinkingLevel, Tool, ToolCall } from "@earendil-works/pi-ai";
|
|
11
11
|
|
|
12
|
+
import { CREDENTIAL_RULE } from "../../credential-scrub.ts";
|
|
12
13
|
import type { ModelCall } from "../call.ts";
|
|
13
14
|
import { timedTransport } from "../timings.ts";
|
|
14
15
|
import { completeWithTransportRetry, type TransportRetryPolicy } from "../transport-retry.ts";
|
|
@@ -16,9 +17,9 @@ import type { CallTrace } from "../types.ts";
|
|
|
16
17
|
import { callUsageOf } from "../usage.ts";
|
|
17
18
|
|
|
18
19
|
/** Recorded as the span's promptRevision in turns mode, as NARRATIVE_PROMPT_REVISION is for the single call. */
|
|
19
|
-
export const TURNS_PROMPT_REVISION = "turns-
|
|
20
|
+
export const TURNS_PROMPT_REVISION = "turns-9";
|
|
20
21
|
|
|
21
|
-
export const TURNS_SYSTEM_PROMPT = `You maintain the SESSION STATE of a coding session that is being compacted. The first message holds the digest of the span: the state so far, the user's messages, the commands and errors (with ids), the files, the workspace and a condensed transcript. Code asks for one section at a time. Answer each with ONE JSON object and nothing else. Use only ids that appear in the digest. Call show(ref) to read a user message (u<n>), command (C<n>), error (E<n>) or tool output (T<n>) in full when the digest cut it
|
|
22
|
+
export const TURNS_SYSTEM_PROMPT = `You maintain the SESSION STATE of a coding session that is being compacted. The first message holds the digest of the span: the state so far, the user's messages, the commands and errors (with ids), the files, the workspace and a condensed transcript. Code asks for one section at a time. Answer each with ONE JSON object and nothing else. Use only ids that appear in the digest. Call show(ref) to read a user message (u<n>), command (C<n>), error (E<n>) or tool output (T<n>) in full when the digest cut it. ${CREDENTIAL_RULE}`;
|
|
22
23
|
|
|
23
24
|
export const SHOW_TOOL: Tool = {
|
|
24
25
|
name: "show",
|
package/src/pass.ts
CHANGED
|
@@ -21,12 +21,14 @@ import { shouldAttemptRoute, type RouteHealthState } from "./narrative/route-hea
|
|
|
21
21
|
import { resolveSummarizationModel, type ModelLookup, type ResolvedSummarizationModel } from "./narrative/routing.ts";
|
|
22
22
|
import { routeProvidersFor, type ResolvedSummarizationRoute } from "./narrative/route-config.ts";
|
|
23
23
|
import { turnTextsOf, type SpanTexts } from "./narrative/named-values.ts";
|
|
24
|
+
import { spanCredentials } from "./span-credentials.ts";
|
|
24
25
|
import { runNarrativeCall, type NarrativeInput } from "./narrative/run.ts";
|
|
25
26
|
import { TURNS_PROMPT_REVISION } from "./narrative/turns/conversation.ts";
|
|
26
27
|
import { runSectionTurns, type TurnsContext } from "./narrative/turns/run.ts";
|
|
27
28
|
import type { NarrativeOutcome } from "./narrative/types.ts";
|
|
28
29
|
import type { WarmBase } from "./narrative/warm.ts";
|
|
29
30
|
import { render } from "./render.ts";
|
|
31
|
+
import { scrubModelWritten } from "./credential-scrub.ts";
|
|
30
32
|
import type { RunningTask } from "./tasks.ts";
|
|
31
33
|
import type { CapFitResult } from "./store/capfit.ts";
|
|
32
34
|
import { closeSpan, type ClosedSpan } from "./store/close.ts";
|
|
@@ -123,6 +125,8 @@ interface Closed {
|
|
|
123
125
|
ledger: { commands: CommandRecord[]; errors: ErrorRecord[] };
|
|
124
126
|
files: ReturnType<typeof carryFileOps>;
|
|
125
127
|
learned: LearnedValue[];
|
|
128
|
+
/** Credential values taken out of what the summarizer wrote (credential-scrub.ts); a count, never the values. */
|
|
129
|
+
redacted: number;
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
function resolveRoute(input: PassInput, config: StateConfig, deps: PassDeps): { route: ResolvedSummarizationRoute; resolved?: ResolvedSummarizationModel; routeAvailable: boolean } {
|
|
@@ -326,30 +330,44 @@ async function callNarrative(input: PassInput, fold: Fold, config: StateConfig,
|
|
|
326
330
|
return runSectionTurns({ ...narrativeInputOf(input, fold, config, deps, at, state), turns: config.narrativeTurns, context }, narrativeDeps);
|
|
327
331
|
}
|
|
328
332
|
|
|
333
|
+
/**
|
|
334
|
+
* Every message the summarizer was shown, as far as credentials go: the span, the previous summary, a switchover's
|
|
335
|
+
* earlier user messages, and in warm mode the session's own last request.
|
|
336
|
+
*/
|
|
337
|
+
function summarizerSaw(input: PassInput, fold: Fold, config: StateConfig, spanMessages: AgentMessage[]): AgentMessage[] {
|
|
338
|
+
const texts = [input.preparation.previousSummary, ...(fold.seed.seedRules ?? []).map((rule) => rule.text), ...(fold.seed.seedContext ?? []).map((message) => message.text)];
|
|
339
|
+
const shown = texts.filter((text): text is string => Boolean(text)).map((text) => ({ role: "custom", content: text, timestamp: 0 }) as unknown as AgentMessage);
|
|
340
|
+
const warm = config.narrativeMode === "warm" ? (input.warmBase?.messages ?? []) as unknown as AgentMessage[] : [];
|
|
341
|
+
return [...spanMessages, ...shown, ...warm];
|
|
342
|
+
}
|
|
343
|
+
|
|
329
344
|
async function closeAndRender(input: PassInput, fold: Fold, outcome: NarrativeOutcome, config: StateConfig, at: string): Promise<Closed> {
|
|
330
345
|
const { preparation, branchEntries } = input;
|
|
331
346
|
const { capture, snapshot, ledger, objectiveEvents, world, files } = await fold.settled();
|
|
332
|
-
const
|
|
347
|
+
const spanMessages = [...preparation.messagesToSummarize, ...(preparation.turnPrefixMessages ?? [])];
|
|
348
|
+
// Before anything is recorded: a credential the summarizer was shown must not reach the span record, nor every later request.
|
|
349
|
+
const scrubbed = scrubModelWritten(outcome.events, outcome.narrative, spanCredentials(summarizerSaw(input, fold, config, spanMessages)));
|
|
350
|
+
const modelEvents = [...objectiveEvents.map((event) => ({ ...event, seq: 0 })), ...scrubbed.events];
|
|
333
351
|
const diffEvents = [...modelEvents, ...seedRuleEvents(fold, outcome, modelEvents, config, at)];
|
|
334
352
|
// Ids from the whole merged ledger, not the fitted slice: a value's source must not vanish with a low-ranked record.
|
|
335
353
|
// The values the summarizer named come first: each was checked against its turn, and its name says what it is.
|
|
336
|
-
const picked = learnedValues(
|
|
354
|
+
const picked = learnedValues(spanMessages, fold.handoff.commands);
|
|
337
355
|
const learned = carryLearnedValues(previousLearnedValues(branchEntries), [...(outcome.values ?? []), ...picked]);
|
|
338
356
|
const tasks = (await fold.tasks) ?? [];
|
|
339
357
|
const measure = (candidate: SessionState) => render({ state: candidate, tokensBefore: preparation.tokensBefore, ledger, snapshot, caps: config.caps, files, learned, tasks, compactedAt: at }).length;
|
|
340
|
-
const closed = closeSpan({ log: fold.log, caps: config.caps, diffEvents: renumber(diffEvents, fold.log.nextSeq), close: { fromRef: fold.spanInput.fromRef, toRef: fold.spanInput.toRef, narrative:
|
|
341
|
-
return { closed, summary: render({ state: closed.state, tokensBefore: preparation.tokensBefore, ledger, snapshot, caps: config.caps, files, learned, tasks, compactedAt: at }), capture, snapshot, ledger, files, learned };
|
|
358
|
+
const closed = closeSpan({ log: fold.log, caps: config.caps, diffEvents: renumber(diffEvents, fold.log.nextSeq), close: { fromRef: fold.spanInput.fromRef, toRef: fold.spanInput.toRef, narrative: scrubbed.narrative, narrativeLess: outcome.kind === "narrative-less", world, ...(outcome.missingChunks.length ? { missingChunks: outcome.missingChunks } : {}) }, at, measure });
|
|
359
|
+
return { closed, summary: render({ state: closed.state, tokensBefore: preparation.tokensBefore, ledger, snapshot, caps: config.caps, files, learned, tasks, compactedAt: at }), capture, snapshot, ledger, files, learned, redacted: scrubbed.redacted };
|
|
342
360
|
}
|
|
343
361
|
|
|
344
362
|
function assembleResult(input: PassInput, config: StateConfig, fold: Fold, outcome: NarrativeOutcome, built: Closed, deps: PassDeps, startedAt: Date, at: string): PassResult {
|
|
345
363
|
const { preparation } = input;
|
|
346
|
-
const { closed, summary, capture, snapshot, ledger, files, learned } = built;
|
|
364
|
+
const { closed, summary, capture, snapshot, ledger, files, learned, redacted } = built;
|
|
347
365
|
const usage = { input: outcome.usage.input, output: outcome.usage.output, calls: outcome.usage.calls };
|
|
348
366
|
const details: StateDetails = { compactor: "state", stateFormat: STATE_FORMAT, promptRevision: outcome.sections ? TURNS_PROMPT_REVISION : NARRATIVE_PROMPT_REVISION, state: closed.state, caps: config.caps, ledger, snapshot, narrative: narrativeTraceOf(outcome), readFiles: files.readFiles, modifiedFiles: files.modifiedFiles, learnedValues: learned };
|
|
349
367
|
const pending: PendingSpan = { span: fold.span, events: closed.log.events.filter((event) => event.span === fold.span), ledger, snapshot, usage, caps: config.coldSliceCaps };
|
|
350
368
|
const route = { spec: fold.route.spec, rule: fold.route.rule, model: fold.resolved ? `${fold.resolved.model.provider}/${fold.resolved.model.id}` : undefined, fallbackReason: fold.resolved?.fallbackReason, routeIgnored: fold.resolved?.routeIgnored, routeAvailable: fold.routeAvailable };
|
|
351
369
|
const logOptions = logOptionsOf(input.env);
|
|
352
|
-
const log = { event: "compaction", at, reason: input.reason, span: fold.span, tokensBefore: preparation.tokensBefore, summaryChars: summary.length, narrative: logSafeNarrative(details.narrative, logOptions), rejected: logSafeRejected(outcome.rejected, logOptions), route, routeHealth: outcome.routeHealth, capFit: closed.capFit.retired, capExhausted: closed.capFit.exhausted, anomalies: fold.log.anomalies, world: { claims: closed.state.world?.claims.length ?? 0, openErrors: closed.state.world?.openErrorIds.length ?? 0, dirty: closed.state.world?.dirty }, carried: { readFiles: files.readFiles.length, modifiedFiles: files.modifiedFiles.length, removed: logSafeRemoved(files.removed, logOptions), learnedValues: learned.length }, workspace: "skipped" in capture ? { skipped: logSafeSkipped(capture.skipped, logOptions) } : { captured: true }, durationMs: deps.now().getTime() - startedAt.getTime() };
|
|
370
|
+
const log = { event: "compaction", at, reason: input.reason, span: fold.span, tokensBefore: preparation.tokensBefore, summaryChars: summary.length, narrative: logSafeNarrative(details.narrative, logOptions), rejected: logSafeRejected(outcome.rejected, logOptions), route, routeHealth: outcome.routeHealth, capFit: closed.capFit.retired, capExhausted: closed.capFit.exhausted, anomalies: fold.log.anomalies, world: { claims: closed.state.world?.claims.length ?? 0, openErrors: closed.state.world?.openErrorIds.length ?? 0, dirty: closed.state.world?.dirty }, carried: { readFiles: files.readFiles.length, modifiedFiles: files.modifiedFiles.length, removed: logSafeRemoved(files.removed, logOptions), learnedValues: learned.length }, redacted, workspace: "skipped" in capture ? { skipped: logSafeSkipped(capture.skipped, logOptions) } : { captured: true }, durationMs: deps.now().getTime() - startedAt.getTime() };
|
|
353
371
|
return {
|
|
354
372
|
compaction: { summary, firstKeptEntryId: preparation.firstKeptEntryId, tokensBefore: preparation.tokensBefore, usage: compactionUsageOf(outcome.usage), details },
|
|
355
373
|
pending, outcome, routeHealth: outcome.routeHealth, route, capFit: closed.capFit, anomalies: fold.log.anomalies, notice: noticeOf(fold.span, outcome, summary.length), log,
|
package/src/remember-tool.ts
CHANGED
|
@@ -2,17 +2,21 @@
|
|
|
2
2
|
* The `remember` tool: one verified write into the Session state, sequential so two calls in one
|
|
3
3
|
* turn never race the fold. Quotes only, resolved newest-first; the store's commitRemember is the
|
|
4
4
|
* whole policy. Argument repair: a quote passed as `text` (or the reverse) is swapped with a Note,
|
|
5
|
-
* because a model that got the field wrong got the intent right.
|
|
5
|
+
* because a model that got the field wrong got the intent right. A decision or dead end is the
|
|
6
|
+
* model's own words, so one that carries a credential value (credential-scrub.ts, with the
|
|
7
|
+
* credentials of the messages still in context, the kept window included) is refused, never stored (security.md T3).
|
|
6
8
|
*/
|
|
7
9
|
import { REMEMBER_TOOL } from "./tool-names.ts";
|
|
8
10
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
9
11
|
import { Type } from "typebox";
|
|
10
12
|
|
|
11
13
|
import { readConfigFile, resolveStateConfig, type ReadFileFn } from "./config.ts";
|
|
14
|
+
import { CREDENTIAL_RULE, scrubCredentials } from "./credential-scrub.ts";
|
|
15
|
+
import { spanCredentials } from "./span-credentials.ts";
|
|
12
16
|
import { commitRemember } from "./store/commit.ts";
|
|
13
17
|
import { EVENT_ENTRY_TYPE, type PersistedEvent } from "./store/events.ts";
|
|
14
18
|
import type { Proposal } from "./store/verify.ts";
|
|
15
|
-
import { buildToolContext } from "./tool-context.ts";
|
|
19
|
+
import { buildToolContext, messagesInContext } from "./tool-context.ts";
|
|
16
20
|
import { formatRemember } from "./tool-text.ts";
|
|
17
21
|
|
|
18
22
|
export const REMEMBER_KINDS = ["correction", "lift", "decision", "dead-end", "done-when"] as const;
|
|
@@ -42,6 +46,13 @@ export function proposalOf(params: RememberParams): { proposal: Proposal; note?:
|
|
|
42
46
|
return { proposal: { kind: "dead-end", text: words, evidence: params.evidence }, note };
|
|
43
47
|
}
|
|
44
48
|
|
|
49
|
+
/** Why a decision or dead end is refused, or undefined: the model's own words may not hold a credential the span showed or one shaped as one. */
|
|
50
|
+
function credentialRefusal(proposal: Proposal, branch: Parameters<typeof messagesInContext>[0]): string | undefined {
|
|
51
|
+
if (proposal.kind !== "decision" && proposal.kind !== "dead-end") return undefined;
|
|
52
|
+
if (scrubCredentials(proposal.text, spanCredentials(messagesInContext(branch))).redacted === 0) return undefined;
|
|
53
|
+
return `the text carries a credential. ${CREDENTIAL_RULE}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
45
56
|
const PARAMETERS = Type.Object({
|
|
46
57
|
kind: Type.Union(REMEMBER_KINDS.map((kind) => Type.Literal(kind)), { description: "correction: a standing rule the user gave (quote). lift: the user withdrew a correction (target + quote). decision / dead-end: your own one sentence (text, evidence). done-when: the user's acceptance words (quote)." }),
|
|
47
58
|
quote: Type.Optional(Type.String({ description: "The user's exact words, a substring of one user message (12+ chars). Example: \"never use mocks in these tests\"." })),
|
|
@@ -64,8 +75,11 @@ export function registerRememberTool(pi: ExtensionAPI, deps: { readConfig: ReadF
|
|
|
64
75
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx: ExtensionContext) {
|
|
65
76
|
const mapped = proposalOf(params as RememberParams);
|
|
66
77
|
if ("error" in mapped) return { content: [{ type: "text", text: `not remembered: ${mapped.error}` }], details: { ok: false, remaining: undefined, rejection: { code: "schema", detail: mapped.error } } };
|
|
78
|
+
const branch = ctx.sessionManager.getBranch();
|
|
79
|
+
const refusal = credentialRefusal(mapped.proposal, branch);
|
|
80
|
+
if (refusal) return { content: [{ type: "text", text: `not remembered: ${refusal}` }], details: { ok: false, remaining: undefined, rejection: { code: "credential", detail: refusal } } };
|
|
67
81
|
const config = resolveStateConfig(deps.readConfig);
|
|
68
|
-
const context = buildToolContext(
|
|
82
|
+
const context = buildToolContext(branch);
|
|
69
83
|
const outcome = commitRemember(mapped.proposal, {
|
|
70
84
|
log: context.log, resolver: context.index, caps: config.caps, nearDuplicateThreshold: config.nearDuplicateThreshold, rememberCap: config.rememberPerSpan,
|
|
71
85
|
now: () => deps.now().toISOString(), appendEntry: (customType: string, data: PersistedEvent) => pi.appendEntry(customType, data),
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The span's credentials (credential-scrub.ts SpanCredentials): the whole values the span itself showed as
|
|
3
|
+
* credentials, which the scrub then takes out of what the summarizer wrote wherever it put them (security.md T3).
|
|
4
|
+
* Strong evidence only, because each value is redacted everywhere it stands: the named-value taint
|
|
5
|
+
* (narrative/named-values.ts), right for refusing a value, counts the words after "pin", "card" or "login" and a
|
|
6
|
+
* secret store's whole output, and on real states it redacted versions, commit hashes and paths in about a third of
|
|
7
|
+
* the fields. A value counts when
|
|
8
|
+
* - the span's text names it as the scrub itself would (credential-scrub.ts namedCredentialValues), unless it is
|
|
9
|
+
* shaped as a name (below);
|
|
10
|
+
* - it is the value of a key named for one credential, ending in its name or a short part after it
|
|
11
|
+
* (`DB_PASSWORD`, `cluster_token`, `WIFI_PASSPHRASE_2400_BAUD`; not `tokens: 1200`, a test runner's `pass`, nor a key
|
|
12
|
+
* about one, `password_policy`, `token_env`), unless a token, secret or key key holds a value shaped as a name (`app-token`,
|
|
13
|
+
* `APP_TOKEN`, a path, a file name; a password key's value is one whatever its shape, `unmade-tremor-oxidize`);
|
|
14
|
+
* - a command gives it as a password argument (`mysql -pX`, `sshpass -p X`, `redis-cli -a X`, `curl -u user:X`,
|
|
15
|
+
* `htpasswd -b FILE USER X`, `--db-password X`, `openssl -pass pass:X`, `keytool -storepass X`);
|
|
16
|
+
* - it is a random-looking word on another credential line (a YAML block under such a key);
|
|
17
|
+
* - a secret store printed it alone (`vault kv get -field=...`), or random-looking among other output;
|
|
18
|
+
* - but never a random-looking word its label names as an id or a hash (`Evaluation ID: <uuid>`, `plan_hash <hex>`,
|
|
19
|
+
* `commit <sha>`), nor the words after a key whose value is computed (`TOKEN=$(vault kv get ... | python3 ...)`), nor
|
|
20
|
+
* a test runner's `--- PASS: TestName`;
|
|
21
|
+
* - or prose gives it: `the password is ...`, `the db pass is ...`.
|
|
22
|
+
* Never a number (except six digits and more given as a password), version, date or path, nor one plain lowercase
|
|
23
|
+
* word ("token: missing"). Measured on real states, these took no version, hash or path out of a field.
|
|
24
|
+
*/
|
|
25
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
26
|
+
|
|
27
|
+
import { credentialCore, isNumberish, isNumericPasswordKey, isRandomLooking, isValueLike, namedCredentialValues, WORD } from "./credential-scrub.ts";
|
|
28
|
+
import { CREDENTIAL_NAME, keyedLine, secretStoreOutputs, splitCredentialLines } from "./learned-secrets.ts";
|
|
29
|
+
import type { Rec } from "./message-text.ts";
|
|
30
|
+
import { everyTextIn } from "./narrative/named-values.ts";
|
|
31
|
+
|
|
32
|
+
/** `the password is X`: the one value prose gives a password outright. */
|
|
33
|
+
const PROSE_PASSWORD = /(?<![A-Za-z])(?:passw(?:or)?d|passwd|pwd|pass|passphrase|passcode)\s+(?:is|was)\s+(\S+)/gi;
|
|
34
|
+
/**
|
|
35
|
+
* A command's password argument, group 1 (as ledger/handoff-commands.ts SECRET_PATTERNS, which marks the line).
|
|
36
|
+
* Every quantifier is bounded, each lazy run stops at a newline.
|
|
37
|
+
*/
|
|
38
|
+
const COMMAND_PASSWORDS = [
|
|
39
|
+
/\b(?:mysql|mysqldump|mariadb)\b[^\n]{0,2000}?\s-p(?!\s)(\S+)/g,
|
|
40
|
+
/\bsshpass\s+(?:(?:-P\s+\S+|-[a-oq-zA-Z]\w*)\s+){0,3}-p\s*(\S+)/g,
|
|
41
|
+
/\bredis-cli\b[^\n]{0,200}?\s-a\s+(\S+)/g,
|
|
42
|
+
/(?:^|\s)(?:-u|--user)\s+[^\s:]+:(\S+)/g,
|
|
43
|
+
/\s-pass(?:in|out)?\s+pass:(\S+)/g,
|
|
44
|
+
/\s-(?:src|dest)?(?:store|key)?pass\s+(?!(?:env|file|fd):|stdin\b)(\S+)/g,
|
|
45
|
+
/(?:^|\s)--?(?:[a-z0-9]{1,32}[-_.]){0,3}(?:password|passwd|passphrase|pass|secret|token|api[-_]?key)(?:=|\s+)(?!-)(\S+)/gi,
|
|
46
|
+
];
|
|
47
|
+
/** `htpasswd -b FILE USER PASS`, or `-nb USER PASS` (no file). */
|
|
48
|
+
const HTPASSWD = /\bhtpasswd\s+(?:-[a-zA-Z]{1,8}\s+){0,3}?-([a-zA-Z]{0,8}b[a-zA-Z]{0,8})\s+(\S+)\s+(\S+)(?:\s+(\S+))?/g;
|
|
49
|
+
/** A test runner's status line, `--- PASS: TestName (0.01s)` or `PASS: name`, upper case: no credential (`pass: x` is one). */
|
|
50
|
+
const TEST_STATUS = /^\s*(?:[-=]{3}\s*)?PASS\s*:/;
|
|
51
|
+
/** A value pointed at rather than held (a variable, a placeholder), once a quote or bracket before it is dropped. */
|
|
52
|
+
const REFERENCE = /^["'`([]*(?:\$|%|\*|\{|<|\.\.\.)/;
|
|
53
|
+
const NOT_A_SECRET = [
|
|
54
|
+
/^v?\d+(?:[.:_-]\d+)*$/i, // a number, a version, a time
|
|
55
|
+
/^\d{4}-\d{2}-\d{2}(?:[T ][\d:.]+Z?)?$/, // a date
|
|
56
|
+
/^(?:\/|~\/|\.\.?\/)/, // a path
|
|
57
|
+
/^[a-z]+$/, // one plain lowercase word: a status
|
|
58
|
+
];
|
|
59
|
+
const isSpanValue = (value: string) => value.length >= 4 && (/^\d{13,}$/.test(value) || !NOT_A_SECRET.some((shape) => shape.test(value)));
|
|
60
|
+
/** A password given as one: a span value, or six digits and more. */
|
|
61
|
+
const isPasswordValue = (value: string) => isSpanValue(value) || /^\d{6,}$/.test(value);
|
|
62
|
+
/**
|
|
63
|
+
* What may follow a credential's name at the end of its key: up to two short parts (`WIFI_PASSPHRASE_2400_BAUD`,
|
|
64
|
+
* `ADMIN_TOKEN_2`), then quotes. A part that says the key is ABOUT the credential (KEY_ABOUT) makes it no key for one.
|
|
65
|
+
*/
|
|
66
|
+
const KEY_TAIL = String.raw`((?:[_.-][A-Za-z0-9]{1,12}){0,2})["'\\]{0,4}$`;
|
|
67
|
+
const KEY_ABOUT = /[_.-](?:policy|disabled|enabled|required|length|len|min|max|algorithm|algo|hash|hashed|salt|rounds|file|path|dir|env|var|name|id|type|kind|mode|set|present|missing|rules|reset|expiry|expires|ttl|age|count|ref|url|header|field|prompt|hint|strength|changed|updated)(?![A-Za-z0-9])/i;
|
|
68
|
+
const endsInCredential = (pattern: RegExp, key: string) => {
|
|
69
|
+
const match = pattern.exec(key.trim());
|
|
70
|
+
return match !== null && !KEY_ABOUT.test(match[1]);
|
|
71
|
+
};
|
|
72
|
+
/** A key that ends in a password word (`DB_PASS`, `"password"`), not one about passwords (`password_policy`). */
|
|
73
|
+
const PASSWORD_KEY = new RegExp(String.raw`(?:passw(?:or)?d|passwd|pwd|pass(?:phrase|code)?)s?` + KEY_TAIL, "i");
|
|
74
|
+
const isPasswordKey = (key: string) => endsInCredential(PASSWORD_KEY, key);
|
|
75
|
+
/** Push without spreading: a spread passes every element as an argument and overflows the stack on a huge line. */
|
|
76
|
+
function pushAll(into: string[], items: Iterable<string>): void {
|
|
77
|
+
for (const item of items) into.push(item);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* A key named for one credential, in at most three words, ending in its name (quotes aside): singular, so
|
|
81
|
+
* `tokens: 1200` and a list of `secrets` are not one, no bare `pass` (a test runner's count), and no key about one
|
|
82
|
+
* (`password_policy`, `token_env`, `PASSWORD_FILE`). Letters may precede the name (`PGPASSWORD`).
|
|
83
|
+
*/
|
|
84
|
+
const CREDENTIAL_KEY = new RegExp(String.raw`(?:passw(?:or)?d|passwd|passphrase|passcode|secret|token|api[_ -]?key|access[_ -]?key|private[_ -]?key|credential)` + KEY_TAIL, "i");
|
|
85
|
+
/** A bare lower-case `pass` key (nodemailer's `auth.pass`, YAML), which CREDENTIAL_KEY leaves out as a test runner's count. */
|
|
86
|
+
const BARE_PASS_KEY = /^["']?pass["']?$/;
|
|
87
|
+
const isCredentialKey = (key: string) => BARE_PASS_KEY.test(key.trim()) || key.length <= 40 && key.trim().split(/\s+/).length <= 3 && endsInCredential(CREDENTIAL_KEY, key);
|
|
88
|
+
/**
|
|
89
|
+
* A value shaped as a name of something rather than a secret: an identifier, an environment variable, a path, a file
|
|
90
|
+
* or host name, or a call that computes it (`secret = json.load(open('app.json'))`).
|
|
91
|
+
*/
|
|
92
|
+
const NAME_SHAPES = [/^[a-z][a-z0-9]*(?:[-_.][a-z0-9]+)+$/, /^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+$/, /\//, /^[\w-]+(?:\.[\w-]+)+$/, /^[A-Za-z_][\w.]{0,64}\(/];
|
|
93
|
+
const isNameShaped = (value: string) => !isRandomLooking(value) && NAME_SHAPES.some((shape) => shape.test(value));
|
|
94
|
+
/** A credential key's value: value-like, or eight characters and more; under a token, secret or key key, not shaped as a name. */
|
|
95
|
+
function isKeyedValue(value: string, key: string): boolean {
|
|
96
|
+
if (/^\d{6,}$/.test(value)) return isNumericPasswordKey(key);
|
|
97
|
+
if (isPasswordKey(key)) return isSpanValue(value) && (isValueLike(value) || value.length >= 6);
|
|
98
|
+
return (isValueLike(value) || value.length >= 8) && !isNameShaped(value) && !isNumberish(value);
|
|
99
|
+
}
|
|
100
|
+
/** A command's password argument: a password, value-like or six characters and more. */
|
|
101
|
+
const isCommandValue = (value: string) => isPasswordValue(value) && (isValueLike(value) || value.length >= 6);
|
|
102
|
+
/** A value prose gives: value-like, or a long run with a non-letter in it (a passphrase), not a long English word. */
|
|
103
|
+
const isProseValue = (value: string) => isValueLike(value) || (value.length >= 12 && /[^A-Za-z]/.test(value));
|
|
104
|
+
|
|
105
|
+
/** The words of `text` that hold a value, each with what wraps it trimmed. */
|
|
106
|
+
function coresOf(text: string): string[] {
|
|
107
|
+
const cores: string[] = [];
|
|
108
|
+
for (const [word] of text.matchAll(WORD)) {
|
|
109
|
+
if (REFERENCE.test(word)) continue;
|
|
110
|
+
const core = credentialCore(word);
|
|
111
|
+
if (core.end > core.start) cores.push(word.slice(core.start, core.end));
|
|
112
|
+
}
|
|
113
|
+
return cores;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** A word that names the value after it as an id or a hash (`Evaluation ID:`, `plan_hash`, `commit`), not a secret. */
|
|
117
|
+
const ID_LABEL = /(?:^|[_.-])(?:id|ids|uuid|hash|sha|sha1|sha256|digest|commit|rev|revision|etag)$/i;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The random-looking words of `text`, except one the word before it on its line labels as an id or a hash (ID_LABEL),
|
|
121
|
+
* unless that label or the word before it names a credential (`secret_id`, `Secret ID`).
|
|
122
|
+
*/
|
|
123
|
+
function randomWordsOf(text: string): string[] {
|
|
124
|
+
const values: string[] = [];
|
|
125
|
+
let label = "";
|
|
126
|
+
let previous = "";
|
|
127
|
+
let end = 0;
|
|
128
|
+
for (const match of text.matchAll(WORD)) {
|
|
129
|
+
if (text.slice(end, match.index).includes("\n")) label = previous = "";
|
|
130
|
+
const core = credentialCore(match[0]);
|
|
131
|
+
const bare = match[0].slice(core.start, core.end);
|
|
132
|
+
const idLabelled = ID_LABEL.test(label) && !CREDENTIAL_NAME.test(`${previous} ${label}`);
|
|
133
|
+
if (!REFERENCE.test(match[0]) && isRandomLooking(bare) && !idLabelled) values.push(bare);
|
|
134
|
+
previous = label;
|
|
135
|
+
label = bare;
|
|
136
|
+
end = match.index + match[0].length;
|
|
137
|
+
}
|
|
138
|
+
return values;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The password arguments of a command line. */
|
|
142
|
+
function commandValuesOf(line: string): string[] {
|
|
143
|
+
const values: string[] = [];
|
|
144
|
+
for (const pattern of COMMAND_PASSWORDS) for (const match of line.matchAll(pattern)) pushAll(values, coresOf(match[1]));
|
|
145
|
+
for (const match of line.matchAll(HTPASSWD)) {
|
|
146
|
+
const value = match[1].includes("n") ? match[3] : match[4];
|
|
147
|
+
if (value) pushAll(values, coresOf(value));
|
|
148
|
+
}
|
|
149
|
+
return values.filter(isCommandValue);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** A password key written as YAML (`password: my Secret#Pass 99`), whose plain value runs to the line's end or a comment. */
|
|
153
|
+
const isYamlPassword = (line: string, key: string) => isPasswordKey(key) && /^\s*:\s/.test(line.trimStart().slice(key.length));
|
|
154
|
+
/** What is left of an assignment once the key is cut at its first colon: Go's `:=` or a type (`: str = `, `: &str = `). */
|
|
155
|
+
const ASSIGNED = /^(?:[^\s=]{1,40}[ \t]{1,8})?:?=(?!=)[ \t]{0,8}/;
|
|
156
|
+
/**
|
|
157
|
+
* What a key holds: its quoted string, or else its first word; the rest of the line is the command or prose around it
|
|
158
|
+
* (`TOKEN=$(vault kv get ... | python3 -c "import sys,json"`). Nothing when the value is computed or pointed at.
|
|
159
|
+
*/
|
|
160
|
+
function heldBy(raw: string, toLineEnd: boolean): string {
|
|
161
|
+
const value = raw.replace(ASSIGNED, "");
|
|
162
|
+
const quoted = /^(\\{0,3}["'])((?:\\.|[^\n\\]){0,512}?)\1/.exec(value);
|
|
163
|
+
const comment = toLineEnd ? value.search(/\s#/) : -1;
|
|
164
|
+
const held = quoted ? quoted[2] : toLineEnd ? (comment < 0 ? value : value.slice(0, comment)) : (/^\S+/.exec(value)?.[0] ?? "");
|
|
165
|
+
return REFERENCE.test(held) || /^["'([]*`/.test(held) ? "" : held;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function valuesOf(text: string): string[] {
|
|
169
|
+
const values = namedCredentialValues(text).filter((value) => !isNameShaped(value));
|
|
170
|
+
for (const line of splitCredentialLines(text).credential) {
|
|
171
|
+
if (TEST_STATUS.test(line)) continue;
|
|
172
|
+
const keyed = keyedLine(line);
|
|
173
|
+
if (keyed && isCredentialKey(keyed.key)) pushAll(values, coresOf(heldBy(keyed.value, isYamlPassword(line, keyed.key))).filter((value) => isKeyedValue(value, keyed.key)));
|
|
174
|
+
else pushAll(values, randomWordsOf(line));
|
|
175
|
+
pushAll(values, commandValuesOf(line));
|
|
176
|
+
}
|
|
177
|
+
for (const match of text.matchAll(PROSE_PASSWORD)) pushAll(values, coresOf(match[1]).filter((value) => isProseValue(value) || /^\d{6,}$/.test(value)));
|
|
178
|
+
return values;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** What a secret store printed: a value printed alone is one whatever it looks like; among other output, a random-looking one. */
|
|
182
|
+
function storeValuesOf(output: string): string[] {
|
|
183
|
+
const words = coresOf(output.trim());
|
|
184
|
+
return words.length === 1 ? words : randomWordsOf(output);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** The span's credentials in `messages`: every whole value one of them showed as a credential. */
|
|
188
|
+
export function spanCredentials(messages: readonly AgentMessage[]): Set<string> {
|
|
189
|
+
const values = new Set<string>();
|
|
190
|
+
const add = (value: string) => { if (isPasswordValue(value)) values.add(value); };
|
|
191
|
+
for (const message of messages as unknown as Rec[]) for (const text of everyTextIn(message)) valuesOf(text).forEach(add);
|
|
192
|
+
for (const output of secretStoreOutputs(messages as unknown as Rec[])) storeValuesOf(output).forEach(add);
|
|
193
|
+
return values;
|
|
194
|
+
}
|
package/src/store/project.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import type { Caps, ForeignCompactor, Now, Quote, Source, StateEvent, StateLog, World } from "./events.ts";
|
|
9
9
|
import { DEFAULT_NEAR_DUPLICATE_THRESHOLD, overlap, tokenize } from "./near-duplicate.ts";
|
|
10
10
|
import { oneLine } from "../one-line.ts";
|
|
11
|
+
import { scrubCredentials } from "../credential-scrub.ts";
|
|
11
12
|
|
|
12
13
|
export interface CorrectionItem {
|
|
13
14
|
id: string;
|
|
@@ -69,6 +70,14 @@ export function emptyState(): SessionState {
|
|
|
69
70
|
|
|
70
71
|
const idNumber = (id: string) => Number(/^[RDX](\d+)$/.exec(id)?.[1] ?? 0);
|
|
71
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Model-written text as the state shows it: with its credential values redacted (credential-scrub.ts). A pass scrubs
|
|
75
|
+
* what it writes against the span's own credentials; this catches what a record written before that carries, by
|
|
76
|
+
* shape and by what the text itself names, wherever the state is read (render, prompt, recall, the state CLI).
|
|
77
|
+
*/
|
|
78
|
+
const clean = (text: string) => scrubCredentials(text).text;
|
|
79
|
+
const cleanNow = (now: Now): Now => ({ inProgress: now.inProgress.map(clean), next: now.next.map(clean), blocked: now.blocked.map(clean), resume: clean(now.resume) });
|
|
80
|
+
|
|
72
81
|
/** Retire the oldest active items of a kind past the cap; returns how many retired. */
|
|
73
82
|
function retireOverCap<T extends { retired: boolean; seq: number }>(items: T[], cap: number): number {
|
|
74
83
|
let excess = items.filter((item) => !item.retired).length - cap;
|
|
@@ -88,7 +97,7 @@ function workLogText(event: Extract<StateEvent, { kind: "span.close" }>): string
|
|
|
88
97
|
if (event.foreign) return `compacted by ${event.foreign === "pi" ? "pi's own summarizer" : "another extension's compactor"}; no state diff for this span`;
|
|
89
98
|
const missing = event.missingChunks?.length ? ` (chunk ${event.missingChunks.join(", ")} of this span did not verify)` : "";
|
|
90
99
|
if (event.narrativeLess || !event.narrative) return `narrative unavailable for this span (narrative-less pass)${missing}`;
|
|
91
|
-
return `${event.narrative}${missing}`;
|
|
100
|
+
return `${clean(event.narrative)}${missing}`;
|
|
92
101
|
}
|
|
93
102
|
|
|
94
103
|
/** Fold the two oldest work-log lines into one consolidated line until the active count fits the cap. */
|
|
@@ -210,17 +219,17 @@ function applyInPlace(state: SessionState, event: StateEvent, caps: Caps): void
|
|
|
210
219
|
break;
|
|
211
220
|
}
|
|
212
221
|
case "decision.add":
|
|
213
|
-
state.decisions.push({ id: event.id, text: event.text, span: event.span, source: event.source, evidence: event.evidence, seq: event.seq, retired: false });
|
|
222
|
+
state.decisions.push({ id: event.id, text: clean(event.text), span: event.span, source: event.source, evidence: event.evidence, seq: event.seq, retired: false });
|
|
214
223
|
state.cursor.D = Math.max(state.cursor.D, idNumber(event.id));
|
|
215
224
|
state.retired.decisions += retireOverCap(state.decisions, caps.decisions);
|
|
216
225
|
break;
|
|
217
226
|
case "deadEnd.add":
|
|
218
|
-
state.deadEnds.push({ id: event.id, text: event.text, span: event.span, source: event.source, evidence: event.evidence, seq: event.seq, retired: false });
|
|
227
|
+
state.deadEnds.push({ id: event.id, text: clean(event.text), span: event.span, source: event.source, evidence: event.evidence, seq: event.seq, retired: false });
|
|
219
228
|
state.cursor.X = Math.max(state.cursor.X, idNumber(event.id));
|
|
220
229
|
state.retired.deadEnds += retireOverCap(state.deadEnds, caps.deadEnds);
|
|
221
230
|
break;
|
|
222
231
|
case "now.set":
|
|
223
|
-
state.now = { ...event.now, span: event.span, carried: false };
|
|
232
|
+
state.now = { ...cleanNow(event.now), span: event.span, carried: false };
|
|
224
233
|
break;
|
|
225
234
|
case "span.close":
|
|
226
235
|
applyClose(state, event, caps);
|
package/src/store/recall.ts
CHANGED
|
@@ -6,9 +6,16 @@
|
|
|
6
6
|
import type { CommandRecord, ErrorRecord } from "../ledger/handoff.ts";
|
|
7
7
|
import type { Caps, StateEvent, StateLog } from "./events.ts";
|
|
8
8
|
import { orderedFiles } from "../files-render.ts";
|
|
9
|
-
import { oneLine
|
|
9
|
+
import { oneLine } from "../one-line.ts";
|
|
10
|
+
import { scrubCredentials } from "../credential-scrub.ts";
|
|
10
11
|
import { eventKey, project } from "./project.ts";
|
|
11
12
|
|
|
13
|
+
/**
|
|
14
|
+
* A hit's text: one line, with its credential values redacted (credential-scrub.ts). Span records written before the
|
|
15
|
+
* scrub still carry what the summarizer copied, and an error line from such a record may carry one too.
|
|
16
|
+
*/
|
|
17
|
+
const sanitize = (text: string) => oneLine(scrubCredentials(text).text);
|
|
18
|
+
|
|
12
19
|
export type RecallWhat = "span" | "commands" | "errors" | "decisions" | "dead-ends" | "corrections" | "message" | "files" | "all";
|
|
13
20
|
|
|
14
21
|
export interface RecallQuery {
|
package/src/tool-context.ts
CHANGED
|
@@ -29,6 +29,22 @@ export function messagesSinceLastCompaction(branch: SessionEntry[]): AgentMessag
|
|
|
29
29
|
return branch.slice(start).filter((entry) => entry.type === "message").map((entry) => (entry as { message: AgentMessage }).message);
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* The messages still in the model's context: from the latest compaction's first kept entry (the kept window), else
|
|
34
|
+
* from the start of the branch. What a remember write is checked against (remember-tool.ts).
|
|
35
|
+
*/
|
|
36
|
+
export function messagesInContext(branch: SessionEntry[]): AgentMessage[] {
|
|
37
|
+
let start = 0;
|
|
38
|
+
for (let index = branch.length - 1; index >= 0; index--) {
|
|
39
|
+
const entry = branch[index];
|
|
40
|
+
if (entry.type !== "compaction") continue;
|
|
41
|
+
const kept = branch.findIndex((candidate) => candidate.id === entry.firstKeptEntryId);
|
|
42
|
+
start = kept >= 0 && kept < index ? kept : index + 1;
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
return branch.slice(start).filter((entry) => entry.type === "message").map((entry) => (entry as { message: AgentMessage }).message);
|
|
46
|
+
}
|
|
47
|
+
|
|
32
48
|
/** The live ledger alone, for a caller that needs no index, log or files (recall what=output). */
|
|
33
49
|
export function liveLedger(branch: SessionEntry[]): ToolContext["ledger"] {
|
|
34
50
|
const handoff = collectHandoff({ branchEntries: branch, messages: messagesSinceLastCompaction(branch) });
|