typeclaw 0.1.2 → 0.1.3

package/src/secrets/storage.ts

@@ -8,8 +8,17 @@ import {
 } from '@mariozechner/pi-coding-agent'
 import lockfile from 'proper-lockfile'
 
+import { providerKeyDefaultEnv } from './defaults'
 import { migrateLegacyAuthJson } from './migrate'
-import { type SecretsFile, parseSecretsFile } from './schema'
+import { resolveSecret, type Secret } from './resolve'
+import {
+  type Channels,
+  type ProviderCredential,
+  type Providers,
+  type SecretsFile,
+  SECRETS_FILE_VERSION,
+  parseSecretsFile,
+} from './schema'
 
 const SCHEMA_REL = './node_modules/typeclaw/secrets.schema.json'
 const FILE_MODE = 0o600
@@ -23,26 +32,44 @@ const ASYNC_LOCK_OPTIONS = {
   stale: 30000,
 } as const
 
-// SecretsBackend implements pi-coding-agent's AuthStorageBackend contract
-// while keeping TypeClaw in control of the on-disk file shape.
+// SecretsBackend bridges TypeClaw's on-disk envelope (v2: providers with
+// Secret-typed keys, channels with per-adapter field shapes) to upstream
+// `AuthStorage`'s flat `Record<provider, AuthCredential>` contract.
 //
-// Upstream's FileAuthStorageBackend assumes the entire file IS the
-// AuthStorageData (a flat Record<string, AuthCredential>). TypeClaw needs the
-// file to also carry version + channels alongside the LLM slice, so we wrap:
-// every withLock cycle reads the full envelope, presents only file.llm to the
-// AuthStorage instance as if it were the whole file, and merges the result
-// back into the envelope on write.
+// READ (withLock's `current` parameter):
+//   - Parse the envelope, walk `providers`, resolve each api-key `Secret` to
+//     a flat string via env-wins (process.env wins over file value).
+//   - OAuth credentials pass through untouched.
+//   - Capture the resolved string for each provider into `readSnapshot` so
+//     the write path can detect "unchanged" without re-resolving (the env
+//     can change between read and write, and re-resolving would misclassify
+//     env mutations as credential mutations).
+//   - Hand AuthStorage a flat-shape JSON string. Upstream is none the wiser.
 //
-// Locking and durability semantics mirror upstream's FileAuthStorageBackend:
-// - proper-lockfile, sync version uses busy-loop retry on ELOCKED so callers
-//   stay synchronous (matching upstream's API contract)
-// - parent directory created with 0o700, file written with 0o600
-// - empty file is created on first access so proper-lockfile has something
-//   to lock against (it requires the target to exist)
+// WRITE (the `next` field):
+//   - AuthStorage hands back the full flat slice as JSON. We do NOT
+//     wholesale-replace the on-disk `providers` slice with this.
+//   - Instead, we DIFF at credential level against the prior envelope using
+//     the read-time `readSnapshot`:
+//       * Provider unchanged (flatKey === readSnapshot[providerId]) → preserve
+//         on-disk Secret bytes verbatim (no flatten, no rewrap). This is the
+//         idempotency rule that prevents OAuth-refresh from accidentally
+//         persisting env-resolved api-key values into the file.
+//       * Provider changed → rewrap as Secret, preserving any prior `env`
+//         field the user authored.
+//       * Provider added → write as string shorthand (no env binding).
+//       * Provider removed → actually remove (do NOT resurrect).
+//   - OAuth credentials stay flat strings in the envelope (no Secret
+//     wrapping) — they're not env-injectable.
+//   - Unknown credential `type` values pass through verbatim, in case
+//     upstream adds a third type in a future release.
+//   - Empty/missing `key` from AuthStorage on api-key is treated as no-op
+//     (preserve prior on-disk Secret if any). The schema requires non-empty
+//     `value`, so writing an empty key would corrupt the file at next read.
 //
-// We additionally write atomically (temp + rename) for durability — upstream
-// uses plain writeFileSync, but we own a richer envelope and a half-write
-// would leave us with neither the old nor the new shape parseable.
+// Locking and durability mirror upstream's FileAuthStorageBackend: sync
+// busy-loop retry on ELOCKED to keep callers synchronous, 0o600 file, 0o700
+// parent, atomic temp+rename.
 export class SecretsBackend implements AuthStorageBackend {
   constructor(private readonly secretsPath: string) {}
 
@@ -53,11 +80,11 @@ export class SecretsBackend implements AuthStorageBackend {
     try {
       release = this.acquireSyncLockWithRetry()
       const envelope = this.readEnvelope()
-      const innerCurrent = JSON.stringify(envelope.llm, null, 2)
+      const { flatJson, readSnapshot } = flattenProvidersForAuthStorage(envelope.providers, process.env)
 
-      const { result, next } = fn(innerCurrent)
+      const { result, next } = fn(flatJson)
       if (next !== undefined) {
-        const merged = mergeLlmIntoEnvelope(envelope, next)
+        const merged = mergeProvidersIntoEnvelope(envelope, next, readSnapshot)
         this.writeEnvelopeAtomic(merged)
       }
       return result
@@ -87,12 +114,12 @@ export class SecretsBackend implements AuthStorageBackend {
       })
       throwIfCompromised()
       const envelope = this.readEnvelope()
-      const innerCurrent = JSON.stringify(envelope.llm, null, 2)
+      const { flatJson, readSnapshot } = flattenProvidersForAuthStorage(envelope.providers, process.env)
 
-      const { result, next } = await fn(innerCurrent)
+      const { result, next } = await fn(flatJson)
       throwIfCompromised()
       if (next !== undefined) {
-        const merged = mergeLlmIntoEnvelope(envelope, next)
+        const merged = mergeProvidersIntoEnvelope(envelope, next, readSnapshot)
         this.writeEnvelopeAtomic(merged)
       }
       throwIfCompromised()
@@ -110,6 +137,92 @@ export class SecretsBackend implements AuthStorageBackend {
     }
   }
 
+  readChannelsSync(): Channels {
+    this.ensureParentDir()
+    this.ensureFileExists()
+    let release: (() => void) | undefined
+    try {
+      release = this.acquireSyncLockWithRetry()
+      return this.readEnvelope().channels
+    } finally {
+      release?.()
+    }
+  }
+
+  tryReadChannelsSync(): Channels | null {
+    if (!existsSync(this.secretsPath)) return null
+    let release: (() => void) | undefined
+    try {
+      release = this.acquireSyncLockWithRetry()
+      return this.readEnvelope().channels
+    } finally {
+      release?.()
+    }
+  }
+
+  writeChannelsSync(next: Channels): void {
+    this.ensureParentDir()
+    this.ensureFileExists()
+    let release: (() => void) | undefined
+    try {
+      release = this.acquireSyncLockWithRetry()
+      const envelope = this.readEnvelope()
+      const merged: SecretsFile = {
+        ...envelope,
+        $schema: envelope.$schema ?? SCHEMA_REL,
+        version: SECRETS_FILE_VERSION,
+        channels: next,
+      }
+      this.writeEnvelopeAtomic(merged)
+    } finally {
+      release?.()
+    }
+  }
+
+  async updateChannelsAsync<T>(
+    fn: (current: Record<string, unknown>) => Promise<{ result: T; next?: Record<string, unknown> }>,
+  ): Promise<T> {
+    this.ensureParentDir()
+    this.ensureFileExists()
+    let release: (() => Promise<void>) | undefined
+    let lockCompromised = false
+    let lockCompromisedError: Error | undefined
+    const throwIfCompromised = (): void => {
+      if (lockCompromised) {
+        throw lockCompromisedError ?? new Error('Secrets store lock was compromised')
+      }
+    }
+    try {
+      release = await lockfile.lock(this.secretsPath, {
+        ...ASYNC_LOCK_OPTIONS,
+        onCompromised: (err: Error) => {
+          lockCompromised = true
+          lockCompromisedError = err
+        },
+      })
+      throwIfCompromised()
+      const envelope = this.readEnvelope()
+      const { result, next } = await fn(envelope.channels as Record<string, unknown>)
+      throwIfCompromised()
+      if (next !== undefined) {
+        const merged: SecretsFile = {
+          ...envelope,
+          $schema: envelope.$schema ?? SCHEMA_REL,
+          channels: next as SecretsFile['channels'],
+        }
+        this.writeEnvelopeAtomic(merged)
+      }
+      throwIfCompromised()
+      return result
+    } finally {
+      if (release) {
+        try {
+          await release()
+        } catch {}
+      }
+    }
+  }
+
   private ensureParentDir(): void {
     const dir = dirname(this.secretsPath)
     if (!existsSync(dir)) {
@@ -117,10 +230,6 @@ export class SecretsBackend implements AuthStorageBackend {
     }
   }
 
-  // proper-lockfile requires the target to exist before locking. We seed an
-  // empty new-shape envelope so the very first call has something to lock,
-  // and so the file is parseable by a third-party reader even before the
-  // first credential is written.
   private ensureFileExists(): void {
     if (existsSync(this.secretsPath)) return
     const seed = newEmptyEnvelope()
@@ -140,11 +249,9 @@ export class SecretsBackend implements AuthStorageBackend {
             : undefined
         if (code !== 'ELOCKED' || attempt === SYNC_LOCK_RETRIES) throw error
         lastError = error
-        // Busy-wait so the call stays synchronous. Matches upstream's
-        // FileAuthStorageBackend.acquireLockSyncWithRetry.
         const start = Date.now()
         while (Date.now() - start < SYNC_LOCK_DELAY_MS) {
-          // intentionally empty
+          // intentionally empty: synchronous busy-wait to match upstream contract
         }
       }
     }
@@ -167,8 +274,6 @@ export class SecretsBackend implements AuthStorageBackend {
     return result.file
   }
 
-  // Atomic temp+rename, same pattern as src/hostd/daemon.ts:persistRegistration.
-  // The temp file lives in the same directory so rename is intra-filesystem.
   private writeEnvelopeAtomic(envelope: SecretsFile): void {
     const tmp = `${this.secretsPath}.${process.pid}.${Date.now()}.tmp`
     writeFileSync(tmp, stringifyEnvelope(envelope), { encoding: 'utf8', mode: FILE_MODE })
@@ -186,44 +291,145 @@ export class SecretsBackend implements AuthStorageBackend {
   }
 }
 
-// createSecretsStoreForAgent is the single seam every TypeClaw caller should
-// use to obtain an AuthStorage tied to an agent folder's secrets file. Keeps
-// the upstream constructor (AuthStorage.fromStorage) usage isolated to one
-// module so a future change to upstream wiring only touches this file.
-//
-// Performs the one-shot auth.json -> secrets.json rename before opening the
-// backend, so callers never observe the legacy filename even on agents that
-// pre-date the rename.
 export function createSecretsStoreForAgent(secretsPath: string): AuthStorage {
   migrateLegacyAuthJson(dirname(secretsPath))
   return AuthStorageImpl.fromStorage(new SecretsBackend(secretsPath))
 }
 
 function newEmptyEnvelope(): SecretsFile {
-  return { $schema: SCHEMA_REL, version: 1, llm: {}, channels: {} }
+  return { $schema: SCHEMA_REL, version: SECRETS_FILE_VERSION, providers: {}, channels: {} }
 }
 
 function stringifyEnvelope(envelope: SecretsFile): string {
   return `${JSON.stringify(envelope, null, 2)}\n`
 }
 
-function mergeLlmIntoEnvelope(envelope: SecretsFile, nextLlmJson: string): SecretsFile {
+type ReadSnapshot = Map<string, string>
+
+// Build the flat shape AuthStorage expects, resolving Secret-typed api-key
+// keys to plain strings on the way out. Also capture each resolved api-key
+// value into a snapshot keyed by providerId; the write path uses this
+// snapshot (NOT a re-resolution against current process.env) to detect
+// untouched providers. OAuth and unknown types are passed through verbatim
+// and never enter the snapshot — they don't participate in env-wins.
+function flattenProvidersForAuthStorage(
+  providers: Providers,
+  env: NodeJS.ProcessEnv,
+): { flatJson: string; readSnapshot: ReadSnapshot } {
+  const flat: Record<string, unknown> = {}
+  const readSnapshot: ReadSnapshot = new Map()
+  for (const [providerId, cred] of Object.entries(providers)) {
+    if (cred.type === 'api_key') {
+      const defaultEnv = providerKeyDefaultEnv(providerId)
+      const resolved = resolveSecret(cred.key, defaultEnv, env) ?? cred.key.value ?? ''
+      flat[providerId] = { type: 'api_key', key: resolved }
+      readSnapshot.set(providerId, resolved)
+    } else {
+      flat[providerId] = cred
+    }
+  }
+  return { flatJson: JSON.stringify(flat, null, 2), readSnapshot }
+}
+
+// Diff-and-preserve merge per the bridge idempotency rule. AuthStorage hands
+// back the full flat provider slice; we walk it credential-by-credential and
+// decide for each provider whether to:
+//   - preserve the prior on-disk Secret bytes verbatim (untouched provider,
+//     detected by comparing AuthStorage's flat value to the read-time
+//     snapshot, NOT a re-resolution against current env),
+//   - reconstruct as Secret with prior `env` preserved (api-key value changed),
+//   - write as new shape (provider added),
+// and we drop providers that disappeared from the flat slice (real removal).
+function mergeProvidersIntoEnvelope(
+  envelope: SecretsFile,
+  nextProvidersJson: string,
+  readSnapshot: ReadSnapshot,
+): SecretsFile {
   let parsed: unknown
   try {
-    parsed = JSON.parse(nextLlmJson)
+    parsed = JSON.parse(nextProvidersJson)
   } catch (err) {
     throw new Error(
-      `AuthStorage produced invalid JSON for the llm slice: ${err instanceof Error ? err.message : String(err)}`,
+      `AuthStorage produced invalid JSON for the providers slice: ${err instanceof Error ? err.message : String(err)}`,
     )
   }
   if (!isPlainObject(parsed)) {
-    throw new Error('AuthStorage produced a non-object llm slice')
+    throw new Error('AuthStorage produced a non-object providers slice')
   }
+
+  const nextProviders: Providers = {}
+  for (const [providerId, raw] of Object.entries(parsed)) {
+    const reconstructed = reconstructProviderCredential(
+      raw,
+      envelope.providers[providerId],
+      readSnapshot.get(providerId),
+    )
+    if (reconstructed !== undefined) {
+      nextProviders[providerId] = reconstructed
+    }
+  }
+
   return {
     ...envelope,
     $schema: envelope.$schema ?? SCHEMA_REL,
-    llm: parsed as SecretsFile['llm'],
+    version: SECRETS_FILE_VERSION,
+    providers: nextProviders,
+  }
+}
+
+function reconstructProviderCredential(
+  raw: unknown,
+  priorOnDisk: ProviderCredential | undefined,
+  resolvedAtRead: string | undefined,
+): ProviderCredential | undefined {
+  if (!isPlainObject(raw)) return undefined
+  const type = raw['type']
+
+  if (type === 'api_key') {
+    const flatKey = typeof raw['key'] === 'string' ? raw['key'] : ''
+
+    // Empty/missing key from AuthStorage on an api-key credential cannot
+    // round-trip: the schema requires `value` to be non-empty, so writing
+    // `{ value: '' }` would make the file unparseable on next read. Treat
+    // it as "no-op" and preserve the prior on-disk Secret if any. A real
+    // deletion comes through as the provider being absent from `next`,
+    // which is handled by the caller dropping it.
+    if (flatKey === '') {
+      if (priorOnDisk !== undefined) return priorOnDisk
+      return undefined
+    }
+
+    // Idempotency: if AuthStorage's flat key matches the resolved value
+    // captured at read time, the credential is untouched. Preserve the
+    // on-disk Secret verbatim — including any `env` binding and any string
+    // shorthand the user authored. Comparing against the read-time snapshot
+    // (not a re-resolution against current process.env) is what makes this
+    // safe against env mutations between read and write.
+    if (priorOnDisk && priorOnDisk.type === 'api_key' && resolvedAtRead === flatKey) {
+      return priorOnDisk
+    }
+
+    // Mutation path: rewrap as Secret, preserving the user's `env` binding
+    // when prior was also an api-key so the next boot's env-wins still
+    // consults the right variable.
+    if (priorOnDisk && priorOnDisk.type === 'api_key' && priorOnDisk.key.env !== undefined) {
+      return { type: 'api_key', key: { value: flatKey, env: priorOnDisk.key.env } satisfies Secret }
+    }
+
+    return { type: 'api_key', key: { value: flatKey } }
   }
+
+  if (type === 'oauth') {
+    // OAuth credentials are not env-injectable. Pass through verbatim,
+    // preserving every upstream-controlled field (access, refresh, expires,
+    // and any future additions covered by the catchall).
+    return raw as ProviderCredential
+  }
+
+  // Unknown credential type. Pass through verbatim as a defensive measure
+  // against upstream adding a third type in a future release. Better to
+  // round-trip user data than drop it.
+  return raw as ProviderCredential
 }
 
 function isPlainObject(value: unknown): value is Record<string, unknown> {

package/src/skills/typeclaw-config/SKILL.md

@@ -539,17 +539,56 @@ Do **not** edit `typeclaw.json` to a model the registry doesn't know, even if th
 
 `typeclaw.json` does **not** hold API keys or OAuth tokens. Credentials live in two gitignored files:
 
-- **`./.env`** (API key providers): the env var depends on which provider's model you've selected.
-  - `OPENAI_API_KEY` — required for any `openai/...` model.
-  - `FIREWORKS_API_KEY` — required for any `fireworks/...` model.
-- **`./secrets.json`** (OAuth providers): structured JSON file managed by `pi-coding-agent`'s `AuthStorage`, wrapped by `SecretsBackend`. Contains refresh + access tokens. The container refreshes tokens on its own with file locking; the host writes once at `typeclaw init` time when the user picks "OAuth (browser login)". (Pre-rename agent folders may carry the file as `auth.json`; it is migrated to `secrets.json` on the next agent boot.)
-  - `openai-codex/...` models — credentials persisted under the `llm` slice as `{ "llm": { "openai-codex": { "type": "oauth", ... } } }`.
+- **`./.env`** (any environment variable, including API keys): plain `KEY=value` lines, loaded by Docker via `--env-file` at container start. The canonical env-var names per provider:
+  - `OPENAI_API_KEY` — for any `openai/...` model.
+  - `FIREWORKS_API_KEY` — for any `fireworks/...` model.
+- **`./secrets.json`** (structured store): a `v2` envelope managed by `SecretsBackend` (wraps `pi-coding-agent`'s `AuthStorage`). Two top-level slices:
+  - `providers.*` — per-provider credentials. API-key providers store `{ type: 'api_key', key: <Secret> }`. OAuth providers store the `pi-coding-agent` token blob `{ type: 'oauth', access_token, refresh_token, expires_at, ... }`. The container auto-refreshes OAuth tokens with file locking; api-key writes only happen on explicit user-driven rotation.
+  - `channels.*` — per-adapter credentials, with named fields per adapter:
+    - `discord-bot: { token: <Secret> }`
+    - `slack-bot: { botToken: <Secret>, appToken: <Secret> }`
+    - `telegram-bot: { token: <Secret> }`
 
-If a user wants to switch from API key to OAuth (or vice versa) for a provider that supports both, the easiest path is to delete the relevant entry from `.env` / `secrets.json` and re-run `typeclaw init` from inside the agent folder — it'll prompt for the auth method again.
+  (Pre-v2 agent folders carry the older `llm` slice and channel-env-var-keyed shape; they are upgraded transparently on first read. Pre-rename folders may even carry the file as `auth.json`; it is renamed to `secrets.json` on the next boot.)
 
-If the user wants to rotate or change the key, edit `.env`, not `typeclaw.json`. After editing `.env`, the same restart rule applies: `typeclaw restart` on the host stage.
+### The `Secret` shape and env-wins resolution
 
-Never echo, log, or commit values from `.env`. `.env` is gitignored by default — keep it that way.
+Every secret-bearing field in `secrets.json` is a **`Secret`**: either a plain string or an object `{ value?, env? }`.
+
+```json
+{
+  "version": 2,
+  "providers": {
+    "fireworks": { "type": "api_key", "key": "fw_xxx" },
+    "openai-codex": { "type": "oauth", "access_token": "...", "refresh_token": "...", "expires_at": 99 }
+  },
+  "channels": {
+    "slack-bot": {
+      "botToken": "xoxb-...",
+      "appToken": { "value": "xapp-...", "env": "MY_CUSTOM_SLACK_APP_TOKEN" }
+    }
+  }
+}
+```
+
+**Resolution at boot, in order:**
+
+1. `process.env[secret.env]` — explicit binding wins (the `env` field on the object form).
+2. `process.env[<canonical env name>]` — canonical-env fallback (`SLACK_BOT_TOKEN`, `FIREWORKS_API_KEY`, etc.).
+3. `secret.value` — the on-disk value.
+4. Otherwise the field is treated as missing.
+
+**Env wins, the file is never auto-mutated.** When the env var is set, that value is used in-memory via `setRuntimeApiKey` (api-keys) or `process.env` injection (channels) — `secrets.json` is **not** rewritten to capture the env value. The user's file stays user-owned.
+
+**Custom env-var binding** — the optional `env` field on the object form lets the user route a credential through an env var of their choosing (e.g., a CI system that exposes `MY_PROD_SLACK_TOKEN` instead of `SLACK_BOT_TOKEN`).
+
+### Switching credentials
+
+If a user wants to switch from API key to OAuth (or vice versa) for a provider that supports both, the easiest path is to delete the relevant entry from `.env` / `secrets.json#providers` and re-run `typeclaw init` from inside the agent folder — it'll prompt for the auth method again.
+
+If the user wants to rotate an api-key, edit either `.env` (env-wins picks it up immediately) or `secrets.json#providers.<provider>.key` (rewrite the `value` field, or remove the entry if the env var should take over). After either, `typeclaw restart` on the host stage.
+
+Never echo, log, or commit values from `.env` or `secrets.json`. Both are gitignored by default — keep them that way.
 
 ## Editing `typeclaw.json` safely
 

package/typeclaw.schema.json

@@ -708,12 +708,12 @@
     },
     "network": {
       "default": {
-        "blockInternal": false
+        "blockInternal": true
       },
       "type": "object",
       "properties": {
         "blockInternal": {
-          "default": false,
+          "default": true,
           "type": "boolean"
         }
       }
@@ -792,6 +792,40 @@
         }
       }
     },
+    "tool-result-cap": {
+      "default": {
+        "enabled": true,
+        "imageMaxBytes": 262144,
+        "textMaxBytes": 65536,
+        "exemptTools": []
+      },
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "default": true,
+          "type": "boolean"
+        },
+        "imageMaxBytes": {
+          "default": 262144,
+          "type": "integer",
+          "minimum": 1024,
+          "maximum": 9007199254740991
+        },
+        "textMaxBytes": {
+          "default": 65536,
+          "type": "integer",
+          "minimum": 1024,
+          "maximum": 9007199254740991
+        },
+        "exemptTools": {
+          "default": [],
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "memory": {
       "default": {
         "idleMs": 10000,

package/src/secrets/env.ts

@@ -1,43 +0,0 @@
-import { readFileSync, writeFileSync } from 'node:fs'
-
-// No-op when the file is missing or the key is absent: the caller has
-// already persisted to `secrets.json` and just wants `.env` to stop being a
-// second source of truth. Parsing matches `parseEnvKeys` in
-// `src/init/index.ts` — line-based, trim, skip blanks/comments, split on the
-// first `=`. Duplicate assignments to the same key are all removed because
-// dotenv resolves "last wins" so every duplicate carries the value we just
-// promoted.
-export function stripEnvKey(path: string, key: string): void {
-  let original: string
-  try {
-    original = readFileSync(path, 'utf8')
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return
-    throw error
-  }
-
-  const next = removeKeyFromEnvText(original, key)
-  if (next === original) return
-  writeFileSync(path, next)
-}
-
-export function removeKeyFromEnvText(content: string, key: string): string {
-  const lines = content.split('\n')
-  const kept: string[] = []
-  for (const line of lines) {
-    const trimmed = line.trim()
-    if (trimmed === '' || trimmed.startsWith('#')) {
-      kept.push(line)
-      continue
-    }
-    const eq = trimmed.indexOf('=')
-    if (eq <= 0) {
-      kept.push(line)
-      continue
-    }
-    const lineKey = trimmed.slice(0, eq).trim()
-    if (lineKey === key) continue
-    kept.push(line)
-  }
-  return kept.join('\n')
-}