pi-warden 0.28.4 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +158 -1
  2. package/README.md +5 -1
  3. package/dist/audit.d.ts +22 -0
  4. package/dist/audit.js +110 -0
  5. package/dist/config.d.ts +14 -0
  6. package/dist/config.js +12 -1
  7. package/dist/conscience-config.d.ts +21 -0
  8. package/dist/conscience-config.js +82 -0
  9. package/dist/conscience-loader.d.ts +26 -0
  10. package/dist/conscience-loader.js +82 -0
  11. package/dist/conscience-policy.d.ts +24 -0
  12. package/dist/conscience-policy.js +28 -0
  13. package/dist/conscience-privacy.d.ts +4 -0
  14. package/dist/conscience-privacy.js +11 -0
  15. package/dist/discover.d.ts +14 -0
  16. package/dist/discover.js +85 -0
  17. package/dist/extension.d.ts +1 -1
  18. package/dist/extension.js +261 -38
  19. package/dist/extension.js.bak +213 -35
  20. package/dist/guard.d.ts +127 -1
  21. package/dist/guard.js +447 -31
  22. package/dist/holds.d.ts +2 -0
  23. package/dist/holds.js +7 -1
  24. package/dist/index.d.ts +9 -3
  25. package/dist/index.js +5 -2
  26. package/dist/init.d.ts +36 -0
  27. package/dist/init.js +199 -0
  28. package/dist/laya-download.d.ts +40 -0
  29. package/dist/laya-download.js +288 -0
  30. package/dist/laya-judge.d.ts +62 -0
  31. package/dist/laya-judge.js +495 -0
  32. package/dist/learning.d.ts +30 -0
  33. package/dist/learning.js +53 -10
  34. package/dist/redact.js +9 -4
  35. package/dist/rules-file.d.ts +18 -0
  36. package/dist/rules-file.js +72 -0
  37. package/dist/rules.d.ts +8 -0
  38. package/dist/rules.js +54 -3
  39. package/dist/shape.js +10 -2
  40. package/dist/stuck.d.ts +9 -0
  41. package/dist/stuck.js +101 -0
  42. package/dist/tool-recommendation.d.ts +15 -0
  43. package/dist/tool-recommendation.js +31 -0
  44. package/dist/widget.d.ts +3 -1
  45. package/dist/widget.js +16 -13
  46. package/package.json +2 -2
package/dist/holds.js CHANGED
@@ -26,10 +26,11 @@ export class HoldLedger {
26
26
  /** One inspected call. `outcome` is set at once when the confirm dialog decided; a steer-mode hold starts pending. */
27
27
  record(verdict, options) {
28
28
  const at = options.at ?? Date.now();
29
+ const { summary } = verdict;
29
30
  const record = {
30
31
  id: this.nextId++,
31
32
  at,
32
- tool: verdict.summary.tool,
33
+ tool: summary.tool,
33
34
  level: verdict.level,
34
35
  source: verdict.source,
35
36
  mode: options.mode,
@@ -39,6 +40,11 @@ export class HoldLedger {
39
40
  planChars: verdict.plan?.length ?? 0,
40
41
  outcome: options.outcome ?? "pending",
41
42
  };
43
+ // Call excerpt for auditing: tool + path only (commands never reach the log).
44
+ if (summary.path)
45
+ record.callExcerpt = `${summary.tool} ${summary.path}`;
46
+ else if (summary.tool)
47
+ record.callExcerpt = summary.tool;
42
48
  const scores = scoresOf(verdict);
43
49
  if (scores)
44
50
  record.scores = scores;
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- export { evaluateAction, describeAction, describePlan, matchPatterns, isReadOnlyCommand, stripDataText, buildRequest, formatVerdict, formatVerdictTokens, steerReason, intentSteer, offTaskSteer, steerFingerprint, SteerRepeatWindow, questions, intentQuestion, visibleQuestion, slopQuestions, approvalQuestion, securityQuestion, regretQuestions } from "./guard.js";
2
- export type { ActionInput, ActionSummary, EvaluateOptions, Judge, Judgment, SlopJudgment, SlopSymptom, Level, PatternHit, PreviousAction, ScannedCommand, ScopeLabel, Severity, Verdict } from "./guard.js";
1
+ export { evaluateAction, describeAction, describePlan, matchPatterns, isReadOnlyCommand, stripDataText, buildRequest, formatVerdict, formatVerdictTokens, steerReason, intentSteer, offTaskSteer, shouldProceedMessage, steerFingerprint, SteerRepeatWindow, questions, intentQuestion, visibleQuestion, slopQuestions, approvalQuestion, securityQuestion, regretQuestions, authorize, isNegated, scopeMatches, isAuthEligible, patternHitsToViolations, escalateBlastRadius, escalateRulesViolation, aggregateLevel, removeAuthorized, scopeFromInput, matchPathRules, parseViolationJudgments } from "./guard.js";
2
+ export type { ActionInput, ActionSummary, EvaluateOptions, Judge, Judgment, SlopJudgment, SlopSymptom, Level, PatternHit, PreviousAction, ScannedCommand, ScopeLabel, Severity, Verdict, Violation, ViolationSource, ViolationScope, Authorization, EscalatedViolation, ViolationJudgmentAnswer } from "./guard.js";
3
3
  export { SLOP_SYMPTOMS, SLOP_LABELS } from "./guard.js";
4
4
  export { ActionGuard } from "./action-guard.js";
5
5
  export type { Conversation, InspectOptions, ToolCallRef } from "./action-guard.js";
@@ -36,7 +36,13 @@ export { renderTemplate, actionTokens, stuckTokens, doneTokens, proseTokens, run
36
36
  export type { WidgetConfig, WidgetPlacement, WidgetBarMode, Tokens } from "./widget.js";
37
37
  export { Trace, actionDetails, stuckDetails, doneDetails, proseDetails, runawayDetails, rulesDetails } from "./trace.js";
38
38
  export type { TraceEntry, GuardName } from "./trace.js";
39
- export { parseRules, condense, globToRegExp, matchGlob, projectPath, RuleStore, rulesFor, describeRuleSet, describeTarget, buildRulesRequest, ruleQuestion, evaluateRules, skipReason, rulesSteer, formatRules, pathNotes, pathNoteSteer, RulesGuard, RULES_FILE, FALLBACK_FILES, MAX_RULES, AGGREGATE_QUESTION, LOCATOR_QUESTION } from "./rules.js";
39
+ export { parseRules, condense, isRuleShaped, globToRegExp, matchGlob, projectPath, RuleStore, rulesFor, describeRuleSet, describeTarget, buildRulesRequest, ruleQuestion, evaluateRules, skipReason, rulesSteer, formatRules, pathNotes, pathNoteSteer, RulesGuard, RULES_FILE, FALLBACK_FILES, MAX_RULES, AGGREGATE_QUESTION, LOCATOR_QUESTION } from "./rules.js";
40
40
  export type { Rule, RuleSet, RuleOutcome, RuleScore, RuleFinding, RulesVerdict, RulesOptions, RulesTarget, EditView, PathNote, RulesCallRef } from "./rules.js";
41
+ export { resolveRulesFile, extractRules, checkPiWardenMissing } from "./rules-file.js";
42
+ export type { ResolvedRulesFile } from "./rules-file.js";
43
+ export { writeStarterRules, generateStarterRules, detectProjectType, buildProjectContext, buildInitPrompt } from "./init.js";
44
+ export type { InitResult } from "./init.js";
41
45
  export { initSchema, recordHold, recordOutcome as recordHoldOutcome, querySmartHistory, calculateSmartConfidence, shouldSkipHold, analyzeThresholds, analyzePatterns, generateRecommendations, analyzeSteerEffectivenessReport } from "./learning.js";
42
46
  export type { HoldRecord, HoldScores, HoldLevel, HoldOutcome, HoldContext, SmartHistory, ConfidenceResult, SkipResult, ThresholdAdjustment, PatternInsight, ContextRecommendation, SteerEffectivenessReport } from "./learning.js";
47
+ export { buildAuditPrompt, snapshotReport, reportOutcome } from "./audit.js";
48
+ export type { ReportSnapshot } from "./audit.js";
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- export { evaluateAction, describeAction, describePlan, matchPatterns, isReadOnlyCommand, stripDataText, buildRequest, formatVerdict, formatVerdictTokens, steerReason, intentSteer, offTaskSteer, steerFingerprint, SteerRepeatWindow, questions, intentQuestion, visibleQuestion, slopQuestions, approvalQuestion, securityQuestion, regretQuestions } from "./guard.js";
1
+ export { evaluateAction, describeAction, describePlan, matchPatterns, isReadOnlyCommand, stripDataText, buildRequest, formatVerdict, formatVerdictTokens, steerReason, intentSteer, offTaskSteer, shouldProceedMessage, steerFingerprint, SteerRepeatWindow, questions, intentQuestion, visibleQuestion, slopQuestions, approvalQuestion, securityQuestion, regretQuestions, authorize, isNegated, scopeMatches, isAuthEligible, patternHitsToViolations, escalateBlastRadius, escalateRulesViolation, aggregateLevel, removeAuthorized, scopeFromInput, matchPathRules, parseViolationJudgments } from "./guard.js";
2
2
  export { SLOP_SYMPTOMS, SLOP_LABELS } from "./guard.js";
3
3
  export { ActionGuard } from "./action-guard.js";
4
4
  export { evaluateProse, buildProseRequest, proseQuestions, proseNudge, describeAudience, ProseTrend, PROSE_SYMPTOMS, PROSE_LABELS } from "./prose.js";
@@ -18,5 +18,8 @@ export { CONFIG_SCHEMA, defaultConfig, loadConfig, applyUserOverrides, applyProj
18
18
  export { commandOf, outputReportsFailure, COMMAND_TOOLS } from "./tools.js";
19
19
  export { renderTemplate, actionTokens, stuckTokens, doneTokens, proseTokens, runawayTokens, rulesTokens, defaultWidgetConfig, DEFAULT_TEMPLATES, SENTENCE_TEMPLATES, pickSentenceTemplate, TOKEN_NAMES } from "./widget.js";
20
20
  export { Trace, actionDetails, stuckDetails, doneDetails, proseDetails, runawayDetails, rulesDetails } from "./trace.js";
21
- export { parseRules, condense, globToRegExp, matchGlob, projectPath, RuleStore, rulesFor, describeRuleSet, describeTarget, buildRulesRequest, ruleQuestion, evaluateRules, skipReason, rulesSteer, formatRules, pathNotes, pathNoteSteer, RulesGuard, RULES_FILE, FALLBACK_FILES, MAX_RULES, AGGREGATE_QUESTION, LOCATOR_QUESTION } from "./rules.js";
21
+ export { parseRules, condense, isRuleShaped, globToRegExp, matchGlob, projectPath, RuleStore, rulesFor, describeRuleSet, describeTarget, buildRulesRequest, ruleQuestion, evaluateRules, skipReason, rulesSteer, formatRules, pathNotes, pathNoteSteer, RulesGuard, RULES_FILE, FALLBACK_FILES, MAX_RULES, AGGREGATE_QUESTION, LOCATOR_QUESTION } from "./rules.js";
22
+ export { resolveRulesFile, extractRules, checkPiWardenMissing } from "./rules-file.js";
23
+ export { writeStarterRules, generateStarterRules, detectProjectType, buildProjectContext, buildInitPrompt } from "./init.js";
22
24
  export { initSchema, recordHold, recordOutcome as recordHoldOutcome, querySmartHistory, calculateSmartConfidence, shouldSkipHold, analyzeThresholds, analyzePatterns, generateRecommendations, analyzeSteerEffectivenessReport } from "./learning.js";
25
+ export { buildAuditPrompt, snapshotReport, reportOutcome } from "./audit.js";
package/dist/init.d.ts ADDED
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Detect a rough project type from common manifest files.
3
+ * Returns a label like "typescript", "rust", "python", "go", or "generic".
4
+ */
5
+ export declare function detectProjectType(cwd: string): string;
6
+ /**
7
+ * Build project context for the starter rules file.
8
+ * Caps total context at 8000 characters.
9
+ */
10
+ export declare function buildProjectContext(cwd: string): string;
11
+ /**
12
+ * Generate a starter pi-warden.md file.
13
+ * Returns the content; does not write it.
14
+ */
15
+ export declare function generateStarterRules(cwd: string): string;
16
+ /**
17
+ * Build a ready-to-paste prompt for the model to generate pi-warden.md.
18
+ * Includes extracted project context and standard safety rules as a foundation.
19
+ */
20
+ export declare function buildInitPrompt(cwd: string): string;
21
+ /**
22
+ * Result of an /warden init attempt.
23
+ */
24
+ export interface InitResult {
25
+ /** Whether a pi-warden.md already existed. */
26
+ alreadyExists: boolean;
27
+ /** The path that was written (or would be overwritten). */
28
+ path: string;
29
+ /** The content written. */
30
+ content: string;
31
+ }
32
+ /**
33
+ * Write a starter pi-warden.md to the project root.
34
+ * If it already exists, returns alreadyExists: true without writing.
35
+ */
36
+ export declare function writeStarterRules(cwd: string, overwrite?: boolean): InitResult;
package/dist/init.js ADDED
@@ -0,0 +1,199 @@
1
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { FALLBACK_FILES } from "./rules.js";
4
+ /** Paths to scan for project context when generating a starter rules file. */
5
+ const CONTEXT_CANDIDATES = [
6
+ "package.json",
7
+ "tsconfig.json",
8
+ "Cargo.toml",
9
+ "pyproject.toml",
10
+ "go.mod",
11
+ ];
12
+ /**
13
+ * Detect a rough project type from common manifest files.
14
+ * Returns a label like "typescript", "rust", "python", "go", or "generic".
15
+ */
16
+ export function detectProjectType(cwd) {
17
+ if (existsSync(join(cwd, "tsconfig.json")))
18
+ return "typescript";
19
+ if (existsSync(join(cwd, "Cargo.toml")))
20
+ return "rust";
21
+ if (existsSync(join(cwd, "pyproject.toml")))
22
+ return "python";
23
+ if (existsSync(join(cwd, "go.mod")))
24
+ return "go";
25
+ if (existsSync(join(cwd, "package.json")))
26
+ return "javascript";
27
+ return "generic";
28
+ }
29
+ /**
30
+ * Read a manifest file and extract a brief summary (name, scripts, dependencies).
31
+ * Returns a short string, or null if the file doesn't parse.
32
+ */
33
+ function manifestSummary(cwd, filename) {
34
+ const path = join(cwd, filename);
35
+ if (!existsSync(path))
36
+ return null;
37
+ try {
38
+ const raw = JSON.parse(readFileSync(path, "utf8"));
39
+ const parts = [];
40
+ if (raw.name)
41
+ parts.push(`name: ${raw.name}`);
42
+ if (raw.scripts) {
43
+ const scripts = Object.keys(raw.scripts).slice(0, 8);
44
+ if (scripts.length)
45
+ parts.push(`scripts: ${scripts.join(", ")}`);
46
+ }
47
+ if (raw.dependencies) {
48
+ const deps = Object.keys(raw.dependencies).slice(0, 10);
49
+ if (deps.length)
50
+ parts.push(`deps: ${deps.join(", ")}`);
51
+ }
52
+ return parts.join("; ") || null;
53
+ }
54
+ catch (err) {
55
+ // Best-effort: unparseable manifest is not a failure, just skip this candidate.
56
+ console.warn("pi-warden: could not parse manifest:", err);
57
+ return null;
58
+ }
59
+ }
60
+ /**
61
+ * Build project context for the starter rules file.
62
+ * Caps total context at 8000 characters.
63
+ */
64
+ export function buildProjectContext(cwd) {
65
+ const parts = [];
66
+ const projectType = detectProjectType(cwd);
67
+ parts.push(`Project type: ${projectType}`);
68
+ for (const candidate of CONTEXT_CANDIDATES) {
69
+ const summary = manifestSummary(cwd, candidate);
70
+ if (summary)
71
+ parts.push(`${candidate}: ${summary}`);
72
+ }
73
+ const context = parts.join("\n");
74
+ return context.length > 8000 ? context.slice(0, 8000) + "\n...(truncated)" : context;
75
+ }
76
+ /** Read the first existing fallback rules file's content, or null if none exists. */
77
+ function readExistingRules(cwd) {
78
+ for (const file of FALLBACK_FILES) {
79
+ const fullPath = join(cwd, file);
80
+ if (existsSync(fullPath)) {
81
+ try {
82
+ return readFileSync(fullPath, "utf8");
83
+ }
84
+ catch {
85
+ continue;
86
+ }
87
+ }
88
+ }
89
+ return null;
90
+ }
91
+ /** Read the existing pi-warden.md content, or null if it doesn't exist. */
92
+ function readExistingPiWarden(cwd) {
93
+ const fullPath = join(cwd, "pi-warden.md");
94
+ if (!existsSync(fullPath))
95
+ return null;
96
+ try {
97
+ return readFileSync(fullPath, "utf8");
98
+ }
99
+ catch {
100
+ return null;
101
+ }
102
+ }
103
+ /** Standard safety rules that ship in every starter pi-warden.md. */
104
+ const SAFETY_RULES = `# No hardcoded secrets
105
+ Source code must not contain passwords, API keys, tokens, or connection URLs with credentials.
106
+ Read them from the environment, a function parameter, or the config module.
107
+
108
+ # Comments explain why, not what
109
+ A comment states a reason, a constraint, a workaround, or a non-obvious invariant. A comment
110
+ that restates what the next line plainly does is a violation.
111
+
112
+ # Errors are not swallowed
113
+ A \`catch\` block must handle the error, report it, or re-raise it. An empty catch block, or
114
+ one whose body is only a comment, is a violation.
115
+
116
+ # No partial implementations
117
+ Implement features fully. A comment that says "for now", "simplified", or "later", or a
118
+ stub body, is a violation. If a part genuinely cannot be done, say so in your reply instead
119
+ of stubbing it.
120
+
121
+ # Do not run destructive commands that erase uncommitted work
122
+ \`git reset --hard\`, \`git checkout -- .\`, \`git clean -fd\`, and similar commands that discard
123
+ untracked or uncommitted changes are forbidden. These destroy work that has no backup. If a
124
+ clean tree is needed, create a worktree instead or ask the user.
125
+ `;
126
+ /**
127
+ * Generate a starter pi-warden.md file.
128
+ * Returns the content; does not write it.
129
+ */
130
+ export function generateStarterRules(cwd) {
131
+ const projectType = detectProjectType(cwd);
132
+ const context = buildProjectContext(cwd);
133
+ const existingRules = readExistingRules(cwd);
134
+ const lines = [
135
+ "Copy this file to the root of your project as `pi-warden.md` and edit it.",
136
+ "Every `#` heading below is one rule. The text under a heading is what Jev reads when it judges a write or an edit.",
137
+ "",
138
+ `<!-- Project context: ${context} -->`,
139
+ "",
140
+ SAFETY_RULES,
141
+ ];
142
+ const existingPiWarden = readExistingPiWarden(cwd);
143
+ if (existingPiWarden) {
144
+ lines.push("", "## Existing pi-warden.md rules (preserved)", existingPiWarden, "");
145
+ }
146
+ else if (existingRules) {
147
+ lines.push("", "## Existing project rules (preserved from fallback file)", existingRules, "");
148
+ }
149
+ // Add type-specific rules.
150
+ if (projectType === "typescript" || projectType === "javascript") {
151
+ lines.push("# No explicit any", "paths: **/*.ts, **/*.tsx", "Do not use the `any` type. Use a precise type, `unknown` with a runtime check, or a generic parameter.", "", "# Exported functions declare their return type", "paths: src/**/*.ts", "Every exported function or method declares its return type instead of relying on inference.", "");
152
+ }
153
+ lines.push("# New exported functions get a test", "paths: src/**", "A newly added exported function or class comes with at least one test that exercises its main behaviour.", "");
154
+ return lines.join("\n");
155
+ }
156
+ /**
157
+ * Build a ready-to-paste prompt for the model to generate pi-warden.md.
158
+ * Includes extracted project context and standard safety rules as a foundation.
159
+ */
160
+ export function buildInitPrompt(cwd) {
161
+ const context = buildProjectContext(cwd);
162
+ const projectType = detectProjectType(cwd);
163
+ const existingRules = readExistingRules(cwd);
164
+ const lines = [
165
+ "Create a `pi-warden.md` file for this project. Every `#` heading is one rule; the text under it is what Jev judges against.",
166
+ "",
167
+ "## Project context",
168
+ context,
169
+ "",
170
+ "## Rules to always include",
171
+ SAFETY_RULES,
172
+ ];
173
+ const existingPiWarden = readExistingPiWarden(cwd);
174
+ if (existingPiWarden) {
175
+ lines.push("", "## Existing pi-warden.md rules (preserve these)", existingPiWarden, "");
176
+ }
177
+ else if (existingRules) {
178
+ lines.push("", "## Existing project rules (preserve these)", existingRules, "");
179
+ }
180
+ if (projectType === "typescript" || projectType === "javascript") {
181
+ lines.push("## Type-specific rules", "- No explicit `any` in **/*.ts, **/*.tsx", "- Exported functions declare their return type in src/**/*.ts", "");
182
+ }
183
+ lines.push("## Requirements", "- Under 50 rules", "- Each rule is a `#` heading with a description under it", "- Optional `paths:` line under a heading limits the rule to matching files", "- No stubs, no TODOs, no placeholders", "- Write the file to pi-warden.md at the project root");
184
+ return lines.join("\n");
185
+ }
186
+ /**
187
+ * Write a starter pi-warden.md to the project root.
188
+ * If it already exists, returns alreadyExists: true without writing.
189
+ */
190
+ export function writeStarterRules(cwd, overwrite = false) {
191
+ const targetPath = join(cwd, "pi-warden.md");
192
+ const alreadyExists = existsSync(targetPath);
193
+ if (alreadyExists && !overwrite) {
194
+ return { alreadyExists: true, path: targetPath, content: readFileSync(targetPath, "utf8") };
195
+ }
196
+ const content = generateStarterRules(cwd);
197
+ writeFileSync(targetPath, content, "utf8");
198
+ return { alreadyExists, path: targetPath, content };
199
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Download laya-mlx model weights from HuggingFace with a progress bar.
3
+ *
4
+ * Downloads to ~/.pi/agent/pi-warden/laya/ using an atomic staging strategy:
5
+ * files are written to a .staging subdirectory first, then promoted to the final
6
+ * location only after all files are present and the required files are validated.
7
+ * A .done marker signals a complete, validated checkpoint.
8
+ */
9
+ /** The directory where model weights are stored. */
10
+ export declare function layaModelDir(): string;
11
+ /**
12
+ * True when the model directory exists, the .done marker is present,
13
+ * and all required files are on disk with non-zero size.
14
+ */
15
+ export declare function layaModelReady(): boolean;
16
+ /**
17
+ * Download the laya-mlx model. Shows a progress bar in the terminal.
18
+ * Returns the model directory path.
19
+ *
20
+ * Uses atomic staging: files are written to a .staging subdirectory,
21
+ * then promoted to the final location only after all files download
22
+ * successfully. An interrupted download leaves .staging behind, which
23
+ * is cleaned up on the next startup.
24
+ *
25
+ * @param onStatus - callback for status messages
26
+ */
27
+ export declare function downloadLayaModel(onStatus?: (msg: string) => void): Promise<string>;
28
+ /**
29
+ * The directory where the pi-warden-managed Laya virtual environment lives.
30
+ * Uses the same agent dir as the model, keeping all Laya state together.
31
+ */
32
+ export declare function layaVenvDir(): string;
33
+ /** True when the venv exists and has laya importable. */
34
+ export declare function layaVenvReady(): boolean;
35
+ /** The Python executable inside the venv. */
36
+ export declare function layaVenvPython(): string;
37
+ /** Mark the venv as ready after successful installation. */
38
+ export declare function markVenvReady(): Promise<void>;
39
+ /** Remove the venv done marker so the next startup rebuilds it. */
40
+ export declare function invalidateVenv(): void;
@@ -0,0 +1,288 @@
1
+ /**
2
+ * Download laya-mlx model weights from HuggingFace with a progress bar.
3
+ *
4
+ * Downloads to ~/.pi/agent/pi-warden/laya/ using an atomic staging strategy:
5
+ * files are written to a .staging subdirectory first, then promoted to the final
6
+ * location only after all files are present and the required files are validated.
7
+ * A .done marker signals a complete, validated checkpoint.
8
+ */
9
+ import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
10
+ import { chmod, rename, rm } from "node:fs/promises";
11
+ import { homedir } from "node:os";
12
+ import { join, dirname } from "node:path";
13
+ const HF_API = "https://huggingface.co";
14
+ const MODEL_REPO = "aac6fef/laya-mlx";
15
+ const LAYA_DIR_NAME = "laya";
16
+ /**
17
+ * Files that must exist for a valid checkpoint.
18
+ * Checked after download; missing files trigger a re-download.
19
+ */
20
+ const REQUIRED_FILES = [
21
+ "model.safetensors",
22
+ "encoder/config.json",
23
+ "mlx_config.json",
24
+ "tokenizer/tokenizer.json",
25
+ "tokenizer/tokenizer_config.json",
26
+ "manifest.json",
27
+ ];
28
+ /** The directory where model weights are stored. */
29
+ export function layaModelDir() {
30
+ const configured = process.env.PI_CODING_AGENT_DIR?.trim();
31
+ const agentDir = configured
32
+ ? (configured === "~" || configured.startsWith("~/") ? join(homedir(), configured.slice(1)) : configured)
33
+ : join(homedir(), ".pi", "agent");
34
+ return join(agentDir, LAYA_DIR_NAME);
35
+ }
36
+ /** Marker file that signals a complete, validated download. */
37
+ function doneMarker(dir) {
38
+ return join(dir, ".done");
39
+ }
40
+ /**
41
+ * True when the model directory exists, the .done marker is present,
42
+ * and all required files are on disk with non-zero size.
43
+ */
44
+ export function layaModelReady() {
45
+ const dir = layaModelDir();
46
+ if (!existsSync(doneMarker(dir)))
47
+ return false;
48
+ return validateCheckpoint(dir);
49
+ }
50
+ /** Validate that all required checkpoint files exist with non-zero size. */
51
+ function validateCheckpoint(dir) {
52
+ for (const file of REQUIRED_FILES) {
53
+ const path = join(dir, file);
54
+ try {
55
+ const st = statSync(path);
56
+ if (st.size === 0)
57
+ return false;
58
+ }
59
+ catch {
60
+ return false;
61
+ }
62
+ }
63
+ return true;
64
+ }
65
+ /**
66
+ * Fetch all files in the repo with their sizes.
67
+ *
68
+ * Uses two HuggingFace endpoints because neither alone gives us everything:
69
+ * - `/api/models/{repo}` siblings: full file list (including nested paths like
70
+ * `encoder/config.json`) but no sizes.
71
+ * - `/api/models/{repo}/tree/{ref}/{path}`: sizes for files in one directory,
72
+ * but only returns direct children (not recursive).
73
+ *
74
+ * Strategy: get the full list from siblings, then walk each directory subtree
75
+ * via /tree to fill in sizes.
76
+ */
77
+ async function listRepoFiles(repo) {
78
+ // 1. Get the complete file list from siblings (includes nested paths).
79
+ const metaUrl = `${HF_API}/api/models/${repo}`;
80
+ const metaResp = await fetchWithTimeout(metaUrl, 15_000);
81
+ if (!metaResp.ok)
82
+ throw new Error(`HuggingFace API ${metaResp.status}: ${metaResp.statusText}`);
83
+ const meta = (await metaResp.json());
84
+ if (!meta.siblings)
85
+ throw new Error("no siblings in model info");
86
+ const allPaths = meta.siblings
87
+ .map(s => s.rfilename)
88
+ .filter(p => typeof p === "string" && p.length > 0);
89
+ if (allPaths.length === 0)
90
+ throw new Error("model repo has no files");
91
+ // 2. Collect directory paths that need size lookups.
92
+ const dirs = new Set();
93
+ for (const p of allPaths) {
94
+ const slash = p.indexOf("/");
95
+ if (slash !== -1)
96
+ dirs.add(p.slice(0, slash));
97
+ }
98
+ // 3. Fetch sizes from /tree for each directory (root + subdirs).
99
+ const sizes = new Map();
100
+ const fetchTree = async (dirPath) => {
101
+ const treeUrl = `${HF_API}/api/models/${repo}/tree/main${dirPath ? "/" + dirPath : ""}`;
102
+ const resp = await fetchWithTimeout(treeUrl, 15_000);
103
+ if (!resp.ok)
104
+ return; // non-fatal: we'll download without size info
105
+ const entries = (await resp.json());
106
+ if (!Array.isArray(entries))
107
+ return;
108
+ for (const e of entries) {
109
+ if (typeof e.path === "string" && typeof e.size === "number" && e.size > 0 && e.type !== "directory") {
110
+ // /tree returns repo-relative paths (e.g. "encoder/config.json"),
111
+ // not paths relative to the queried directory.
112
+ sizes.set(e.path, e.size);
113
+ }
114
+ }
115
+ };
116
+ // Fetch root + all known subdirectories in parallel.
117
+ await Promise.all([fetchTree(""), ...[...dirs].map(d => fetchTree(d))]);
118
+ // 4. Build the final file list. Files without a size from /tree get size 0
119
+ // (the download will still work; the progress bar just won't know the total
120
+ // for that file).
121
+ return allPaths
122
+ .filter(p => !p.endsWith("/")) // skip pure directories
123
+ .map(p => ({ path: p, size: sizes.get(p) ?? 0 }));
124
+ }
125
+ /** Download a single file with progress tracking. */
126
+ async function downloadFile(url, dest, totalBytes, progress, onProgress) {
127
+ const response = await fetchWithTimeout(url, 60_000);
128
+ if (!response.ok)
129
+ throw new Error(`download ${url}: ${response.status}`);
130
+ if (!response.body)
131
+ throw new Error(`no body for ${url}`);
132
+ const fileStream = createWriteStream(dest);
133
+ try {
134
+ const reader = response.body.getReader();
135
+ let done = false;
136
+ while (!done) {
137
+ const result = await reader.read();
138
+ done = result.done;
139
+ if (result.value) {
140
+ fileStream.write(result.value);
141
+ progress.downloaded += result.value.length;
142
+ onProgress(progress.downloaded, progress.total);
143
+ }
144
+ }
145
+ fileStream.end();
146
+ await new Promise((resolve, reject) => {
147
+ fileStream.on("finish", resolve);
148
+ fileStream.on("error", reject);
149
+ });
150
+ }
151
+ catch (err) {
152
+ fileStream.destroy();
153
+ throw err;
154
+ }
155
+ }
156
+ /** fetch with a timeout (using AbortController). */
157
+ async function fetchWithTimeout(url, timeoutMs) {
158
+ const controller = new AbortController();
159
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
160
+ try {
161
+ return await fetch(url, { signal: controller.signal, redirect: "follow" });
162
+ }
163
+ finally {
164
+ clearTimeout(timer);
165
+ }
166
+ }
167
+ /**
168
+ * Download the laya-mlx model. Shows a progress bar in the terminal.
169
+ * Returns the model directory path.
170
+ *
171
+ * Uses atomic staging: files are written to a .staging subdirectory,
172
+ * then promoted to the final location only after all files download
173
+ * successfully. An interrupted download leaves .staging behind, which
174
+ * is cleaned up on the next startup.
175
+ *
176
+ * @param onStatus - callback for status messages
177
+ */
178
+ export async function downloadLayaModel(onStatus) {
179
+ const dir = layaModelDir();
180
+ if (layaModelReady())
181
+ return dir;
182
+ // Clean up any leftover staging directory from a previous interrupted download.
183
+ const stagingDir = `${dir}.staging`;
184
+ await rm(stagingDir, { recursive: true, force: true }).catch((err) => {
185
+ console.warn(`pi-warden: could not clean staging dir: ${err instanceof Error ? err.message : err}`);
186
+ });
187
+ // If the final dir exists but is incomplete, remove it so we start fresh.
188
+ if (existsSync(dir) && !validateCheckpoint(dir)) {
189
+ onStatus?.("incomplete model cache detected; re-downloading...");
190
+ await rm(dir, { recursive: true, force: true }).catch((err) => {
191
+ console.warn(`pi-warden: could not remove incomplete model dir: ${err instanceof Error ? err.message : err}`);
192
+ });
193
+ }
194
+ mkdirSync(stagingDir, { recursive: true });
195
+ onStatus?.("please wait, downloading laya-mlx model...");
196
+ // 1. List files from the repo.
197
+ const files = await listRepoFiles(MODEL_REPO);
198
+ if (files.length === 0)
199
+ throw new Error("HuggingFace returned no files for the model repo");
200
+ const totalBytes = files.reduce((sum, f) => sum + f.size, 0);
201
+ // 2. Download each file into the staging directory.
202
+ const progress = { downloaded: 0, total: totalBytes, filesDone: 0, filesTotal: files.length };
203
+ const lastReport = { bytes: 0, time: Date.now() };
204
+ const onProgress = (downloaded, total) => {
205
+ const now = Date.now();
206
+ if (now - lastReport.time < 200 && downloaded < total)
207
+ return;
208
+ lastReport.bytes = downloaded;
209
+ lastReport.time = now;
210
+ const pct = total > 0 ? (downloaded / total) * 100 : 0;
211
+ const mb = (downloaded / (1024 * 1024)).toFixed(1);
212
+ const totalMb = (total / (1024 * 1024)).toFixed(1);
213
+ const barLen = 30;
214
+ const filled = Math.round((pct / 100) * barLen);
215
+ const bar = "█".repeat(filled) + "░".repeat(barLen - filled);
216
+ process.stderr.write(`\r ${bar} ${pct.toFixed(0)}% (${mb}/${totalMb} MB)`);
217
+ };
218
+ for (const file of files) {
219
+ const fileUrl = `${HF_API}/${MODEL_REPO}/resolve/main/${file.path}`;
220
+ const dest = join(stagingDir, file.path);
221
+ mkdirSync(dirname(dest), { recursive: true });
222
+ await downloadFile(fileUrl, dest, file.size, progress, onProgress);
223
+ progress.filesDone++;
224
+ }
225
+ process.stderr.write("\n");
226
+ // 3. Validate required files in the staging directory.
227
+ if (!validateCheckpoint(stagingDir)) {
228
+ await rm(stagingDir, { recursive: true, force: true }).catch((err) => {
229
+ console.warn(`pi-warden: could not clean staging dir after validation failure: ${err instanceof Error ? err.message : err}`);
230
+ });
231
+ throw new Error("laya-mlx: download completed but required checkpoint files are missing");
232
+ }
233
+ // 4. Promote staging to final location (atomic on the same filesystem).
234
+ if (existsSync(dir)) {
235
+ await rm(dir, { recursive: true, force: true }).catch((err) => {
236
+ console.warn(`pi-warden: could not remove old model dir before promote: ${err instanceof Error ? err.message : err}`);
237
+ });
238
+ }
239
+ await rename(stagingDir, dir);
240
+ // 5. Write done marker.
241
+ writeFileSync(doneMarker(dir), JSON.stringify({ files: files.map(f => f.path), totalBytes, downloadedAt: new Date().toISOString() }));
242
+ await chmod(doneMarker(dir), 0o600);
243
+ onStatus?.("laya-mlx model ready.");
244
+ return dir;
245
+ }
246
+ // ---------------------------------------------------------------------------
247
+ // Managed virtual environment for Laya.
248
+ // ---------------------------------------------------------------------------
249
+ /**
250
+ * The directory where the pi-warden-managed Laya virtual environment lives.
251
+ * Uses the same agent dir as the model, keeping all Laya state together.
252
+ */
253
+ export function layaVenvDir() {
254
+ const configured = process.env.PI_CODING_AGENT_DIR?.trim();
255
+ const agentDir = configured
256
+ ? (configured === "~" || configured.startsWith("~/") ? join(homedir(), configured.slice(1)) : configured)
257
+ : join(homedir(), ".pi", "agent");
258
+ return join(agentDir, "laya-venv");
259
+ }
260
+ /** Marker file that signals a working venv with laya importable. */
261
+ function venvDoneMarker(dir) {
262
+ return join(dir, ".done");
263
+ }
264
+ /** True when the venv exists and has laya importable. */
265
+ export function layaVenvReady() {
266
+ return existsSync(venvDoneMarker(layaVenvDir()));
267
+ }
268
+ /** The Python executable inside the venv. */
269
+ export function layaVenvPython() {
270
+ return join(layaVenvDir(), "bin", "python3");
271
+ }
272
+ /** Mark the venv as ready after successful installation. */
273
+ export async function markVenvReady() {
274
+ const marker = venvDoneMarker(layaVenvDir());
275
+ writeFileSync(marker, JSON.stringify({ createdAt: new Date().toISOString() }));
276
+ await chmod(marker, 0o600);
277
+ }
278
+ /** Remove the venv done marker so the next startup rebuilds it. */
279
+ export function invalidateVenv() {
280
+ const marker = venvDoneMarker(layaVenvDir());
281
+ try {
282
+ statSync(marker);
283
+ unlinkSync(marker);
284
+ }
285
+ catch (err) {
286
+ console.warn(`pi-warden: could not invalidate venv marker: ${err instanceof Error ? err.message : err}`);
287
+ }
288
+ }