pi-fluency 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +171 -0
- package/extensions/pi-fluency/analytics.ts +253 -0
- package/extensions/pi-fluency/analyzer.ts +171 -0
- package/extensions/pi-fluency/collector.ts +102 -0
- package/extensions/pi-fluency/context.ts +65 -0
- package/extensions/pi-fluency/diff.ts +147 -0
- package/extensions/pi-fluency/generation-marker.ts +51 -0
- package/extensions/pi-fluency/history-codec.ts +266 -0
- package/extensions/pi-fluency/index.ts +459 -0
- package/extensions/pi-fluency/overlay.ts +637 -0
- package/extensions/pi-fluency/retention.ts +48 -0
- package/extensions/pi-fluency/sanitize.ts +40 -0
- package/extensions/pi-fluency/setup.ts +29 -0
- package/extensions/pi-fluency/state-reducer.ts +192 -0
- package/extensions/pi-fluency/status.ts +39 -0
- package/extensions/pi-fluency/store.ts +589 -0
- package/extensions/pi-fluency/taxonomy.ts +73 -0
- package/extensions/pi-fluency/types.ts +138 -0
- package/extensions/pi-fluency/worker.ts +144 -0
- package/package.json +63 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ihar Trafimovich
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# Pi Fluency
|
|
2
|
+
|
|
3
|
+
Pi Fluency is a Pi extension that turns recurring English mistakes in human-authored prompts into private, keyboard-first coaching. It analyzes sanitized interactive prose after Pi settles, asks you to review each detected occurrence, and tracks accepted mistakes per 1,000 English words across projects and sessions.
|
|
4
|
+
|
|
5
|
+
## Privacy
|
|
6
|
+
|
|
7
|
+
Pi Fluency collects only `input` events with `source === "interactive"`. It excludes RPC/API input, extension-injected messages such as Ralph and subagent control prompts, slash commands, fenced and indented code, inline code, assistant messages, and tool output.
|
|
8
|
+
|
|
9
|
+
Before analysis it:
|
|
10
|
+
|
|
11
|
+
- strips terminal control sequences;
|
|
12
|
+
- removes code;
|
|
13
|
+
- redacts common API keys, tokens, secrets, passwords, private keys, JWTs, cloud credentials, and URL user-info credentials;
|
|
14
|
+
- hashes the sanitized prose for replay protection.
|
|
15
|
+
|
|
16
|
+
Language classification belongs to the selected analyzer model. English results create word-count observations; non-English results must contain no mistakes or demonstrated fixes and contribute nothing to analytics.
|
|
17
|
+
|
|
18
|
+
After explicit consent, Pi Fluency sends the selected provider:
|
|
19
|
+
|
|
20
|
+
- filtered, redacted prose;
|
|
21
|
+
- up to 500 newest eligible pending or accepted rules, represented by internal key, explanation, and ERRANT type;
|
|
22
|
+
- controlled ERRANT error types and context-scope choices for structured output.
|
|
23
|
+
|
|
24
|
+
Full prompts are never written to history. Local history stores prompt hashes, timestamps, local dates, English word counts, occurrence decisions, and bounded sanitized finding excerpts/corrections/explanations. An excerpt may equal an entire short sanitized prompt. Demonstrated-fix evidence is processed in memory but omitted from persisted events. Data goes only to the model provider selected during setup.
|
|
25
|
+
|
|
26
|
+
Global data lives under `~/.pi/agent/pi-fluency/`. The directory is restricted to mode `0700`; history and settings files use `0600`.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
**Requires Pi 0.80.10 or newer.**
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
pi install npm:pi-fluency
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Run `/reload` after installation.
|
|
37
|
+
|
|
38
|
+
For local development:
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
git clone https://github.com/unutranyholas/pi-fluency.git
|
|
42
|
+
cd pi-fluency
|
|
43
|
+
npm install
|
|
44
|
+
pi -e ./extensions/pi-fluency/index.ts
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## First run
|
|
48
|
+
|
|
49
|
+
Run `/fluency`. Select an available low-cost analyzer model, review the provider disclosure, and accept consent. Analysis remains disabled unless setup completes. Provider credentials come from Pi's model registry.
|
|
50
|
+
|
|
51
|
+
## Review model
|
|
52
|
+
|
|
53
|
+
Each Inbox card represents every currently pending occurrence of one concrete rule.
|
|
54
|
+
|
|
55
|
+
- **Accept** confirms only the current pending batch. Accepted occurrences enter analytics. A later recurrence opens the rule in Inbox again.
|
|
56
|
+
- **Dismiss** rejects only the current pending batch. It does not suppress future recurrence and does not enter the mistake-rate numerator.
|
|
57
|
+
- **Ignore** persistently hides an exact rule or ERRANT category. Hidden pending occurrences remain stored and return when restored. Already accepted history remains accepted.
|
|
58
|
+
- **Clear** removes coaching and analytics history while preserving settings, model choice, and consent.
|
|
59
|
+
|
|
60
|
+
Rules use stable namespaced keys and full ERRANT types such as `M:DET`, `U:PUNCT`, and `R:VERB:FORM`. Internal keys and ERRANT codes are not shown in coaching or Stats UI.
|
|
61
|
+
|
|
62
|
+
## Toolbar
|
|
63
|
+
|
|
64
|
+
Normal toolbar example:
|
|
65
|
+
|
|
66
|
+
```text
|
|
67
|
+
12 6 ▆▄▃▂▁▂▂ 8.4/k
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
| Part | Meaning |
|
|
71
|
+
| --- | --- |
|
|
72
|
+
| ` 12` | 12 visible pending occurrences; `` means zero |
|
|
73
|
+
| ` 6` | 6 recurring rule-explanation groups with accepted occurrences in the trailing seven days |
|
|
74
|
+
| `▆▄▃▂▁▂▂` | Seven rolling seven-day accepted-mistake rates |
|
|
75
|
+
| `8.4/k` | Latest accepted mistakes per 1,000 English words |
|
|
76
|
+
|
|
77
|
+
Counts are real and unclamped. A missing denominator renders `—/k`; missing sparkline points render `·`. Startup shows a loading-shaped toolbar. Stable errors are bounded to:
|
|
78
|
+
|
|
79
|
+
```text
|
|
80
|
+
ERR auth
|
|
81
|
+
ERR model
|
|
82
|
+
ERR analyze
|
|
83
|
+
ERR store
|
|
84
|
+
ERR migrate
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Full sanitized error detail appears in notifications. `/fluency status` reports state, model, queue, drops, and storage warning count. Stock Pi receives the complete toolbar text. Powerbar receives the leading Nerd Font icon separately to avoid duplication. With Powerbar installed, add **Pi Fluency** through `/extension-settings`.
|
|
88
|
+
|
|
89
|
+
## Commands and keyboard controls
|
|
90
|
+
|
|
91
|
+
`Ctrl+Shift+L` opens Inbox when configured.
|
|
92
|
+
|
|
93
|
+
| Command | Action |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| `/fluency` | Run setup when unconfigured; otherwise open Inbox |
|
|
96
|
+
| `/fluency stats` | Open local Stats directly, including while paused or model-offline |
|
|
97
|
+
| `/fluency pause` | Stop analysis and hide status |
|
|
98
|
+
| `/fluency resume` | Resume a valid consented configuration |
|
|
99
|
+
| `/fluency status` | Show state, model, queue, drops, and storage warnings |
|
|
100
|
+
| `/fluency model` | Select another analyzer model and review provider disclosure |
|
|
101
|
+
| `/fluency clear` | Confirm, then remove coaching and analytics history |
|
|
102
|
+
|
|
103
|
+
Inside the overlay:
|
|
104
|
+
|
|
105
|
+
- Left/Right changes cards.
|
|
106
|
+
- Up/Down, `j`/`k`, and Page Up/Down scroll.
|
|
107
|
+
- `a` accepts the current Inbox batch; `l` remains a compatibility alias.
|
|
108
|
+
- `d` dismisses the current Inbox batch.
|
|
109
|
+
- `i` ignores an exact rule or ERRANT category.
|
|
110
|
+
- `u` restores every ignore affecting an item in Ignored.
|
|
111
|
+
- Tab cycles Inbox, Accepted, Ignored, and Stats.
|
|
112
|
+
- Esc closes.
|
|
113
|
+
|
|
114
|
+
Actions auto-advance to the next card. Stats is read-only. Compact diffs use `└─` for replacements, strikethrough for deletions, and underline for insertions.
|
|
115
|
+
|
|
116
|
+
## Analytics and Stats
|
|
117
|
+
|
|
118
|
+
For a period `P`:
|
|
119
|
+
|
|
120
|
+
```text
|
|
121
|
+
accepted mistake rate(P) = accepted occurrences in P / English words in P × 1000
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Every analyzer-classified English prompt contributes words, including prompts with zero findings. Pending, dismissed, hidden-unaccepted, and non-English occurrences do not enter the numerator.
|
|
125
|
+
|
|
126
|
+
Stats covers 30 local calendar days and shows:
|
|
127
|
+
|
|
128
|
+
- 30-day accepted rate, English words, accepted, dismissed, and visible pending totals;
|
|
129
|
+
- a single aggregate count for one-off accepted mistakes in the period;
|
|
130
|
+
- review coverage: `(accepted + dismissed) / (accepted + dismissed + visible pending)`;
|
|
131
|
+
- active trailing-seven-day recurring rules;
|
|
132
|
+
- rolling seven-day toolbar trend;
|
|
133
|
+
- recurring-rule rates, sparklines, and `improving`, `worsening`, `stable`, or `new` trends.
|
|
134
|
+
|
|
135
|
+
A rule becomes recurring after at least two accepted occurrences across retained history. One-offs remain part of accepted totals and rates, but do not clutter the toolbar count, Concrete rules list, or trend totals. Rule trends compare adjacent 30-day windows. Improving/worsening requires both at least 20% relative change and at least 0.5 mistakes per 1,000 words absolute change. Rules are grouped and displayed by human explanation, never by internal key or broad category.
|
|
136
|
+
|
|
137
|
+
## Storage and migration
|
|
138
|
+
|
|
139
|
+
`settings.json`, `history.jsonl`, and the private clear-generation marker are shared globally across Pi projects and sessions. Analyzer responses and current settings use schema v3. History uses strict schema v4 events with deterministic occurrence IDs. Compaction:
|
|
140
|
+
|
|
141
|
+
- retains every pending occurrence regardless of age;
|
|
142
|
+
- retains 365 local calendar days of reviewed observations and occurrences;
|
|
143
|
+
- preserves hashes referenced by retained observations;
|
|
144
|
+
- writes state-free schema-v4 snapshots under an atomic cross-process lock.
|
|
145
|
+
|
|
146
|
+
History schema v4 is a clean break. Non-empty v1, v2, or v3 history is not interpreted, normalized, or rewritten. Pi Fluency reports `ERR migrate` and blocks history mutations until confirmed clear. There is no automatic migration or backup. Current schema-v3 settings, provider choice, and consent remain intact.
|
|
147
|
+
|
|
148
|
+
### Safe history reset
|
|
149
|
+
|
|
150
|
+
Direct `history.jsonl` edits are unsupported, especially while the extension is running. To recover from old, polluted, or unwanted history:
|
|
151
|
+
|
|
152
|
+
1. Run `/reload` after installing the updated extension.
|
|
153
|
+
2. Run `/fluency clear`.
|
|
154
|
+
3. Confirm the prompt.
|
|
155
|
+
4. Verify `/fluency stats` is empty and settings/model/consent remain configured.
|
|
156
|
+
|
|
157
|
+
Pi Fluency adopts the English error taxonomy from the MIT-licensed [ERRANT toolkit](https://github.com/chrisjbryant/errant) and its [ACL 2017 paper](https://aclanthology.org/P17-1074/). It does not bundle ERRANT, Python, spaCy, language models, or learner corpora.
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```sh
|
|
162
|
+
npm install
|
|
163
|
+
npm run check
|
|
164
|
+
pi -e ./extensions/pi-fluency/index.ts
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Automated tests always use temporary storage roots and never touch `~/.pi/agent/pi-fluency/`.
|
|
168
|
+
|
|
169
|
+
## Known limitations
|
|
170
|
+
|
|
171
|
+
The overlay is available only in interactive TUI mode. Pi has no native low-priority scheduler, so Pi Fluency queues bounded work and starts analysis only after the main agent settles and Pi reports idle. Redaction is defense in depth, not a substitute for avoiding secrets in prompts.
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { errantCategory } from "./taxonomy.js";
|
|
2
|
+
import type {
|
|
3
|
+
EnglishObservation,
|
|
4
|
+
MistakeOccurrence,
|
|
5
|
+
MistakePattern,
|
|
6
|
+
} from "./types.js";
|
|
7
|
+
import type { ErrantCategory } from "./taxonomy.js";
|
|
8
|
+
|
|
9
|
+
const ENGLISH_WORD = /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)*/gu;
|
|
10
|
+
const SPARK_GLYPHS = "▁▂▃▄▅▆▇█";
|
|
11
|
+
const DATE_KEY = /^\d{4}-\d{2}-\d{2}$/;
|
|
12
|
+
const TREND_DAYS = 30;
|
|
13
|
+
|
|
14
|
+
/** Count word-like runs after the analyzer has classified sanitized prose as English. */
|
|
15
|
+
export function countEnglishWords(prose: string): number {
|
|
16
|
+
return [...prose.matchAll(ENGLISH_WORD)].length;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export type RuleTrend = "improving" | "worsening" | "stable" | "new";
|
|
20
|
+
|
|
21
|
+
export interface RuleAnalytics {
|
|
22
|
+
patternId: string;
|
|
23
|
+
explanation: string;
|
|
24
|
+
accepted: number;
|
|
25
|
+
ratePerThousand: number | undefined;
|
|
26
|
+
sparkline: string;
|
|
27
|
+
trend: RuleTrend;
|
|
28
|
+
changePercent?: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface FluencyAnalytics {
|
|
32
|
+
pendingOccurrences: number;
|
|
33
|
+
periodPendingOccurrences: number;
|
|
34
|
+
activeRules: number;
|
|
35
|
+
currentRatePerThousand?: number;
|
|
36
|
+
periodRatePerThousand?: number;
|
|
37
|
+
toolbarSparkline: string;
|
|
38
|
+
englishWords: number;
|
|
39
|
+
accepted: number;
|
|
40
|
+
dismissed: number;
|
|
41
|
+
oneOffAccepted: number;
|
|
42
|
+
reviewCoverage?: number;
|
|
43
|
+
rules: RuleAnalytics[];
|
|
44
|
+
trendCounts: Record<RuleTrend, number>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface AnalyticsInput {
|
|
48
|
+
observations: Iterable<EnglishObservation>;
|
|
49
|
+
occurrences: Iterable<MistakeOccurrence>;
|
|
50
|
+
patterns: Iterable<MistakePattern>;
|
|
51
|
+
ignoredPatternKeys: ReadonlySet<string>;
|
|
52
|
+
ignoredCategories: ReadonlySet<ErrantCategory>;
|
|
53
|
+
now: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function ratePerThousand(accepted: number, words: number): number | undefined {
|
|
57
|
+
return words > 0 ? accepted * 1_000 / words : undefined;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function renderSparkline(values: Array<number | undefined>): string {
|
|
61
|
+
const finite = values.filter((value): value is number =>
|
|
62
|
+
typeof value === "number" && Number.isFinite(value));
|
|
63
|
+
if (finite.length === 0) return values.map(() => "·").join("");
|
|
64
|
+
const minimum = Math.min(...finite);
|
|
65
|
+
const maximum = Math.max(...finite);
|
|
66
|
+
return values.map((value) => {
|
|
67
|
+
if (value === undefined || !Number.isFinite(value)) return "·";
|
|
68
|
+
if (minimum === maximum) return maximum === 0 ? SPARK_GLYPHS[0]! : SPARK_GLYPHS[3]!;
|
|
69
|
+
const index = Math.round((value - minimum) / (maximum - minimum) * (SPARK_GLYPHS.length - 1));
|
|
70
|
+
return SPARK_GLYPHS[Math.max(0, Math.min(SPARK_GLYPHS.length - 1, index))]!;
|
|
71
|
+
}).join("");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function classifyRuleTrend(
|
|
75
|
+
current: number | undefined,
|
|
76
|
+
previous: number | undefined,
|
|
77
|
+
): RuleTrend {
|
|
78
|
+
if (current === undefined || !Number.isFinite(current) || current <= 0) {
|
|
79
|
+
if (previous === undefined || !Number.isFinite(previous) || previous <= 0) return "stable";
|
|
80
|
+
}
|
|
81
|
+
if (current !== undefined && Number.isFinite(current) && current > 0
|
|
82
|
+
&& (previous === undefined || !Number.isFinite(previous) || previous <= 0)) return "new";
|
|
83
|
+
if (current === undefined || previous === undefined || !Number.isFinite(current) || !Number.isFinite(previous)) {
|
|
84
|
+
return "stable";
|
|
85
|
+
}
|
|
86
|
+
const absolute = Math.abs(current - previous);
|
|
87
|
+
const relative = previous > 0 ? absolute / previous : 0;
|
|
88
|
+
if (absolute < 0.5 || relative < 0.2) return "stable";
|
|
89
|
+
return current < previous ? "improving" : "worsening";
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function localDateKey(now: number): string {
|
|
93
|
+
const date = new Date(now);
|
|
94
|
+
const year = date.getFullYear();
|
|
95
|
+
const month = String(date.getMonth() + 1).padStart(2, "0");
|
|
96
|
+
const day = String(date.getDate()).padStart(2, "0");
|
|
97
|
+
return `${year}-${month}-${day}`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function shiftDate(localDate: string, days: number): string {
|
|
101
|
+
const [year, month, day] = localDate.split("-").map(Number);
|
|
102
|
+
return new Date(Date.UTC(year!, month! - 1, day! + days)).toISOString().slice(0, 10);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function within(localDate: string, start: string, end: string): boolean {
|
|
106
|
+
return DATE_KEY.test(localDate) && localDate >= start && localDate <= end;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
interface WindowTotals {
|
|
110
|
+
words: number;
|
|
111
|
+
accepted: number;
|
|
112
|
+
dismissed: number;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
interface RuleGroup {
|
|
116
|
+
patternId: string;
|
|
117
|
+
explanation: string;
|
|
118
|
+
patternIds: Set<string>;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function computeFluencyAnalytics(input: AnalyticsInput): FluencyAnalytics {
|
|
122
|
+
const observations = [...input.observations];
|
|
123
|
+
const occurrences = [...input.occurrences];
|
|
124
|
+
const patterns = [...input.patterns];
|
|
125
|
+
const today = localDateKey(input.now);
|
|
126
|
+
const patternsById = new Map(patterns.map((pattern) => [pattern.id, pattern]));
|
|
127
|
+
|
|
128
|
+
const isIgnored = (occurrence: MistakeOccurrence): boolean => {
|
|
129
|
+
const pattern = patternsById.get(occurrence.patternId);
|
|
130
|
+
if (input.ignoredPatternKeys.has(pattern?.patternKey ?? occurrence.patternKey)) return true;
|
|
131
|
+
return pattern !== undefined && input.ignoredCategories.has(errantCategory(pattern.errorType));
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
const totals = (end: string, days: number): WindowTotals => {
|
|
135
|
+
const start = shiftDate(end, -(days - 1));
|
|
136
|
+
return {
|
|
137
|
+
words: observations
|
|
138
|
+
.filter((observation) => within(observation.localDate, start, end))
|
|
139
|
+
.reduce((sum, observation) => sum + Math.max(0, observation.wordCount), 0),
|
|
140
|
+
accepted: occurrences.filter((occurrence) =>
|
|
141
|
+
occurrence.decision === "accepted" && within(occurrence.localDate, start, end)).length,
|
|
142
|
+
dismissed: occurrences.filter((occurrence) =>
|
|
143
|
+
occurrence.decision === "dismissed" && within(occurrence.localDate, start, end)).length,
|
|
144
|
+
};
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const trailingRates = Array.from({ length: 7 }, (_, index) => {
|
|
148
|
+
const end = shiftDate(today, index - 6);
|
|
149
|
+
const window = totals(end, 7);
|
|
150
|
+
return ratePerThousand(window.accepted, window.words);
|
|
151
|
+
});
|
|
152
|
+
const currentSeven = totals(today, 7);
|
|
153
|
+
const currentThirty = totals(today, TREND_DAYS);
|
|
154
|
+
const previousEnd = shiftDate(today, -TREND_DAYS);
|
|
155
|
+
const previousThirty = totals(previousEnd, TREND_DAYS);
|
|
156
|
+
const currentStart = shiftDate(today, -(TREND_DAYS - 1));
|
|
157
|
+
|
|
158
|
+
const visiblePending = occurrences.filter((occurrence) =>
|
|
159
|
+
occurrence.decision === "pending" && !isIgnored(occurrence));
|
|
160
|
+
const visiblePendingInPeriod = visiblePending.filter((occurrence) =>
|
|
161
|
+
within(occurrence.localDate, currentStart, today)).length;
|
|
162
|
+
const reviewed = currentThirty.accepted + currentThirty.dismissed;
|
|
163
|
+
const reviewable = reviewed + visiblePendingInPeriod;
|
|
164
|
+
|
|
165
|
+
const groupsByExplanation = new Map<string, RuleGroup>();
|
|
166
|
+
for (const pattern of patterns) {
|
|
167
|
+
const explanation = pattern.explanation;
|
|
168
|
+
const existing = groupsByExplanation.get(explanation);
|
|
169
|
+
if (existing) {
|
|
170
|
+
existing.patternIds.add(pattern.id);
|
|
171
|
+
if (pattern.id.localeCompare(existing.patternId) < 0) existing.patternId = pattern.id;
|
|
172
|
+
} else {
|
|
173
|
+
groupsByExplanation.set(explanation, {
|
|
174
|
+
patternId: pattern.id,
|
|
175
|
+
explanation,
|
|
176
|
+
patternIds: new Set([pattern.id]),
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const acceptedForGroup = (group: RuleGroup, end: string, days: number): number => {
|
|
182
|
+
const start = shiftDate(end, -(days - 1));
|
|
183
|
+
return occurrences.filter((occurrence) =>
|
|
184
|
+
occurrence.decision === "accepted"
|
|
185
|
+
&& group.patternIds.has(occurrence.patternId)
|
|
186
|
+
&& within(occurrence.localDate, start, end)).length;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
const lifetimeAcceptedForGroup = (group: RuleGroup): number => occurrences.filter((occurrence) =>
|
|
190
|
+
occurrence.decision === "accepted" && group.patternIds.has(occurrence.patternId)).length;
|
|
191
|
+
const recurringGroups = [...groupsByExplanation.values()].filter((group) =>
|
|
192
|
+
lifetimeAcceptedForGroup(group) >= 2);
|
|
193
|
+
const oneOffAccepted = [...groupsByExplanation.values()]
|
|
194
|
+
.filter((group) => lifetimeAcceptedForGroup(group) === 1)
|
|
195
|
+
.reduce((sum, group) => sum + acceptedForGroup(group, today, TREND_DAYS), 0);
|
|
196
|
+
|
|
197
|
+
const rules = recurringGroups.flatMap((group): RuleAnalytics[] => {
|
|
198
|
+
const accepted = acceptedForGroup(group, today, TREND_DAYS);
|
|
199
|
+
const previousAccepted = acceptedForGroup(group, previousEnd, TREND_DAYS);
|
|
200
|
+
if (accepted === 0 && previousAccepted === 0) return [];
|
|
201
|
+
const currentRate = ratePerThousand(accepted, currentThirty.words);
|
|
202
|
+
const previousRate = ratePerThousand(previousAccepted, previousThirty.words);
|
|
203
|
+
const trend = previousAccepted === 0 && accepted > 0
|
|
204
|
+
? "new"
|
|
205
|
+
: classifyRuleTrend(currentRate, previousRate);
|
|
206
|
+
const sparklineValues = Array.from({ length: 7 }, (_, index) => {
|
|
207
|
+
const end = shiftDate(today, index - 6);
|
|
208
|
+
const window = totals(end, 7);
|
|
209
|
+
return ratePerThousand(acceptedForGroup(group, end, 7), window.words);
|
|
210
|
+
});
|
|
211
|
+
const changePercent = trend !== "new" && trend !== "stable"
|
|
212
|
+
&& previousRate !== undefined && previousRate > 0 && currentRate !== undefined
|
|
213
|
+
? (currentRate - previousRate) / previousRate * 100
|
|
214
|
+
: undefined;
|
|
215
|
+
return [{
|
|
216
|
+
patternId: group.patternId,
|
|
217
|
+
explanation: group.explanation,
|
|
218
|
+
accepted,
|
|
219
|
+
ratePerThousand: currentRate,
|
|
220
|
+
sparkline: renderSparkline(sparklineValues),
|
|
221
|
+
trend,
|
|
222
|
+
...(changePercent === undefined ? {} : { changePercent }),
|
|
223
|
+
}];
|
|
224
|
+
}).sort((left, right) =>
|
|
225
|
+
right.accepted - left.accepted
|
|
226
|
+
|| (right.ratePerThousand ?? -Infinity) - (left.ratePerThousand ?? -Infinity)
|
|
227
|
+
|| left.explanation.localeCompare(right.explanation));
|
|
228
|
+
|
|
229
|
+
const trendCounts: Record<RuleTrend, number> = { improving: 0, worsening: 0, stable: 0, new: 0 };
|
|
230
|
+
for (const rule of rules) trendCounts[rule.trend] += 1;
|
|
231
|
+
|
|
232
|
+
const activeRules = recurringGroups.filter((group) =>
|
|
233
|
+
acceptedForGroup(group, today, 7) > 0).length;
|
|
234
|
+
|
|
235
|
+
const currentRatePerThousand = ratePerThousand(currentSeven.accepted, currentSeven.words);
|
|
236
|
+
const periodRatePerThousand = ratePerThousand(currentThirty.accepted, currentThirty.words);
|
|
237
|
+
const reviewCoverage = reviewable > 0 ? reviewed / reviewable : undefined;
|
|
238
|
+
return {
|
|
239
|
+
pendingOccurrences: visiblePending.length,
|
|
240
|
+
periodPendingOccurrences: visiblePendingInPeriod,
|
|
241
|
+
activeRules,
|
|
242
|
+
...(currentRatePerThousand === undefined ? {} : { currentRatePerThousand }),
|
|
243
|
+
...(periodRatePerThousand === undefined ? {} : { periodRatePerThousand }),
|
|
244
|
+
toolbarSparkline: renderSparkline(trailingRates),
|
|
245
|
+
englishWords: currentThirty.words,
|
|
246
|
+
accepted: currentThirty.accepted,
|
|
247
|
+
dismissed: currentThirty.dismissed,
|
|
248
|
+
oneOffAccepted,
|
|
249
|
+
...(reviewCoverage === undefined ? {} : { reviewCoverage }),
|
|
250
|
+
rules,
|
|
251
|
+
trendCounts,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import type { Api, Model, UserMessage } from "@earendil-works/pi-ai";
|
|
2
|
+
import { complete } from "@earendil-works/pi-ai/compat";
|
|
3
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
4
|
+
import {
|
|
5
|
+
ANALYSIS_SCHEMA_VERSION,
|
|
6
|
+
type AnalysisResult,
|
|
7
|
+
type CollectedPrompt,
|
|
8
|
+
type MistakePattern,
|
|
9
|
+
type RawAnalysisResult,
|
|
10
|
+
} from "./types.js";
|
|
11
|
+
import {
|
|
12
|
+
ERRANT_ERROR_TYPES,
|
|
13
|
+
isErrantErrorType,
|
|
14
|
+
} from "./taxonomy.js";
|
|
15
|
+
import { materializeMistake } from "./context.js";
|
|
16
|
+
import { sanitizeAnalyzerField } from "./sanitize.js";
|
|
17
|
+
|
|
18
|
+
const CONTEXT_SCOPES = ["sentence", "previous-and-current", "current-and-next"] as const;
|
|
19
|
+
const MAX_KNOWN_PATTERNS = 500;
|
|
20
|
+
const MAX_MISTAKES = 20;
|
|
21
|
+
const MAX_DEMONSTRATED_FIXES = 25;
|
|
22
|
+
|
|
23
|
+
export class AnalyzerConfigurationError extends Error {
|
|
24
|
+
override readonly name = "AnalyzerConfigurationError";
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface Analyzer {
|
|
28
|
+
analyze(
|
|
29
|
+
prompt: CollectedPrompt,
|
|
30
|
+
activePatterns: MistakePattern[],
|
|
31
|
+
signal: AbortSignal,
|
|
32
|
+
): Promise<AnalysisResult>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function buildAnalysisPrompt(prompt: CollectedPrompt, activePatterns: MistakePattern[]): string {
|
|
36
|
+
const knownPatterns = activePatterns
|
|
37
|
+
.slice(0, MAX_KNOWN_PATTERNS)
|
|
38
|
+
.map(({ patternKey, explanation, errorType }) => ({ patternKey, explanation, errorType }));
|
|
39
|
+
return JSON.stringify({
|
|
40
|
+
prose: prompt.prose,
|
|
41
|
+
knownPatterns,
|
|
42
|
+
allowedErrorTypes: ERRANT_ERROR_TYPES,
|
|
43
|
+
allowedContextScopes: CONTEXT_SCOPES,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
48
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function validateAnalysisResult(value: unknown, minimumConfidence: number): RawAnalysisResult {
|
|
52
|
+
if (
|
|
53
|
+
!isRecord(value)
|
|
54
|
+
|| value.schemaVersion !== ANALYSIS_SCHEMA_VERSION
|
|
55
|
+
|| (value.language !== "en" && value.language !== "other")
|
|
56
|
+
|| !Array.isArray(value.mistakes)
|
|
57
|
+
|| !Array.isArray(value.demonstratedFixes)
|
|
58
|
+
) throw new Error("Invalid analysis result");
|
|
59
|
+
|
|
60
|
+
if (value.language === "other" && (value.mistakes.length > 0 || value.demonstratedFixes.length > 0)) {
|
|
61
|
+
throw new Error("Invalid analysis result");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const scopes = new Set<string>(CONTEXT_SCOPES);
|
|
65
|
+
const validConfidence = (confidence: unknown): confidence is number =>
|
|
66
|
+
typeof confidence === "number" && Number.isFinite(confidence) && confidence >= 0 && confidence <= 1;
|
|
67
|
+
|
|
68
|
+
const mistakes: RawAnalysisResult["mistakes"] = [];
|
|
69
|
+
for (const candidate of value.mistakes.slice(0, MAX_MISTAKES)) {
|
|
70
|
+
if (!isRecord(candidate)) throw new Error("Invalid analysis result");
|
|
71
|
+
const original = sanitizeAnalyzerField(candidate.original);
|
|
72
|
+
const correction = sanitizeAnalyzerField(candidate.correction, true);
|
|
73
|
+
const explanation = sanitizeAnalyzerField(candidate.explanation);
|
|
74
|
+
const patternKey = sanitizeAnalyzerField(candidate.patternKey);
|
|
75
|
+
const contextScope = candidate.contextScope;
|
|
76
|
+
if (
|
|
77
|
+
original === undefined || correction === undefined || explanation === undefined || patternKey === undefined
|
|
78
|
+
|| !/^[a-z]+(?:[.-][a-z0-9]+)+$/.test(patternKey)
|
|
79
|
+
|| typeof contextScope !== "string" || !scopes.has(contextScope)
|
|
80
|
+
|| !validConfidence(candidate.confidence)
|
|
81
|
+
) throw new Error("Invalid analysis result");
|
|
82
|
+
if (candidate.confidence < minimumConfidence) continue;
|
|
83
|
+
const errorType = isErrantErrorType(candidate.errorType) ? candidate.errorType : "R:OTHER";
|
|
84
|
+
mistakes.push({ original, correction, explanation, patternKey, contextScope: contextScope as RawAnalysisResult["mistakes"][number]["contextScope"], errorType, confidence: candidate.confidence });
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const demonstratedFixes: RawAnalysisResult["demonstratedFixes"] = [];
|
|
88
|
+
for (const candidate of value.demonstratedFixes.slice(0, MAX_DEMONSTRATED_FIXES)) {
|
|
89
|
+
if (!isRecord(candidate)) throw new Error("Invalid analysis result");
|
|
90
|
+
const patternKey = sanitizeAnalyzerField(candidate.patternKey);
|
|
91
|
+
const evidence = sanitizeAnalyzerField(candidate.evidence);
|
|
92
|
+
if (patternKey === undefined || evidence === undefined || !validConfidence(candidate.confidence)) {
|
|
93
|
+
throw new Error("Invalid analysis result");
|
|
94
|
+
}
|
|
95
|
+
if (candidate.confidence >= minimumConfidence) demonstratedFixes.push({ patternKey, evidence, confidence: candidate.confidence });
|
|
96
|
+
}
|
|
97
|
+
return { schemaVersion: ANALYSIS_SCHEMA_VERSION, language: value.language, mistakes, demonstratedFixes };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const SYSTEM_PROMPT = `You are Pi Fluency's English learning analyzer.
|
|
101
|
+
Return only one JSON object with exactly this shape:
|
|
102
|
+
{"schemaVersion":3,"language":"en","mistakes":[{"original":"exact erroneous quote","correction":"corrected quote","contextScope":"sentence","explanation":"brief rule","errorType":"R:DET","patternKey":"grammar.articles.example-rule","confidence":0.95}],"demonstratedFixes":[{"patternKey":"grammar.articles.example-rule","evidence":"exact correct quote","confidence":0.95}]}
|
|
103
|
+
Classify the supplied prose as English or other. For language "other", return empty mistakes and demonstratedFixes. Do not correct or translate non-English prose.
|
|
104
|
+
Use empty arrays when no mistakes or fixes exist. Do not rename fields or add fields.
|
|
105
|
+
For each mistake, original must be the shortest unique exact quote containing the error; choose contextScope and errorType only from supplied controlled values.
|
|
106
|
+
Report genuine grammar, spelling, word-choice, or idiomatic-usage errors only.
|
|
107
|
+
Do not report style preferences, capitalization of product names, or informal-but-valid wording.
|
|
108
|
+
A demonstrated fix must match one supplied knownPatterns pattern and quote comparable correct evidence.
|
|
109
|
+
Never infer a demonstrated fix merely because an earlier mistake is absent.
|
|
110
|
+
Reuse an existing knownPatterns patternKey for the same rule; mint a new namespaced lowercase key only when no known rule matches.
|
|
111
|
+
Keep every explanation under 240 characters. Return JSON only.`;
|
|
112
|
+
|
|
113
|
+
export class ModelAnalyzer implements Analyzer {
|
|
114
|
+
constructor(private readonly options: {
|
|
115
|
+
model: Model<Api>;
|
|
116
|
+
registry: ModelRegistry;
|
|
117
|
+
minimumConfidence: number;
|
|
118
|
+
}) {}
|
|
119
|
+
|
|
120
|
+
async analyze(
|
|
121
|
+
prompt: CollectedPrompt,
|
|
122
|
+
activePatterns: MistakePattern[],
|
|
123
|
+
signal: AbortSignal,
|
|
124
|
+
): Promise<AnalysisResult> {
|
|
125
|
+
let auth: Awaited<ReturnType<ModelRegistry["getApiKeyAndHeaders"]>>;
|
|
126
|
+
try {
|
|
127
|
+
auth = await this.options.registry.getApiKeyAndHeaders(this.options.model);
|
|
128
|
+
} catch {
|
|
129
|
+
throw new AnalyzerConfigurationError("Unable to resolve model authentication");
|
|
130
|
+
}
|
|
131
|
+
if (!auth.ok || !auth.apiKey) {
|
|
132
|
+
throw new AnalyzerConfigurationError("Model authentication unavailable");
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const message: UserMessage = {
|
|
136
|
+
role: "user",
|
|
137
|
+
content: buildAnalysisPrompt(prompt, activePatterns),
|
|
138
|
+
timestamp: Date.now(),
|
|
139
|
+
};
|
|
140
|
+
const response = await complete(
|
|
141
|
+
this.options.model,
|
|
142
|
+
{ systemPrompt: SYSTEM_PROMPT, messages: [message] },
|
|
143
|
+
{
|
|
144
|
+
apiKey: auth.apiKey,
|
|
145
|
+
...(auth.headers === undefined ? {} : { headers: auth.headers }),
|
|
146
|
+
...(auth.env === undefined ? {} : { env: auth.env }),
|
|
147
|
+
signal,
|
|
148
|
+
},
|
|
149
|
+
);
|
|
150
|
+
const text = response.content
|
|
151
|
+
.filter((block): block is { type: "text"; text: string } => block.type === "text")
|
|
152
|
+
.map((block) => block.text)
|
|
153
|
+
.join("\n")
|
|
154
|
+
.trim();
|
|
155
|
+
let parsed: unknown;
|
|
156
|
+
try {
|
|
157
|
+
parsed = JSON.parse(text) as unknown;
|
|
158
|
+
} catch {
|
|
159
|
+
throw new Error("Invalid analysis result");
|
|
160
|
+
}
|
|
161
|
+
const validated = validateAnalysisResult(parsed, this.options.minimumConfidence);
|
|
162
|
+
const knownPatternKeys = new Set(activePatterns.map((pattern) => pattern.patternKey));
|
|
163
|
+
return {
|
|
164
|
+
schemaVersion: ANALYSIS_SCHEMA_VERSION,
|
|
165
|
+
language: validated.language,
|
|
166
|
+
mistakes: validated.mistakes.map((mistake) => materializeMistake(prompt.prose, mistake)),
|
|
167
|
+
demonstratedFixes: validated.demonstratedFixes.filter((fix) =>
|
|
168
|
+
knownPatternKeys.has(fix.patternKey) && prompt.prose.includes(fix.evidence)),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|