shapewall 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/CHANGELOG.md +267 -0
  2. package/README.md +39 -2
  3. package/THIRD_PARTY.md +15 -0
  4. package/dist/agent/package-lock.json +594 -2
  5. package/dist/client/assets/{BoardChat-DdwBcb2N.js → BoardChat-Bv4UDOjO.js} +1 -1
  6. package/dist/client/assets/{BuzzDialog-DpUcnU6S.js → BuzzDialog-DwEAN6va.js} +1 -1
  7. package/dist/client/assets/{ChecksPanel-CngYV4yz.js → ChecksPanel-C6r_Tc9g.js} +1 -1
  8. package/dist/client/assets/{History-CD0saQA5.js → History-ajFROfWC.js} +1 -1
  9. package/dist/client/assets/{LocalSettings-MZy2baig.js → LocalSettings-BSbmmpbN.js} +1 -1
  10. package/dist/client/assets/{Management-CSvZ-SHO.js → Management-mo8uv2qG.js} +1 -1
  11. package/dist/client/assets/{QuickConnect-foLh2QUm.js → QuickConnect-DlaijESK.js} +1 -1
  12. package/dist/client/assets/{ShareDialog-DPWDftoL.js → ShareDialog-CdLdd6WQ.js} +1 -1
  13. package/dist/client/assets/{app-update-dialog-CtiXpTRD.js → app-update-dialog-BHVNR5Cp.js} +1 -1
  14. package/dist/client/assets/{circle-x-CteRBtXC.js → circle-x-D5MkDLJk.js} +1 -1
  15. package/dist/client/assets/index-B4AoX3ZW.css +2 -0
  16. package/dist/client/assets/index-CQeHY7en.js +37 -0
  17. package/dist/client/assets/{play-BjRNIGSf.js → play--xrZ7VCR.js} +1 -1
  18. package/dist/client/assets/{refresh-cw-r_tZiFfB.js → refresh-cw-B-npxKae.js} +1 -1
  19. package/dist/client/assets/{ui-zQpn-PdL.js → ui-2lMx-4bc.js} +2 -2
  20. package/dist/client/build-info.json +1 -1
  21. package/dist/client/index.html +3 -3
  22. package/dist/client/sw.js +2 -2
  23. package/dist/console/assets/LocalAccountsApp-C1sCag6a.js +1 -0
  24. package/dist/console/assets/LocalConsole-LXQmzRFT.js +2 -0
  25. package/dist/console/assets/LocalUsage-Br3Ddp7_.js +1 -0
  26. package/dist/console/assets/app-update-dialog-CVniDJKA.js +1 -0
  27. package/dist/console/assets/app-version-CTHwlY6g.js +1 -0
  28. package/dist/console/assets/console-CMNNLtFw.js +11 -0
  29. package/dist/console/assets/console-sMMn_Rwc.css +2 -0
  30. package/dist/console/assets/local-agents-DwYJ9cPb.js +6 -0
  31. package/dist/console/assets/minus-887msXez.js +1 -0
  32. package/dist/console/assets/tailnet--wXsG3uh.js +2 -0
  33. package/dist/console/assets/theme-CPfJOmcz.js +1 -0
  34. package/dist/console/assets/ui-DnyJhoQU.js +41 -0
  35. package/dist/console/assets/usage-C5CJHejQ.js +5 -0
  36. package/dist/console/assets/usage-CIPWnHaS.css +1 -0
  37. package/dist/console/assets/usage-DjmUlnXL.js +56 -0
  38. package/dist/console/build-info.json +1 -1
  39. package/dist/console/console.html +8 -2
  40. package/docs/agent-hardening.md +31 -17
  41. package/docs/agents.md +35 -0
  42. package/docs/architecture.md +10 -4
  43. package/docs/deployment.md +39 -17
  44. package/docs/design-system.md +56 -7
  45. package/docs/hosted.md +1004 -43
  46. package/docs/images/board-overview.png +0 -0
  47. package/docs/images/journeys.png +0 -0
  48. package/docs/images/live-comment.png +0 -0
  49. package/docs/images/phone-comment.png +0 -0
  50. package/docs/images/phone-overview.png +0 -0
  51. package/docs/images/variant.png +0 -0
  52. package/docs/where-to-run.md +82 -1
  53. package/package.json +6 -1
  54. package/src/agent/backends/acp.js +224 -26
  55. package/src/agent/routes.js +21 -0
  56. package/src/agent/usage-log.js +251 -0
  57. package/src/cli/args.js +24 -0
  58. package/src/cli/help.js +37 -2
  59. package/src/cli/init.js +2 -1
  60. package/src/cli/serve.js +20 -5
  61. package/src/cli.js +8 -0
  62. package/src/hosted/access-cli.js +99 -1
  63. package/src/hosted/activity-core.js +232 -0
  64. package/src/hosted/activity.js +375 -0
  65. package/src/hosted/admin.js +80 -13
  66. package/src/hosted/agent-approvals.js +2 -2
  67. package/src/hosted/agent-routes.js +3 -3
  68. package/src/hosted/agents.js +11 -4
  69. package/src/hosted/app.js +89 -7
  70. package/src/hosted/auth-routes.js +156 -15
  71. package/src/hosted/auth.js +123 -20
  72. package/src/hosted/board-routes.js +3 -3
  73. package/src/hosted/chat-frames.js +5 -15
  74. package/src/hosted/chat-permissions.js +12 -0
  75. package/src/hosted/chat-routes.js +8 -8
  76. package/src/hosted/chat-store.js +20 -3
  77. package/src/hosted/comment-files.js +5 -2
  78. package/src/hosted/console-routes.js +111 -34
  79. package/src/hosted/db.js +130 -0
  80. package/src/hosted/device-routes.js +25 -3
  81. package/src/hosted/device.js +23 -4
  82. package/src/hosted/guards.js +337 -27
  83. package/src/hosted/http.js +6 -3
  84. package/src/hosted/membership.js +13 -2
  85. package/src/hosted/operator-access.js +418 -0
  86. package/src/hosted/operator.js +106 -2
  87. package/src/hosted/ops-cli.js +238 -3
  88. package/src/hosted/pages.js +2 -0
  89. package/src/hosted/passkey-policy.js +75 -0
  90. package/src/hosted/permissions.js +14 -6
  91. package/src/hosted/presence.js +14 -2
  92. package/src/hosted/projects.js +2 -1
  93. package/src/hosted/public.js +4 -4
  94. package/src/hosted/ratelimit.js +6 -0
  95. package/src/hosted/retention.js +18 -2
  96. package/src/hosted/server.js +123 -11
  97. package/src/hosted/shares.js +2 -2
  98. package/src/hosted/step-up-settings.js +68 -0
  99. package/src/hosted/step-up.js +135 -0
  100. package/src/hosted/tailnet-identity.js +24 -0
  101. package/src/hosted/tailnet-routes.js +342 -0
  102. package/src/hosted/tailnet.js +220 -0
  103. package/src/hosted/tailscale.js +75 -9
  104. package/src/hosted/tenancy.js +48 -1
  105. package/src/hosted/token-api.js +2 -2
  106. package/src/hosted/tokens.js +9 -2
  107. package/src/hosted/usage-core.js +624 -0
  108. package/src/hosted/usage-layout.js +724 -0
  109. package/src/hosted/usage-routes.js +375 -0
  110. package/src/hosted/usage-shape.js +256 -0
  111. package/src/hosted/usage.js +641 -0
  112. package/src/remote/agent-connect.js +37 -1
  113. package/src/remote/agent-link.js +2 -0
  114. package/src/remote/login.js +33 -8
  115. package/src/remote/share.js +1 -1
  116. package/src/remote/tailnet-cli.js +547 -0
  117. package/src/server/app.js +15 -0
  118. package/src/server/pairing-routes.js +34 -5
  119. package/src/server/tailscale.js +97 -4
  120. package/src/tailnet-proof.js +766 -0
  121. package/dist/client/assets/index-Cxz1UVcV.css +0 -2
  122. package/dist/client/assets/index-uUZVBYUS.js +0 -37
  123. package/dist/console/assets/LocalAccountsApp-DIdy1KnN.js +0 -1
  124. package/dist/console/assets/LocalConsole-DfcQztF-.js +0 -1
  125. package/dist/console/assets/app-update-dialog-ttl9lF5A.js +0 -1
  126. package/dist/console/assets/console-OqAjD0kV.js +0 -55
  127. package/dist/console/assets/console-kvjrAArQ.css +0 -2
@@ -0,0 +1,641 @@
1
+ // Usage of agents in a workspace: every query the Usage pages, their exports
2
+ // and a later admin tool need, over the usage records (agent_runs: one row
3
+ // per agent turn, numbers and ids only, never text).
4
+ //
5
+ // One query layer. Every function takes the same `filter` and the same
6
+ // `scope`, and builds its WHERE clause with the same function (where()), so a
7
+ // tile, a chart and the table always count the same rows for the same filter.
8
+ //
9
+ // scope who is asking (usageScope): owners and admins read the whole
10
+ // workspace; a member reads the turns they asked for and every turn
11
+ // of an agent they connected (it ran on their machine); viewers and
12
+ // guests read nothing (null: the caller refuses).
13
+ // filter cleanUsageFilter(query): a period (from, to: instants; the
14
+ // viewer's time zone for day, week and month boundaries), lists of
15
+ // agents, projects, people, kinds, models and outcome groups, ranges
16
+ // of tokens and duration, and a search over ids.
17
+ //
18
+ // "Not reported" is not zero: a sum over rows where no agent reported a
19
+ // value is null, and each aggregate says how many rows reported
20
+ // (`reported`). Cost is only ever what an agent or its endpoint reported.
21
+ //
22
+ // All of it is SQL on indexed columns through the tenant handle `t`
23
+ // (src/hosted/tenancy.js): only fixed fragments make up a statement, values
24
+ // are bound.
25
+
26
+ import { agentPolicy } from './agent-policy.js'
27
+ import { newId } from './crypto.js'
28
+ import { HttpError } from './http.js'
29
+ import { readLayout } from './usage-layout.js'
30
+ import { OUTCOME_GROUPS, USAGE_DIMENSIONS, USAGE_ROW_BYTES, USAGE_SORTS } from './usage-shape.js'
31
+ import {
32
+ USAGE_LIMITS,
33
+ USAGE_RETENTION_DAYS,
34
+ aggregate,
35
+ breakdownView,
36
+ budgetDay,
37
+ checkedLayout,
38
+ cleanView,
39
+ cleanViewLayout,
40
+ dimensionLabel,
41
+ latencyBuckets,
42
+ latencyView,
43
+ previousPeriod,
44
+ projectBudgets,
45
+ seriesView,
46
+ tooManyViews,
47
+ topAgents,
48
+ topThread,
49
+ usageRowView,
50
+ } from './usage-core.js'
51
+
52
+ // Everything that is not SQL lives in usage-core.js (the public demo's
53
+ // in-browser backend shares it); callers keep importing it from here.
54
+ export {
55
+ OTHER_KEY,
56
+ USAGE_CSV_COLUMNS,
57
+ USAGE_LIMITS,
58
+ USAGE_RETENTION_DAYS,
59
+ bucketKeys,
60
+ bucketOf,
61
+ cleanBucket,
62
+ cleanDimension,
63
+ cleanMetric,
64
+ cleanTopLimit,
65
+ cleanUsageFilter,
66
+ csvCell,
67
+ localDate,
68
+ previousPeriod,
69
+ usageCsvHeader,
70
+ usageCsvLine,
71
+ usageScope,
72
+ validZone,
73
+ zoneOffset,
74
+ zoneSegments,
75
+ } from './usage-core.js'
76
+
77
+ const DAY = 86_400_000
78
+
79
+ // ---- SQL fragments (fixed text; values are bound) ----
80
+
81
+ const quoted = (values) => values.map((value) => `'${value}'`).join(', ')
82
+ const TOKENS =
83
+ 'CASE WHEN r.input_tokens IS NULL AND r.output_tokens IS NULL THEN NULL ELSE COALESCE(r.input_tokens, 0) + COALESCE(r.output_tokens, 0) END'
84
+ const OUTCOME = `CASE WHEN r.outcome IS NULL THEN 'running' ${Object.entries(OUTCOME_GROUPS)
85
+ .filter(([group]) => group !== 'other')
86
+ .map(([group, values]) => `WHEN r.outcome IN (${quoted(values)}) THEN '${group}'`)
87
+ .join(' ')} ELSE 'other' END`
88
+ const DIMENSION = {
89
+ agent: "COALESCE(r.agent_id, '')",
90
+ project: 'r.project_id',
91
+ person: "COALESCE(r.asked_by, '')",
92
+ kind: 'r.kind',
93
+ model: "COALESCE(r.model, '')",
94
+ outcome: OUTCOME,
95
+ none: "''",
96
+ }
97
+ const inGroup = (group) => `SUM(CASE WHEN r.outcome IN (${quoted(OUTCOME_GROUPS[group])}) THEN 1 ELSE 0 END)`
98
+ const AGGREGATES = `COUNT(*) AS calls, SUM(CASE WHEN r.ended_at IS NULL THEN 1 ELSE 0 END) AS running,
99
+ SUM(r.input_tokens) AS input, SUM(r.output_tokens) AS output, SUM(r.cached_tokens) AS cached,
100
+ SUM(${TOKENS}) AS tokens, COUNT(${TOKENS}) AS tokens_n,
101
+ SUM(r.cost_micros) AS cost, COUNT(r.cost_micros) AS cost_n,
102
+ SUM(r.duration_ms) AS duration, COUNT(r.duration_ms) AS duration_n, SUM(r.steps) AS steps,
103
+ SUM(r.approvals_asked) AS asked, SUM(r.approvals_allowed) AS allowed, SUM(r.approvals_denied) AS denied, SUM(r.approvals_auto) AS auto,
104
+ ${inGroup('completed')} AS completed, ${inGroup('needs_person')} AS needs_person, ${inGroup('failed')} AS failed,
105
+ ${inGroup('stopped')} AS stopped,
106
+ SUM(CASE WHEN r.kind = 'fix' THEN 1 ELSE 0 END) AS fixes,
107
+ SUM(CASE WHEN r.kind = 'fix' AND r.outcome IN (${quoted(OUTCOME_GROUPS.completed)}) THEN 1 ELSE 0 END) AS fixes_done`
108
+ // The local time of a row: its UTC instant moved by the zone's offset then.
109
+ // One WHEN per clock change of the period (`segments`: how many offsets).
110
+ const localSql = (segments) =>
111
+ segments === 1
112
+ ? ':o0'
113
+ : `CASE ${Array.from({ length: segments - 1 }, (_, index) => `WHEN r.started_at < :z${index + 1} THEN :o${index}`).join(' ')} ELSE :o${segments - 1} END`
114
+ const bucketSql = (bucket, filter) => {
115
+ const local = localSql(filter.segments.length)
116
+ if (bucket === 'week') return `date(r.started_at, ${local}, 'weekday 0', '-6 days')`
117
+ if (bucket === 'month') return `strftime('%Y-%m-01', r.started_at, ${local})`
118
+ return `date(r.started_at, ${local})`
119
+ }
120
+ // What a table sorts by. A turn that is still running, or that finished after
121
+ // the first page was read (`:asof`), sorts as it did then: with no value yet.
122
+ // So a turn that ends while someone pages through the table does not move.
123
+ const settled = (column) => `CASE WHEN r.ended_at IS NULL THEN -1 WHEN r.ended_at > :asof THEN -1 ELSE COALESCE(${column}, -1) END`
124
+ const SORT = {
125
+ started: 'r.started_at',
126
+ duration: settled('r.duration_ms'),
127
+ tokens: settled(TOKENS),
128
+ input: settled('r.input_tokens'),
129
+ output: settled('r.output_tokens'),
130
+ cached: settled('r.cached_tokens'),
131
+ cost: settled('r.cost_micros'),
132
+ steps: settled('r.steps'),
133
+ }
134
+ // What the id search reads. A private chat's thread id is not in it.
135
+ const HAYSTACK = `lower(r.id || ' ' || COALESCE(r.agent_id, '') || ' ' || COALESCE(r.claim_id, '') || ' ' || r.project_id || ' ' ||
136
+ COALESCE(r.board_id, '') || ' #' || COALESCE(r.comment_number, '') || ' ' ||
137
+ CASE WHEN r.thread_id LIKE 'chat-%' THEN '' ELSE COALESCE(r.thread_id, '') END)`
138
+
139
+ const iso = (ms) => new Date(ms).toISOString()
140
+ const zoneParams = (filter) => {
141
+ const out = {}
142
+ filter.segments.forEach((segment, index) => {
143
+ out[`o${index}`] = `${segment.offset >= 0 ? '+' : '-'}${Math.abs(segment.offset)} minutes`
144
+ if (segment.until !== null) out[`z${index + 1}`] = iso(segment.until)
145
+ })
146
+ return out
147
+ }
148
+
149
+ // The WHERE clause every usage query shares, and its parameters.
150
+ function where(filter, scope) {
151
+ if (!scope) throw new HttpError(403, 'forbidden', 'Usage is for members, admins and owners.')
152
+ const clauses = ['r.workspace_id = :ws', 'r.started_at >= :from', 'r.started_at < :to']
153
+ const params = { from: iso(filter.fromMs), to: iso(filter.toMs) }
154
+ if (!scope.all) {
155
+ // Their own turns, and every turn of an agent they connected.
156
+ clauses.push(':me IN (r.asked_by, r.agent_owner)')
157
+ params.me = scope.userId
158
+ }
159
+ const within = (values, column, name) => {
160
+ if (!values) return
161
+ clauses.push(`${column} IN (SELECT value FROM json_each(:${name}))`)
162
+ params[name] = JSON.stringify(values)
163
+ }
164
+ within(filter.agents, DIMENSION.agent, 'agents')
165
+ within(filter.projects, DIMENSION.project, 'projects')
166
+ within(filter.people, DIMENSION.person, 'people')
167
+ within(filter.kinds, DIMENSION.kind, 'kinds')
168
+ within(filter.models, DIMENSION.model, 'models')
169
+ within(filter.outcomes, OUTCOME, 'outcomes')
170
+ const range = (value, sql, name) => {
171
+ if (value === null || value === undefined) return
172
+ clauses.push(sql)
173
+ params[name] = value
174
+ }
175
+ range(filter.tokensMin, `${TOKENS} >= :tokens_min`, 'tokens_min')
176
+ range(filter.tokensMax, `${TOKENS} <= :tokens_max`, 'tokens_max')
177
+ range(filter.durationMin, 'r.duration_ms >= :duration_min', 'duration_min')
178
+ range(filter.durationMax, 'r.duration_ms <= :duration_max', 'duration_max')
179
+ if (filter.q) {
180
+ clauses.push(`instr(${HAYSTACK}, :q) > 0`)
181
+ params.q = filter.q
182
+ }
183
+ return { sql: clauses.join(' AND '), params }
184
+ }
185
+
186
+ // A cursor: the sort key and id of the last row, and when the first page was read.
187
+ const encodeCursor = (key, id, asOf) => Buffer.from(JSON.stringify([key, id, asOf])).toString('base64url')
188
+ function decodeCursor(text) {
189
+ if (text === undefined || text === null || text === '') return null
190
+ try {
191
+ const value = JSON.parse(Buffer.from(String(text).slice(0, 400), 'base64url').toString('utf8'))
192
+ if (
193
+ Array.isArray(value) &&
194
+ value.length === 3 &&
195
+ ['string', 'number'].includes(typeof value[0]) &&
196
+ typeof value[1] === 'string' &&
197
+ typeof value[2] === 'string' &&
198
+ Number.isFinite(Date.parse(value[2]))
199
+ )
200
+ return value
201
+ } catch {
202
+ /* not one of ours */
203
+ }
204
+ throw new HttpError(400, 'invalid_cursor', 'cursor: the value a page of rows came with.')
205
+ }
206
+
207
+ const ROW_SELECT = `SELECT r.*, k.name AS agent_name, a.machine AS agent_machine, a.kind AS agent_kind, p.name AS project_name,
208
+ u.display_name AS person_name, u.handle AS person_handle, ${TOKENS} AS total_tokens`
209
+ const ROW_JOINS = `LEFT JOIN agents a ON a.id = r.agent_id AND a.workspace_id = r.workspace_id
210
+ LEFT JOIN api_tokens k ON k.id = a.token_id AND k.workspace_id = r.workspace_id
211
+ LEFT JOIN projects p ON p.id = r.project_id AND p.workspace_id = r.workspace_id
212
+ LEFT JOIN users u ON u.id = r.asked_by`
213
+
214
+ export function createUsage(t, { now = () => new Date() } = {}) {
215
+ // ---- Names for ids ----
216
+
217
+ let names = null
218
+ function loadNames() {
219
+ if (names) return names
220
+ names = { agent: new Map(), project: new Map(), person: new Map() }
221
+ for (const row of t.all(
222
+ `SELECT a.id, a.machine, a.kind, a.project_id, k.name FROM agents a JOIN api_tokens k ON k.id = a.token_id AND k.workspace_id = a.workspace_id
223
+ WHERE a.workspace_id = :ws`,
224
+ ))
225
+ names.agent.set(row.id, { label: row.name, machine: row.machine || null, kind: row.kind, projectId: row.project_id })
226
+ for (const row of t.all('SELECT id, name FROM projects WHERE workspace_id = :ws')) names.project.set(row.id, { label: row.name })
227
+ for (const row of t.all(
228
+ 'SELECT m.user_id, u.display_name, u.handle FROM memberships m JOIN users u ON u.id = m.user_id WHERE m.workspace_id = :ws',
229
+ ))
230
+ names.person.set(row.user_id, { label: row.display_name, handle: row.handle })
231
+ return names
232
+ }
233
+ // A key of a dimension as people read it (names are read only when needed).
234
+ const labelOf = (dimension, key) => dimensionLabel(dimension, key, loadNames)
235
+ const rowView = (row, scope, { detail = false } = {}) =>
236
+ usageRowView(row, scope, {
237
+ detail,
238
+ labelOf,
239
+ isMember: (userId) => loadNames().person.has(userId),
240
+ linkUrl: (item) =>
241
+ `/w/${t.workspaceId}/p/${item.project_id}/?board=${encodeURIComponent(item.board_id)}${
242
+ item.comment_number ? `&c=${item.comment_number}` : ''
243
+ }`,
244
+ })
245
+
246
+ // ---- Aggregates ----
247
+
248
+ // Everything in the period, as one set of numbers.
249
+ function totals(filter, scope) {
250
+ const { sql, params } = where(filter, scope)
251
+ return aggregate(t.get(`SELECT ${AGGREGATES} FROM agent_runs r WHERE ${sql}`, params))
252
+ }
253
+ // Median and 95th percentile turn duration (nearest rank), per bucket or
254
+ // (bucket null) for the whole period. Turns still running have none.
255
+ function latency(filter, scope, bucket = null) {
256
+ const { sql, params } = where(filter, scope)
257
+ const by = bucket ? bucketSql(bucket, filter) : "''"
258
+ const rows = t.all(
259
+ `SELECT b, MAX(n) AS n, MAX(CASE WHEN rn = (n + 1) / 2 THEN d END) AS p50,
260
+ MAX(CASE WHEN rn = MAX(1, (n * 90 + 99) / 100) THEN d END) AS p90,
261
+ MAX(CASE WHEN rn = MAX(1, (n * 95 + 99) / 100) THEN d END) AS p95,
262
+ MAX(CASE WHEN rn = MAX(1, (n * 99 + 99) / 100) THEN d END) AS p99, MAX(d) AS longest
263
+ FROM (SELECT ${by} AS b, r.duration_ms AS d, ROW_NUMBER() OVER (PARTITION BY ${by} ORDER BY r.duration_ms) AS rn,
264
+ COUNT(*) OVER (PARTITION BY ${by}) AS n
265
+ FROM agent_runs r WHERE ${sql} AND r.duration_ms IS NOT NULL)
266
+ GROUP BY b ORDER BY b`,
267
+ bucket ? { ...params, ...zoneParams(filter) } : params,
268
+ )
269
+ return bucket ? latencyBuckets(rows, filter, bucket) : latencyView(rows[0])
270
+ }
271
+ // The period and the one before it, with latency: the summary tiles.
272
+ function summary(filter, scope) {
273
+ const before = previousPeriod(filter)
274
+ return {
275
+ period: { from: iso(filter.fromMs), to: iso(filter.toMs), tz: filter.tz },
276
+ current: { ...totals(filter, scope), latency: latency(filter, scope) },
277
+ previous: { from: iso(before.fromMs), to: iso(before.toMs), ...totals(before, scope), latency: latency(before, scope) },
278
+ }
279
+ }
280
+ // The period split by one dimension, largest first.
281
+ // `previous`: the same split over the period to compare with.
282
+ function breakdown(filter, scope, groupBy = 'agent', { previous = false } = {}) {
283
+ const { sql, params } = where(previous ? previousPeriod(filter) : filter, scope)
284
+ const rows = t.all(`SELECT ${DIMENSION[groupBy]} AS k, ${AGGREGATES} FROM agent_runs r WHERE ${sql} GROUP BY k`, params)
285
+ return breakdownView(rows, groupBy, labelOf)
286
+ }
287
+ // The period over time: one point per bucket and group, every bucket
288
+ // present. More groups than a chart can show fold into "Other", ranked by
289
+ // `metric`. `groupBy` "none": one series (the outcomes and approvals charts).
290
+ function series(filter, scope, { bucket = 'day', groupBy = 'none', metric = 'calls' } = {}) {
291
+ const { sql, params } = where(filter, scope)
292
+ const rows = t.all(
293
+ `SELECT ${bucketSql(bucket, filter)} AS b, ${DIMENSION[groupBy]} AS k, ${AGGREGATES} FROM agent_runs r WHERE ${sql} GROUP BY b, k`,
294
+ { ...params, ...zoneParams(filter) },
295
+ )
296
+ return seriesView(rows, filter, { bucket, groupBy, metric }, labelOf)
297
+ }
298
+
299
+ // ---- Rows ----
300
+
301
+ // One page of turns: sorted, filtered and paged here. `cursor`: what the
302
+ // page before came with. The first page also counts the whole set.
303
+ function rows(filter, scope, { sort = 'started', dir = 'desc', cursor = null, limit = USAGE_LIMITS.page } = {}) {
304
+ if (!USAGE_SORTS.includes(sort)) throw new HttpError(400, 'invalid_sort', `sort: one of ${USAGE_SORTS.join(', ')}.`)
305
+ if (!['asc', 'desc'].includes(dir)) throw new HttpError(400, 'invalid_sort', 'dir: asc or desc.')
306
+ const size = Math.min(Math.max(Number.parseInt(limit, 10) || USAGE_LIMITS.page, 1), USAGE_LIMITS.pageMax)
307
+ const after = decodeCursor(cursor)
308
+ // The table as it stood when its first page was read: turns that started
309
+ // later are not in it, and a turn that ended later keeps its place.
310
+ const asOf = after ? new Date(Date.parse(after[2])).toISOString() : now().toISOString()
311
+ const base = where(filter, scope)
312
+ const sql = `${base.sql} AND r.started_at <= :asof`
313
+ const params = { ...base.params, asof: asOf }
314
+ const key = SORT[sort]
315
+ const compare = dir === 'desc' ? '<' : '>'
316
+ const order = dir === 'desc' ? 'DESC' : 'ASC'
317
+ const page = t.all(
318
+ `${ROW_SELECT} FROM (SELECT r.*, ${key} AS sort_key FROM agent_runs r WHERE ${sql}${
319
+ after ? ` AND (${key}, r.id) ${compare} (:cursor_key, :cursor_id)` : ''
320
+ } ORDER BY ${key} ${order}, r.id ${order} LIMIT :limit) r ${ROW_JOINS} ORDER BY r.sort_key ${order}, r.id ${order}`,
321
+ { ...params, limit: size + 1, ...(after ? { cursor_key: after[0], cursor_id: after[1] } : {}) },
322
+ )
323
+ const more = page.length > size
324
+ const shown = page.slice(0, size)
325
+ const last = shown.at(-1)
326
+ return {
327
+ rows: shown.map((row) => rowView(row, scope)),
328
+ next: more ? encodeCursor(last.sort_key, last.id, asOf) : null,
329
+ ...(after ? {} : { total: t.get(`SELECT COUNT(*) AS n FROM agent_runs r WHERE ${sql}`, params).n }),
330
+ }
331
+ }
332
+ // One turn with everything recorded about it, or null when the reader may
333
+ // not see it (which reads the same as "no such row").
334
+ function row(id, scope) {
335
+ if (!scope) throw new HttpError(403, 'forbidden', 'Usage is for members, admins and owners.')
336
+ if (typeof id !== 'string' || !/^run_[a-f0-9]{24}$/.test(id)) return null
337
+ const found = scope.all
338
+ ? t.get(`${ROW_SELECT} FROM agent_runs r ${ROW_JOINS} WHERE r.workspace_id = :ws AND r.id = :id`, { id })
339
+ : t.get(
340
+ `${ROW_SELECT} FROM agent_runs r ${ROW_JOINS} WHERE r.workspace_id = :ws AND r.id = :id AND :me IN (r.asked_by, r.agent_owner)`,
341
+ { id, me: scope.userId },
342
+ )
343
+ return found ? rowView(found, scope, { detail: true }) : null
344
+ }
345
+ // Every row of a filter, newest first, a page at a time, up to `max`:
346
+ // for the exports. Yields arrays of row views (with their detail).
347
+ function* pages(filter, scope, { max = USAGE_LIMITS.exportRows } = {}) {
348
+ const { sql, params } = where(filter, scope)
349
+ let after = null
350
+ let sent = 0
351
+ while (sent < max) {
352
+ const size = Math.min(USAGE_LIMITS.exportPage, max - sent)
353
+ const page = t.all(
354
+ `${ROW_SELECT} FROM (SELECT r.* FROM agent_runs r WHERE ${sql}${
355
+ after ? ' AND (r.started_at, r.id) < (:cursor_key, :cursor_id)' : ''
356
+ } ORDER BY r.started_at DESC, r.id DESC LIMIT :limit) r ${ROW_JOINS} ORDER BY r.started_at DESC, r.id DESC`,
357
+ { ...params, limit: size, ...(after ? { cursor_key: after[0], cursor_id: after[1] } : {}) },
358
+ )
359
+ if (!page.length) return
360
+ sent += page.length
361
+ after = [page.at(-1).started_at, page.at(-1).id]
362
+ yield page.map((item) => rowView(item, scope, { detail: true }))
363
+ if (page.length < size) return
364
+ }
365
+ }
366
+
367
+ // ---- The "top" lists ----
368
+
369
+ function top(filter, scope, { limit = 5 } = {}) {
370
+ const { sql, params } = where(filter, scope)
371
+ const agents = breakdown(filter, scope, 'agent')
372
+ const threads = t
373
+ .all(
374
+ `SELECT r.project_id, r.board_id, r.thread_id, MAX(r.comment_number) AS comment_number, MAX(r.asked_by) AS asked_by,
375
+ COUNT(*) AS calls, SUM(${TOKENS}) AS tokens, p.name AS project_name
376
+ FROM agent_runs r LEFT JOIN projects p ON p.id = r.project_id AND p.workspace_id = r.workspace_id
377
+ WHERE ${sql} AND r.thread_id IS NOT NULL GROUP BY r.project_id, r.board_id, r.thread_id
378
+ HAVING SUM(${TOKENS}) IS NOT NULL ORDER BY tokens DESC LIMIT :top`,
379
+ { ...params, top: limit },
380
+ )
381
+ .map((item) => topThread(item, scope))
382
+ const fromAgents = topAgents(agents, limit)
383
+ return {
384
+ agents: fromAgents.agents,
385
+ threads,
386
+ slowest: rows(filter, scope, { sort: 'duration', dir: 'desc', limit }).rows.filter((item) => item.durationMs !== null),
387
+ denied: fromAgents.denied,
388
+ }
389
+ }
390
+
391
+ // ---- Filter pickers ----
392
+
393
+ // What there is to filter by, in the reader's scope, over the last
394
+ // USAGE_LIMITS.days: each dimension's values with a count.
395
+ function options(scope) {
396
+ const end = now().getTime() + DAY
397
+ const filter = { fromMs: end - USAGE_LIMITS.days * DAY, toMs: end }
398
+ const out = {}
399
+ for (const dimension of USAGE_DIMENSIONS) {
400
+ const { sql, params } = where(filter, scope)
401
+ out[dimension] = t
402
+ .all(
403
+ `SELECT ${DIMENSION[dimension]} AS k, COUNT(*) AS n FROM agent_runs r WHERE ${sql} GROUP BY k ORDER BY n DESC LIMIT 200`,
404
+ params,
405
+ )
406
+ .map((item) => ({ key: item.k, calls: item.n, ...labelOf(dimension, item.k) }))
407
+ }
408
+ return out
409
+ }
410
+
411
+ // ---- Budgets ----
412
+
413
+ // Each project's agent policy budgets with today's use (the day is UTC, as
414
+ // the policy counts it), the last week's daily average, and a forecast for
415
+ // the end of the day when enough of the day has passed to make one.
416
+ // Budgets are part of the agent policy: admins and owners.
417
+ function budgets(scope) {
418
+ if (!scope?.all) return []
419
+ const day = budgetDay(now())
420
+ const { dayStart, weekStart } = day
421
+ const out = []
422
+ for (const project of t.all('SELECT id, name FROM projects WHERE workspace_id = :ws ORDER BY name')) {
423
+ const policy = agentPolicy(t, project.id)
424
+ const used = t.get(
425
+ `SELECT SUM(CASE WHEN r.kind IN ('chat', 'review') THEN 1 ELSE 0 END) AS turns, SUM(CASE WHEN r.kind = 'fix' THEN 1 ELSE 0 END) AS fixes
426
+ FROM agent_runs r WHERE r.workspace_id = :ws AND r.project_id = :project AND r.started_at >= :start`,
427
+ { project: project.id, start: dayStart },
428
+ )
429
+ const week = t.get(
430
+ `SELECT SUM(CASE WHEN r.kind IN ('chat', 'review') THEN 1 ELSE 0 END) AS turns, SUM(CASE WHEN r.kind = 'fix' THEN 1 ELSE 0 END) AS fixes
431
+ FROM agent_runs r WHERE r.workspace_id = :ws AND r.project_id = :project AND r.started_at >= :start AND r.started_at < :end`,
432
+ { project: project.id, start: weekStart, end: dayStart },
433
+ )
434
+ out.push(projectBudgets(project, policy, used, week, day))
435
+ }
436
+ return out
437
+ }
438
+
439
+ // ---- Storage, retention ----
440
+
441
+ const count = () => t.get('SELECT COUNT(*) AS n FROM agent_runs r WHERE r.workspace_id = :ws').n
442
+ // What the usage records count toward the workspace's storage quota.
443
+ const bytes = () => count() * USAGE_ROW_BYTES
444
+ // Removes finished usage records older than `days`. Returns how many.
445
+ function prune(days, { dryRun = false } = {}) {
446
+ const cutoff = iso(now().getTime() - days * DAY)
447
+ if (dryRun)
448
+ return t.get(
449
+ 'SELECT COUNT(*) AS n FROM agent_runs r WHERE r.workspace_id = :ws AND r.started_at < :cutoff AND r.ended_at IS NOT NULL',
450
+ {
451
+ cutoff,
452
+ },
453
+ ).n
454
+ return Number(
455
+ t.run('DELETE FROM agent_runs WHERE workspace_id = :ws AND started_at < :cutoff AND ended_at IS NOT NULL', { cutoff }).changes,
456
+ )
457
+ }
458
+
459
+ return { totals, latency, summary, breakdown, series, rows, row, pages, top, options, budgets, count, bytes, prune, labelOf }
460
+ }
461
+
462
+ // How long a workspace keeps usage records: its own setting, else the default.
463
+ export function usageRetentionDays(t) {
464
+ return t.get('SELECT retention_usage_days FROM workspaces WHERE workspace_id = :ws')?.retention_usage_days ?? USAGE_RETENTION_DAYS
465
+ }
466
+ // Sets it (null: back to the default). The one function that changes this
467
+ // setting; `audit(action, target, detail)` hears it with before and after.
468
+ export function setUsageRetention(t, days, { audit = null } = {}) {
469
+ if (days !== null && (!Number.isInteger(days) || days < 1 || days > 3650))
470
+ throw new HttpError(400, 'invalid_retention', 'Usage retention: 1 to 3650 days, or the default.')
471
+ const before = t.get('SELECT retention_usage_days FROM workspaces WHERE workspace_id = :ws')?.retention_usage_days ?? null
472
+ t.run('UPDATE workspaces SET retention_usage_days = :days WHERE workspace_id = :ws', { days })
473
+ if (before !== days) audit?.('usage_retention_changed', null, { before, after: days })
474
+ return { days: days ?? USAGE_RETENTION_DAYS, own: days !== null }
475
+ }
476
+
477
+ // ---- Saved views ----
478
+
479
+ // A person's named filters on a page. A view is a name and a query string
480
+ // (the same one the page keeps in its address): opening one runs the same
481
+ // scoped queries as typing the filters, so a view cannot show its reader
482
+ // more than they may see. A view is its maker's; sharing one is sharing the
483
+ // page's address.
484
+ //
485
+ // A view of the Usage page may carry a dashboard layout ("include this
486
+ // layout"). Whoever opens such a view by its link sees that layout without
487
+ // it replacing their own (layoutOf: anyone in the workspace who has the
488
+ // Usage page, by the view's id).
489
+ export function createViews(t, { now = () => new Date() } = {}) {
490
+ const layouts = createLayouts(t, { now })
491
+ const withLayout = () =>
492
+ new Set(t.all("SELECT subject FROM usage_layouts WHERE workspace_id = :ws AND kind = 'view'").map((row) => row.subject))
493
+ const view = (row, carrying = withLayout()) => ({
494
+ id: row.id,
495
+ page: row.page,
496
+ name: row.name,
497
+ query: row.query,
498
+ updatedAt: row.updated_at,
499
+ hasLayout: carrying.has(row.id),
500
+ })
501
+ const listOf = (userId, page) => {
502
+ const carrying = withLayout()
503
+ return t
504
+ .all('SELECT * FROM saved_views WHERE workspace_id = :ws AND user_id = :user AND page = :page ORDER BY name, id', {
505
+ user: userId,
506
+ page,
507
+ })
508
+ .map((row) => view(row, carrying))
509
+ }
510
+ // The layout a view carries, for anyone who was handed its link: the
511
+ // view's name and its layout, or nothing when it carries none. `mine`:
512
+ // whether the reader (`userId`) is the view's maker.
513
+ function layoutOf(id, userId = null) {
514
+ const row = t.get("SELECT id, name, user_id FROM saved_views WHERE workspace_id = :ws AND id = :id AND page = 'usage'", {
515
+ id: String(id || ''),
516
+ })
517
+ const layout = row ? layouts.ofView(row.id) : null
518
+ return layout ? { id: row.id, name: row.name, mine: row.user_id === userId, layout } : null
519
+ }
520
+ function save(userId, given) {
521
+ const { page, name: label, query } = cleanView(given)
522
+ // `layout`: the dashboard layout to keep with the view (null or absent: none).
523
+ const layout = cleanViewLayout(page, given?.layout)
524
+ const keep = (id) => (layout ? layouts.saveForView(id, userId, layout) : layouts.removeForView(id))
525
+ const at = now().toISOString()
526
+ // The same name again replaces that view.
527
+ const same = t.get('SELECT id FROM saved_views WHERE workspace_id = :ws AND user_id = :user AND page = :page AND name = :name', {
528
+ user: userId,
529
+ page,
530
+ name: label,
531
+ })
532
+ if (same) {
533
+ t.run('UPDATE saved_views SET query = :query, updated_at = :at WHERE workspace_id = :ws AND id = :id AND user_id = :user', {
534
+ query,
535
+ at,
536
+ id: same.id,
537
+ user: userId,
538
+ })
539
+ keep(same.id)
540
+ return view(t.get('SELECT * FROM saved_views WHERE workspace_id = :ws AND id = :id', { id: same.id }))
541
+ }
542
+ const mine = t.get('SELECT COUNT(*) AS n FROM saved_views WHERE workspace_id = :ws AND user_id = :user AND page = :page', {
543
+ user: userId,
544
+ page,
545
+ }).n
546
+ if (mine >= USAGE_LIMITS.views) throw tooManyViews()
547
+ const id = newId('vw')
548
+ t.run(
549
+ 'INSERT INTO saved_views (workspace_id, id, user_id, page, name, query, created_at, updated_at) VALUES (:ws, :id, :user, :page, :name, :query, :at, :at)',
550
+ { id, user: userId, page, name: label, query, at },
551
+ )
552
+ keep(id)
553
+ return view(t.get('SELECT * FROM saved_views WHERE workspace_id = :ws AND id = :id', { id }))
554
+ }
555
+ // Only its maker removes a view.
556
+ function remove(userId, id) {
557
+ const result = t.run('DELETE FROM saved_views WHERE workspace_id = :ws AND id = :id AND user_id = :user', {
558
+ id: String(id || ''),
559
+ user: userId,
560
+ })
561
+ if (!Number(result.changes)) throw new HttpError(404, 'unknown_view', 'No such saved view.')
562
+ layouts.removeForView(String(id))
563
+ }
564
+ // When someone leaves the workspace, their views go with them (and their layouts).
565
+ const removeOf = (userId) => {
566
+ layouts.removeOf(userId)
567
+ t.run('DELETE FROM saved_views WHERE workspace_id = :ws AND user_id = :user', { user: userId })
568
+ }
569
+ return { list: listOf, layoutOf, save, remove, removeOf }
570
+ }
571
+
572
+ // ---- Dashboard layouts ----
573
+
574
+ // The layouts of the Usage dashboard (the model: usage-layout.js). A person
575
+ // has at most one of their own per workspace; a workspace has at most one
576
+ // default, which people without their own start from; a saved view may
577
+ // carry one. One function per operation, so the routes, the public demo and
578
+ // a later admin tool do the same thing. Every layout is checked before it
579
+ // is stored and again when it is read: what a stored row holds is never
580
+ // drawn as it is.
581
+ //
582
+ // Who may call what is the caller's to check (usage-routes.js): a person
583
+ // reads and writes only their own; the default is read by everyone who has
584
+ // the Usage page and set by owners (NEEDS.usageDefaultLayout).
585
+ export function createLayouts(t, { now = () => new Date() } = {}) {
586
+ const read = (kind, subject) => {
587
+ const row = t.get('SELECT layout FROM usage_layouts WHERE workspace_id = :ws AND kind = :kind AND subject = :subject', {
588
+ kind,
589
+ subject,
590
+ })
591
+ if (!row) return null
592
+ try {
593
+ return readLayout(JSON.parse(row.layout))
594
+ } catch {
595
+ return null
596
+ }
597
+ }
598
+ const write = (kind, subject, given, by) => {
599
+ const layout = checkedLayout(given)
600
+ t.run(
601
+ `INSERT INTO usage_layouts (workspace_id, kind, subject, layout, updated_by, updated_at) VALUES (:ws, :kind, :subject, :layout, :by, :at)
602
+ ON CONFLICT (workspace_id, kind, subject) DO UPDATE SET layout = excluded.layout, updated_by = excluded.updated_by, updated_at = excluded.updated_at`,
603
+ { kind, subject, layout: JSON.stringify(layout), by, at: now().toISOString() },
604
+ )
605
+ return layout
606
+ }
607
+ const drop = (kind, subject) =>
608
+ Number(t.run('DELETE FROM usage_layouts WHERE workspace_id = :ws AND kind = :kind AND subject = :subject', { kind, subject }).changes) >
609
+ 0
610
+
611
+ // A person's own layout (null: they have not changed the dashboard).
612
+ const mine = (userId) => read('person', userId)
613
+ // The layout everyone in the workspace starts from (null: the built-in one).
614
+ const shared = () => read('default', '')
615
+ // Saves a person's own layout. Returns it as stored.
616
+ const save = (userId, layout) => write('person', userId, layout, userId)
617
+ // Back to the workspace's default: forgets a person's own layout.
618
+ const reset = (userId) => drop('person', userId)
619
+ // Sets the layout everyone starts from. `audit(action, target, detail)` hears it.
620
+ function setDefault(userId, layout, { audit = null } = {}) {
621
+ const stored = write('default', '', layout, userId)
622
+ audit?.('usage_layout_default_set', null, { widgets: stored.items.length })
623
+ return stored
624
+ }
625
+ // Removes it: people without their own layout see the built-in one again.
626
+ function clearDefault({ audit = null } = {}) {
627
+ if (drop('default', '')) audit?.('usage_layout_default_removed', null, {})
628
+ }
629
+ const ofView = (viewId) => read('view', viewId)
630
+ const saveForView = (viewId, userId, layout) => write('view', viewId, layout, userId)
631
+ const removeForView = (viewId) => drop('view', viewId)
632
+ // When someone leaves the workspace: their own layout, and the layouts of their views.
633
+ function removeOf(userId) {
634
+ t.run(
635
+ "DELETE FROM usage_layouts WHERE workspace_id = :ws AND kind = 'view' AND subject IN (SELECT id FROM saved_views WHERE workspace_id = :ws AND user_id = :user)",
636
+ { user: userId },
637
+ )
638
+ drop('person', userId)
639
+ }
640
+ return { mine, shared, save, reset, setDefault, clearDefault, ofView, saveForView, removeForView, removeOf }
641
+ }