pi-roundtable-line 0.9.13 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +203 -19
- package/package.json +3 -3
- package/src/api.ts +179 -15
- package/src/consumer.ts +503 -0
- package/src/content.ts +210 -0
- package/src/icon.ts +46 -0
- package/src/index.ts +34 -0
- package/src/intake.ts +2 -2
- package/src/join.ts +136 -0
- package/src/messages.ts +267 -0
- package/src/migrations.ts +100 -1
- package/src/options.ts +86 -2
- package/src/outbox-worker.ts +469 -0
- package/src/outbox.ts +273 -0
- package/src/plugin.ts +125 -9
- package/src/port.ts +23 -6
- package/src/quota.ts +195 -0
- package/src/retention.ts +243 -0
- package/src/route.ts +71 -0
- package/src/sender.ts +110 -0
- package/src/surface.ts +76 -0
package/src/consumer.ts
ADDED
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
import type { SQL } from "bun";
|
|
2
|
+
import type { InboundMessage, Logger } from "pi-roundtable";
|
|
3
|
+
import { errorCode, LineError } from "./errors.ts";
|
|
4
|
+
import type { InboxEvent, InboxProcessor, InboxResult } from "./inbox.ts";
|
|
5
|
+
import type { JoinNotifier } from "./join.ts";
|
|
6
|
+
import {
|
|
7
|
+
type LineContext,
|
|
8
|
+
type LineEditedEvent,
|
|
9
|
+
type LineMessageEvent,
|
|
10
|
+
type LineSubscriber,
|
|
11
|
+
type LineUnsentEvent,
|
|
12
|
+
messageOf,
|
|
13
|
+
nonEmpty,
|
|
14
|
+
readBody,
|
|
15
|
+
readMembers,
|
|
16
|
+
senderOf,
|
|
17
|
+
} from "./messages.ts";
|
|
18
|
+
import { inboundOf } from "./route.ts";
|
|
19
|
+
|
|
20
|
+
/** The most references a subscriber may return for one message, and the longest one. */
|
|
21
|
+
const MAX_REFS = 100;
|
|
22
|
+
const MAX_REF_CHARS = 512;
|
|
23
|
+
|
|
24
|
+
export interface ConsumerDeps {
|
|
25
|
+
sql: SQL;
|
|
26
|
+
logger: Logger;
|
|
27
|
+
now: () => Date;
|
|
28
|
+
/** Answers a `join` with the notice and says how it went; absent, a join's notice is `"unconfirmed"`. */
|
|
29
|
+
joins?: Pick<JoinNotifier, "send">;
|
|
30
|
+
/**
|
|
31
|
+
* Hands a text message of an allowed chat to the host's router, once the subscriber has mirrored
|
|
32
|
+
* it. Throwing fails the event, which is retried. Absent: nothing is routed.
|
|
33
|
+
*/
|
|
34
|
+
route?: (message: InboundMessage) => void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const checkRef = (ref: unknown): string => {
|
|
38
|
+
if (
|
|
39
|
+
typeof ref !== "string" ||
|
|
40
|
+
ref === "" ||
|
|
41
|
+
ref.length > MAX_REF_CHARS ||
|
|
42
|
+
ref.includes("\u0000")
|
|
43
|
+
)
|
|
44
|
+
throw new LineError(
|
|
45
|
+
"bad_refs",
|
|
46
|
+
`line: a ref is a non-empty string of at most ${MAX_REF_CHARS} characters`,
|
|
47
|
+
);
|
|
48
|
+
return ref;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const tooManyRefs = (): LineError =>
|
|
52
|
+
new LineError(
|
|
53
|
+
"bad_refs",
|
|
54
|
+
`line: a message may have at most ${MAX_REFS} refs`,
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The references a subscriber answered, checked: strings that PostgreSQL can store, once each, in
|
|
59
|
+
* order. Nothing is a message with no references to add; anything else than `{ refs }` is refused,
|
|
60
|
+
* so an answer meant as something else (a retry request, say) is not taken for a mirror.
|
|
61
|
+
*/
|
|
62
|
+
function cleanRefs(answer: unknown): string[] {
|
|
63
|
+
if (answer === undefined) return [];
|
|
64
|
+
if (typeof answer !== "object" || answer === null || !("refs" in answer))
|
|
65
|
+
throw new LineError(
|
|
66
|
+
"bad_refs",
|
|
67
|
+
"line: a subscriber answers a message with { refs } or nothing; to give up, throw",
|
|
68
|
+
);
|
|
69
|
+
const refs: unknown = answer.refs;
|
|
70
|
+
if (!Array.isArray(refs) || refs.length > MAX_REFS) throw tooManyRefs();
|
|
71
|
+
return [...new Set(refs.map(checkRef))];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The context of an event that keeps nothing. */
|
|
75
|
+
export const NO_KEEP: LineContext = {
|
|
76
|
+
keep: async () => {
|
|
77
|
+
throw new LineError(
|
|
78
|
+
"keep_unavailable",
|
|
79
|
+
"line: references are kept only while a message is being mirrored",
|
|
80
|
+
);
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/** A token PostgreSQL can store, or null. */
|
|
85
|
+
const storable = (value: unknown): string | null => {
|
|
86
|
+
const text = nonEmpty(value);
|
|
87
|
+
return text && !text.includes("\u0000") ? text : null;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/** The lock that orders everything that decides the fate of one message. */
|
|
91
|
+
const lockOf = (sourceKey: string, messageId: string): string =>
|
|
92
|
+
`${sourceKey} ${messageId}`;
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Turns inbox events into the typed events of the one subscriber and keeps what is needed to take a
|
|
96
|
+
* mirror back: the references the subscriber returned, and a tombstone for every unsent message.
|
|
97
|
+
*
|
|
98
|
+
* Everything that decides what happens to one message (storing its references, writing its
|
|
99
|
+
* tombstone) runs under one advisory lock, so an unsend and a mirror in flight cannot both miss each
|
|
100
|
+
* other: either the references are stored first and the unsend hands them over, or the tombstone is
|
|
101
|
+
* written first and the mirror finds it and is taken back at once.
|
|
102
|
+
*
|
|
103
|
+
* Delivery is at least once, so every step is safe to repeat. The references of an unsent message
|
|
104
|
+
* stay stored until the subscriber has taken them back, so a subscriber that failed halfway is handed
|
|
105
|
+
* the same references again.
|
|
106
|
+
*/
|
|
107
|
+
export class LineConsumer {
|
|
108
|
+
readonly #sql: SQL;
|
|
109
|
+
readonly #logger: Logger;
|
|
110
|
+
readonly #now: () => Date;
|
|
111
|
+
readonly #joins: Pick<JoinNotifier, "send"> | undefined;
|
|
112
|
+
readonly #route: ((message: InboundMessage) => void) | undefined;
|
|
113
|
+
|
|
114
|
+
constructor(deps: ConsumerDeps) {
|
|
115
|
+
this.#sql = deps.sql;
|
|
116
|
+
this.#logger = deps.logger;
|
|
117
|
+
this.#now = deps.now;
|
|
118
|
+
this.#joins = deps.joins;
|
|
119
|
+
this.#route = deps.route;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** An inbox processor that hands typed events to `handler`. */
|
|
123
|
+
processorFor(handler: LineSubscriber): InboxProcessor {
|
|
124
|
+
return async (event, signal) => this.#process(handler, event, signal);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async #process(
|
|
128
|
+
handler: LineSubscriber,
|
|
129
|
+
event: InboxEvent,
|
|
130
|
+
signal: AbortSignal,
|
|
131
|
+
): Promise<InboxResult> {
|
|
132
|
+
const sourceKey = event.sourceKey;
|
|
133
|
+
if (!sourceKey) {
|
|
134
|
+
this.#logger.warn(
|
|
135
|
+
{ event: event.id, type: event.type },
|
|
136
|
+
"a LINE event names no chat and was skipped",
|
|
137
|
+
);
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
const base = {
|
|
141
|
+
eventId: event.id,
|
|
142
|
+
sourceKey,
|
|
143
|
+
timestamp: event.timestamp,
|
|
144
|
+
};
|
|
145
|
+
switch (event.type) {
|
|
146
|
+
case "message":
|
|
147
|
+
await this.#message(handler, event, sourceKey, signal);
|
|
148
|
+
return undefined;
|
|
149
|
+
case "messageEdited":
|
|
150
|
+
await this.#edited(handler, event, sourceKey, signal);
|
|
151
|
+
return undefined;
|
|
152
|
+
case "unsend":
|
|
153
|
+
await this.#unsent(handler, event, sourceKey, signal);
|
|
154
|
+
return undefined;
|
|
155
|
+
case "join": {
|
|
156
|
+
// The notice goes first, on the event's own reply token, which lapses within minutes; the
|
|
157
|
+
// subscriber hears how it went.
|
|
158
|
+
const notice = this.#joins
|
|
159
|
+
? await this.#joins.send(event)
|
|
160
|
+
: "unconfirmed";
|
|
161
|
+
await handler({ type: "joined", ...base, notice }, signal, NO_KEEP);
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
164
|
+
case "leave":
|
|
165
|
+
await handler({ type: "left", ...base }, signal, NO_KEEP);
|
|
166
|
+
return undefined;
|
|
167
|
+
case "memberJoined":
|
|
168
|
+
case "memberLeft": {
|
|
169
|
+
const joined = event.type === "memberJoined";
|
|
170
|
+
const userIds = readMembers(event.payload, joined ? "joined" : "left");
|
|
171
|
+
if (userIds.length > 0)
|
|
172
|
+
await handler(
|
|
173
|
+
{ type: event.type, ...base, userIds },
|
|
174
|
+
signal,
|
|
175
|
+
NO_KEEP,
|
|
176
|
+
);
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
default:
|
|
180
|
+
// follow, postback, beacon and the like carry nothing a mirror shows.
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async #message(
|
|
186
|
+
handler: LineSubscriber,
|
|
187
|
+
event: InboxEvent,
|
|
188
|
+
sourceKey: string,
|
|
189
|
+
signal: AbortSignal,
|
|
190
|
+
): Promise<void> {
|
|
191
|
+
const messageId = event.messageId;
|
|
192
|
+
if (!messageId)
|
|
193
|
+
throw new LineError("bad_message", "line: a message event has no id");
|
|
194
|
+
if (await this.#tombstoned(sourceKey, messageId)) {
|
|
195
|
+
// An unsent message is not mirrored. References left over from a mirror that was cut short
|
|
196
|
+
// are still handed back, so the post they name is taken down.
|
|
197
|
+
await this.#retract(handler, event, sourceKey, messageId, signal, false);
|
|
198
|
+
this.#logger.info(
|
|
199
|
+
{ event: event.id },
|
|
200
|
+
"a LINE message that was unsent is not mirrored",
|
|
201
|
+
);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
if (await this.#mirrored(sourceKey, messageId)) return;
|
|
205
|
+
const message = messageOf(event.payload);
|
|
206
|
+
const userId = senderOf(event.payload);
|
|
207
|
+
const quoted = nonEmpty(message.quotedMessageId);
|
|
208
|
+
const typed: LineMessageEvent = {
|
|
209
|
+
type: "message",
|
|
210
|
+
eventId: event.id,
|
|
211
|
+
sourceKey,
|
|
212
|
+
timestamp: event.timestamp,
|
|
213
|
+
messageId,
|
|
214
|
+
...(userId ? { userId } : {}),
|
|
215
|
+
body: readBody(message),
|
|
216
|
+
...(quoted ? { quotedMessageId: quoted } : {}),
|
|
217
|
+
};
|
|
218
|
+
// Every ref `keep` has stored: whether it stayed stored or an unsend took it back, the answer at
|
|
219
|
+
// the end does not offer it again, so a ref is never handed back twice for the same take-back.
|
|
220
|
+
const kept = new Set<string>();
|
|
221
|
+
const context: LineContext = {
|
|
222
|
+
keep: async (ref) => {
|
|
223
|
+
const unsent = await this.#keep(sourceKey, messageId, checkRef(ref));
|
|
224
|
+
kept.add(ref);
|
|
225
|
+
if (unsent) {
|
|
226
|
+
await this.#retract(
|
|
227
|
+
handler,
|
|
228
|
+
event,
|
|
229
|
+
sourceKey,
|
|
230
|
+
messageId,
|
|
231
|
+
signal,
|
|
232
|
+
false,
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
return { unsent };
|
|
236
|
+
},
|
|
237
|
+
};
|
|
238
|
+
const refs = cleanRefs(await handler(typed, signal, context)).filter(
|
|
239
|
+
(ref) => !kept.has(ref),
|
|
240
|
+
);
|
|
241
|
+
// The router gets the text only inside the store, once the message is known not to be unsent:
|
|
242
|
+
// a router that throws rolls the store back for a retry, and a store that fails routes nothing.
|
|
243
|
+
const route = this.#route;
|
|
244
|
+
const inbound = route ? inboundOf(typed, message) : undefined;
|
|
245
|
+
if (route && !inbound)
|
|
246
|
+
this.#logger.info(
|
|
247
|
+
{ event: event.id },
|
|
248
|
+
"a LINE message was not routed: it is not text from a named sender",
|
|
249
|
+
);
|
|
250
|
+
const late = await this.#store(
|
|
251
|
+
event,
|
|
252
|
+
sourceKey,
|
|
253
|
+
messageId,
|
|
254
|
+
typed,
|
|
255
|
+
refs,
|
|
256
|
+
route && inbound ? () => route(inbound) : undefined,
|
|
257
|
+
);
|
|
258
|
+
if (late) {
|
|
259
|
+
// An unsend landed while the subscriber was mirroring: take the new mirror back at once.
|
|
260
|
+
await this.#retract(handler, event, sourceKey, messageId, signal, false);
|
|
261
|
+
this.#logger.info(
|
|
262
|
+
{ event: event.id },
|
|
263
|
+
"a LINE message was unsent while it was mirrored and was taken back",
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Stores one reference the subscriber has just made, under the message's lock. True when the
|
|
270
|
+
* message has been unsent: the reference is stored all the same, so it can be handed back.
|
|
271
|
+
*/
|
|
272
|
+
async #keep(
|
|
273
|
+
sourceKey: string,
|
|
274
|
+
messageId: string,
|
|
275
|
+
ref: string,
|
|
276
|
+
): Promise<boolean> {
|
|
277
|
+
const now = this.#now();
|
|
278
|
+
return this.#sql.begin(async (tx) => {
|
|
279
|
+
await tx`SELECT pg_advisory_xact_lock(hashtextextended(${lockOf(sourceKey, messageId)}, 11))`;
|
|
280
|
+
await this.#addRefs(tx, sourceKey, messageId, [ref], now);
|
|
281
|
+
return this.#tombstonedIn(tx, sourceKey, messageId);
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Adds references after the ones stored, in order; one already stored keeps its place. The lock is held. */
|
|
286
|
+
async #addRefs(
|
|
287
|
+
tx: SQL,
|
|
288
|
+
sourceKey: string,
|
|
289
|
+
messageId: string,
|
|
290
|
+
refs: readonly string[],
|
|
291
|
+
now: Date,
|
|
292
|
+
): Promise<void> {
|
|
293
|
+
for (const ref of refs)
|
|
294
|
+
await tx`INSERT INTO line_message_refs (source_key, message_id, ref, position, created_at)
|
|
295
|
+
SELECT ${sourceKey}, ${messageId}, ${ref}, COALESCE(MAX(position) + 1, 0), ${now}
|
|
296
|
+
FROM line_message_refs WHERE source_key = ${sourceKey} AND message_id = ${messageId}
|
|
297
|
+
ON CONFLICT (source_key, message_id, ref) DO NOTHING`;
|
|
298
|
+
const [count] = (await tx`SELECT count(*)::int AS n FROM line_message_refs
|
|
299
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId}`) as {
|
|
300
|
+
n: number;
|
|
301
|
+
}[];
|
|
302
|
+
if ((count?.n ?? 0) > MAX_REFS) throw tooManyRefs();
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Keeps the message and the references it answered, unless it has been unsent meanwhile, in which
|
|
307
|
+
* case only the references are kept (so they can be handed back) and true is returned. `deliver`
|
|
308
|
+
* runs under the message's lock after the rows are written, only when the message is not unsent;
|
|
309
|
+
* if it throws nothing is stored.
|
|
310
|
+
*/
|
|
311
|
+
async #store(
|
|
312
|
+
event: InboxEvent,
|
|
313
|
+
sourceKey: string,
|
|
314
|
+
messageId: string,
|
|
315
|
+
typed: LineMessageEvent,
|
|
316
|
+
refs: readonly string[],
|
|
317
|
+
deliver?: () => void,
|
|
318
|
+
): Promise<boolean> {
|
|
319
|
+
const now = this.#now();
|
|
320
|
+
const message = messageOf(event.payload);
|
|
321
|
+
const replyToken = storable(event.payload.replyToken);
|
|
322
|
+
const quoteToken = storable(message.quoteToken);
|
|
323
|
+
return this.#sql.begin(async (tx) => {
|
|
324
|
+
await tx`SELECT pg_advisory_xact_lock(hashtextextended(${lockOf(sourceKey, messageId)}, 11))`;
|
|
325
|
+
const unsent = await this.#tombstonedIn(tx, sourceKey, messageId);
|
|
326
|
+
if (!unsent)
|
|
327
|
+
await tx`INSERT INTO line_messages
|
|
328
|
+
(source_key, message_id, message_type, event_ts, reply_token, reply_token_at, quote_token, created_at)
|
|
329
|
+
VALUES (${sourceKey}, ${messageId}, ${typed.body.kind}, ${event.timestamp},
|
|
330
|
+
${replyToken}, ${replyToken ? new Date(event.timestamp) : null}, ${quoteToken}, ${now})
|
|
331
|
+
ON CONFLICT (source_key, message_id) DO NOTHING`;
|
|
332
|
+
await this.#addRefs(tx, sourceKey, messageId, refs, now);
|
|
333
|
+
if (!unsent) deliver?.();
|
|
334
|
+
return unsent;
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
async #edited(
|
|
339
|
+
handler: LineSubscriber,
|
|
340
|
+
event: InboxEvent,
|
|
341
|
+
sourceKey: string,
|
|
342
|
+
signal: AbortSignal,
|
|
343
|
+
): Promise<void> {
|
|
344
|
+
const messageId = event.messageId;
|
|
345
|
+
if (!messageId)
|
|
346
|
+
throw new LineError("bad_message", "line: an edit event has no id");
|
|
347
|
+
if (await this.#tombstoned(sourceKey, messageId)) return;
|
|
348
|
+
const [row] = (await this.#sql`SELECT edit_ts FROM line_messages
|
|
349
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId}`) as {
|
|
350
|
+
edit_ts: string | number;
|
|
351
|
+
}[];
|
|
352
|
+
if (!row) {
|
|
353
|
+
// The original was never mirrored (it died, or it is older than retention): nothing to edit.
|
|
354
|
+
this.#logger.info(
|
|
355
|
+
{ event: event.id },
|
|
356
|
+
"a LINE edit of a message that is not mirrored was skipped",
|
|
357
|
+
);
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
// Edits may arrive out of order; the one with the largest time is the latest state.
|
|
361
|
+
if (Number(row.edit_ts) >= event.timestamp) return;
|
|
362
|
+
const message = messageOf(event.payload);
|
|
363
|
+
if (message.type !== "text" || typeof message.text !== "string") return;
|
|
364
|
+
const typed: LineEditedEvent = {
|
|
365
|
+
type: "edited",
|
|
366
|
+
eventId: event.id,
|
|
367
|
+
sourceKey,
|
|
368
|
+
timestamp: event.timestamp,
|
|
369
|
+
messageId,
|
|
370
|
+
text: message.text,
|
|
371
|
+
refs: await this.#refs(sourceKey, messageId),
|
|
372
|
+
};
|
|
373
|
+
await handler(typed, signal, NO_KEEP);
|
|
374
|
+
await this
|
|
375
|
+
.#sql`UPDATE line_messages SET edit_ts = GREATEST(edit_ts, ${event.timestamp})
|
|
376
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId}`;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async #unsent(
|
|
380
|
+
handler: LineSubscriber,
|
|
381
|
+
event: InboxEvent,
|
|
382
|
+
sourceKey: string,
|
|
383
|
+
signal: AbortSignal,
|
|
384
|
+
): Promise<void> {
|
|
385
|
+
const messageId = event.messageId;
|
|
386
|
+
if (!messageId)
|
|
387
|
+
throw new LineError("bad_message", "line: an unsend event has no id");
|
|
388
|
+
const now = this.#now();
|
|
389
|
+
await this.#sql.begin(async (tx) => {
|
|
390
|
+
await tx`SELECT pg_advisory_xact_lock(hashtextextended(${lockOf(sourceKey, messageId)}, 11))`;
|
|
391
|
+
await tx`INSERT INTO line_tombstones (source_key, message_id, created_at)
|
|
392
|
+
VALUES (${sourceKey}, ${messageId}, ${now})
|
|
393
|
+
ON CONFLICT (source_key, message_id) DO NOTHING`;
|
|
394
|
+
// Whatever the message said is gone, wherever a copy waits: the event itself, edits of it.
|
|
395
|
+
await tx`UPDATE line_inbox SET payload = NULL
|
|
396
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId} AND payload IS NOT NULL`;
|
|
397
|
+
// Its metadata (tokens, type) goes too; the references stay until they have been handed back.
|
|
398
|
+
await tx`DELETE FROM line_messages WHERE source_key = ${sourceKey} AND message_id = ${messageId}`;
|
|
399
|
+
});
|
|
400
|
+
await this.#retract(handler, event, sourceKey, messageId, signal, true);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Hands the stored references of a message to the subscriber as `unsent`, then forgets them.
|
|
405
|
+
* Without `always` nothing is handed over when there are no references: the subscriber has the
|
|
406
|
+
* unsend event itself, and has nothing of this message to take back.
|
|
407
|
+
*/
|
|
408
|
+
async #retract(
|
|
409
|
+
handler: LineSubscriber,
|
|
410
|
+
event: Pick<InboxEvent, "id" | "timestamp">,
|
|
411
|
+
sourceKey: string,
|
|
412
|
+
messageId: string,
|
|
413
|
+
signal: AbortSignal,
|
|
414
|
+
always: boolean,
|
|
415
|
+
): Promise<void> {
|
|
416
|
+
const refs = await this.#refs(sourceKey, messageId);
|
|
417
|
+
if (refs.length === 0 && !always) return;
|
|
418
|
+
const typed: LineUnsentEvent = {
|
|
419
|
+
type: "unsent",
|
|
420
|
+
eventId: event.id,
|
|
421
|
+
sourceKey,
|
|
422
|
+
timestamp: event.timestamp,
|
|
423
|
+
messageId,
|
|
424
|
+
refs,
|
|
425
|
+
};
|
|
426
|
+
await handler(typed, signal, NO_KEEP);
|
|
427
|
+
if (refs.length > 0)
|
|
428
|
+
await this.#sql`DELETE FROM line_message_refs
|
|
429
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId} AND ref IN ${this.#sql([...refs])}`;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Hands back, as `unsent`, the refs that are still stored for a message that has a tombstone and
|
|
434
|
+
* that were stored before `before`, the message having been unsent before `before` too. They are strays: a handler that outlived its lease kept a ref
|
|
435
|
+
* after the unsend had taken the message back, and nobody was left to hand it over. The pass leaves
|
|
436
|
+
* a ref alone until both it and the unsend are old enough that no unsend or `keep` in flight
|
|
437
|
+
* can still be taking it back. A message whose take-back fails keeps its refs for the next pass. Returns how many
|
|
438
|
+
* messages were handed back.
|
|
439
|
+
*/
|
|
440
|
+
async handBackStrays(
|
|
441
|
+
handler: LineSubscriber,
|
|
442
|
+
signal: AbortSignal,
|
|
443
|
+
before: Date,
|
|
444
|
+
): Promise<number> {
|
|
445
|
+
const strays: { source_key: string; message_id: string }[] = await this
|
|
446
|
+
.#sql`SELECT DISTINCT r.source_key, r.message_id
|
|
447
|
+
FROM line_message_refs r
|
|
448
|
+
JOIN line_tombstones t ON t.source_key = r.source_key AND t.message_id = r.message_id
|
|
449
|
+
WHERE r.created_at < ${before} AND t.created_at < ${before} LIMIT 100`;
|
|
450
|
+
let handed = 0;
|
|
451
|
+
for (const stray of strays) {
|
|
452
|
+
signal.throwIfAborted();
|
|
453
|
+
try {
|
|
454
|
+
await this.#retract(
|
|
455
|
+
handler,
|
|
456
|
+
{
|
|
457
|
+
id: `stray:${stray.source_key}:${stray.message_id}`,
|
|
458
|
+
timestamp: this.#now().getTime(),
|
|
459
|
+
},
|
|
460
|
+
stray.source_key,
|
|
461
|
+
stray.message_id,
|
|
462
|
+
signal,
|
|
463
|
+
false,
|
|
464
|
+
);
|
|
465
|
+
handed++;
|
|
466
|
+
} catch (error) {
|
|
467
|
+
if (signal.aborted) throw error;
|
|
468
|
+
this.#logger.warn(
|
|
469
|
+
{ code: errorCode(error) },
|
|
470
|
+
"refs of an unsent LINE message could not be handed back",
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
return handed;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async #refs(sourceKey: string, messageId: string): Promise<string[]> {
|
|
478
|
+
const rows: { ref: string }[] = await this
|
|
479
|
+
.#sql`SELECT ref FROM line_message_refs
|
|
480
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId} ORDER BY position, ref`;
|
|
481
|
+
return rows.map((row) => row.ref);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
async #tombstoned(sourceKey: string, messageId: string): Promise<boolean> {
|
|
485
|
+
return this.#tombstonedIn(this.#sql, sourceKey, messageId);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
async #tombstonedIn(
|
|
489
|
+
sql: SQL,
|
|
490
|
+
sourceKey: string,
|
|
491
|
+
messageId: string,
|
|
492
|
+
): Promise<boolean> {
|
|
493
|
+
const rows = await sql`SELECT 1 FROM line_tombstones
|
|
494
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId}`;
|
|
495
|
+
return rows.length > 0;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
async #mirrored(sourceKey: string, messageId: string): Promise<boolean> {
|
|
499
|
+
const rows = await this.#sql`SELECT 1 FROM line_messages
|
|
500
|
+
WHERE source_key = ${sourceKey} AND message_id = ${messageId}`;
|
|
501
|
+
return rows.length > 0;
|
|
502
|
+
}
|
|
503
|
+
}
|
package/src/content.ts
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import type { SQL } from "bun";
|
|
2
|
+
import type { LineApi } from "./api.ts";
|
|
3
|
+
import { LineApiError, LineError } from "./errors.ts";
|
|
4
|
+
|
|
5
|
+
export interface MediaOptions {
|
|
6
|
+
/** The most bytes of one message's content held in memory; default 10 MiB. Past it the result is `{ tooLarge }`. */
|
|
7
|
+
maxBytes: number;
|
|
8
|
+
/** How long to wait between looks at a video or audio that LINE is still transcoding; default 2 seconds. */
|
|
9
|
+
transcodingPollMs: number;
|
|
10
|
+
/**
|
|
11
|
+
* How long to wait for it altogether, requests included; default 60 seconds. Past it the call
|
|
12
|
+
* fails with `transcoding_timeout`.
|
|
13
|
+
*/
|
|
14
|
+
transcodingMaxMs: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const MEDIA_DEFAULTS: MediaOptions = {
|
|
18
|
+
maxBytes: 10 * 1024 * 1024,
|
|
19
|
+
transcodingPollMs: 2_000,
|
|
20
|
+
transcodingMaxMs: 60_000,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/** What the content of a message can be, or why it cannot. */
|
|
24
|
+
export type LineContentResult =
|
|
25
|
+
| { bytes: Uint8Array; contentType: string }
|
|
26
|
+
/** The content is bigger than `media.maxBytes`; none of it is kept. */
|
|
27
|
+
| { tooLarge: true }
|
|
28
|
+
/** LINE no longer has it (it expires), the message was unsent, or LINE could not transcode it. */
|
|
29
|
+
| { expired: true; reason: "expired" | "unsent" | "transcoding_failed" };
|
|
30
|
+
|
|
31
|
+
export interface LineContentRequest {
|
|
32
|
+
/**
|
|
33
|
+
* The message's kind, as the `message` event gave it. A `"video"` or `"audio"` is waited for until
|
|
34
|
+
* LINE has transcoded it; without a kind the content is fetched at once.
|
|
35
|
+
*/
|
|
36
|
+
kind?: "image" | "video" | "audio" | "file";
|
|
37
|
+
signal?: AbortSignal;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The longest a download of content may take, body included. */
|
|
41
|
+
const DOWNLOAD_TIMEOUT_MS = 60_000;
|
|
42
|
+
/** The longest one look at the transcoding status may take, when the wait has more time than that left. */
|
|
43
|
+
const STATUS_TIMEOUT_MS = 15_000;
|
|
44
|
+
const MESSAGE_ID = /^[A-Za-z0-9_-]{1,64}$/;
|
|
45
|
+
|
|
46
|
+
function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
if (signal?.aborted) return reject(signal.reason);
|
|
49
|
+
const onAbort = () => {
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
reject(signal?.reason);
|
|
52
|
+
};
|
|
53
|
+
const timer = setTimeout(() => {
|
|
54
|
+
signal?.removeEventListener("abort", onAbort);
|
|
55
|
+
resolve();
|
|
56
|
+
}, ms);
|
|
57
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** LINE answers 404 (and 410) for content it no longer holds or never had. */
|
|
62
|
+
function isGone(error: unknown): boolean {
|
|
63
|
+
return (
|
|
64
|
+
error instanceof LineApiError &&
|
|
65
|
+
(error.status === 404 || error.status === 410)
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Fetches the content of a message into memory. Nothing is written to disk or to a log, and no URL
|
|
71
|
+
* is made public: the bytes go to the caller and nowhere else. A message that was unsent is not
|
|
72
|
+
* fetched at all.
|
|
73
|
+
*/
|
|
74
|
+
export class MessageContent {
|
|
75
|
+
readonly #sql: SQL;
|
|
76
|
+
readonly #api: LineApi;
|
|
77
|
+
readonly options: MediaOptions;
|
|
78
|
+
|
|
79
|
+
constructor(sql: SQL, api: LineApi, options: Partial<MediaOptions> = {}) {
|
|
80
|
+
this.#sql = sql;
|
|
81
|
+
this.#api = api;
|
|
82
|
+
this.options = { ...MEDIA_DEFAULTS, ...options };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async fetch(
|
|
86
|
+
messageId: string,
|
|
87
|
+
request: LineContentRequest = {},
|
|
88
|
+
): Promise<LineContentResult> {
|
|
89
|
+
if (!MESSAGE_ID.test(messageId))
|
|
90
|
+
throw new LineError(
|
|
91
|
+
"bad_message_id",
|
|
92
|
+
"line: a message id is up to 64 letters, digits, - and _",
|
|
93
|
+
);
|
|
94
|
+
const { signal } = request;
|
|
95
|
+
const unsent = await this
|
|
96
|
+
.#sql`SELECT 1 FROM line_tombstones WHERE message_id = ${messageId} LIMIT 1`;
|
|
97
|
+
if (unsent.length > 0) return { expired: true, reason: "unsent" };
|
|
98
|
+
const path = `/v2/bot/message/${messageId}/content`;
|
|
99
|
+
if (request.kind === "video" || request.kind === "audio") {
|
|
100
|
+
const state = await this.#transcoded(path, signal);
|
|
101
|
+
if (state) return state;
|
|
102
|
+
}
|
|
103
|
+
let response: Response;
|
|
104
|
+
try {
|
|
105
|
+
response = await this.#api.request(path, {
|
|
106
|
+
host: "data",
|
|
107
|
+
timeoutMs: DOWNLOAD_TIMEOUT_MS,
|
|
108
|
+
...(signal ? { signal } : {}),
|
|
109
|
+
});
|
|
110
|
+
} catch (error) {
|
|
111
|
+
if (isGone(error)) return { expired: true, reason: "expired" };
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
return this.#read(response, signal);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Waits until LINE has transcoded a video or audio; undefined when it is ready, else the result to answer. */
|
|
118
|
+
async #transcoded(
|
|
119
|
+
path: string,
|
|
120
|
+
signal: AbortSignal | undefined,
|
|
121
|
+
): Promise<LineContentResult | undefined> {
|
|
122
|
+
const deadline = Date.now() + this.options.transcodingMaxMs;
|
|
123
|
+
for (let first = true; ; first = false) {
|
|
124
|
+
const remaining = deadline - Date.now();
|
|
125
|
+
// At least one look, however short the wait is.
|
|
126
|
+
if (remaining <= 0 && !first) break;
|
|
127
|
+
let status: unknown;
|
|
128
|
+
try {
|
|
129
|
+
const body = (await this.#api.json(`${path}/transcoding`, {
|
|
130
|
+
host: "data",
|
|
131
|
+
// A slow answer cannot outlast the wait: each request gets what is left of it.
|
|
132
|
+
timeoutMs: Math.max(1, Math.min(remaining, STATUS_TIMEOUT_MS)),
|
|
133
|
+
...(signal ? { signal } : {}),
|
|
134
|
+
})) as { status?: unknown };
|
|
135
|
+
status = body?.status;
|
|
136
|
+
} catch (error) {
|
|
137
|
+
if (isGone(error)) return { expired: true, reason: "expired" };
|
|
138
|
+
if (
|
|
139
|
+
error instanceof LineError &&
|
|
140
|
+
error.code === "api_unreachable" &&
|
|
141
|
+
remaining <= STATUS_TIMEOUT_MS
|
|
142
|
+
)
|
|
143
|
+
break;
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
if (status === "succeeded") return undefined;
|
|
147
|
+
if (status === "failed")
|
|
148
|
+
return { expired: true, reason: "transcoding_failed" };
|
|
149
|
+
await sleep(
|
|
150
|
+
Math.max(
|
|
151
|
+
0,
|
|
152
|
+
Math.min(this.options.transcodingPollMs, deadline - Date.now()),
|
|
153
|
+
),
|
|
154
|
+
signal,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
throw new LineError(
|
|
158
|
+
"transcoding_timeout",
|
|
159
|
+
"line: LINE did not finish transcoding the content in time",
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Reads the body up to the cap; past it nothing is kept. */
|
|
164
|
+
async #read(
|
|
165
|
+
response: Response,
|
|
166
|
+
signal: AbortSignal | undefined,
|
|
167
|
+
): Promise<LineContentResult> {
|
|
168
|
+
const max = this.options.maxBytes;
|
|
169
|
+
const declared = Number(response.headers.get("content-length"));
|
|
170
|
+
if (declared > max) {
|
|
171
|
+
await response.body?.cancel().catch(() => undefined);
|
|
172
|
+
return { tooLarge: true };
|
|
173
|
+
}
|
|
174
|
+
const contentType =
|
|
175
|
+
response.headers
|
|
176
|
+
.get("content-type")
|
|
177
|
+
?.split(";")[0]
|
|
178
|
+
?.trim()
|
|
179
|
+
.toLowerCase() || "application/octet-stream";
|
|
180
|
+
const reader = response.body?.getReader();
|
|
181
|
+
if (!reader) return { bytes: new Uint8Array(0), contentType };
|
|
182
|
+
const chunks: Uint8Array[] = [];
|
|
183
|
+
let total = 0;
|
|
184
|
+
try {
|
|
185
|
+
for (;;) {
|
|
186
|
+
const { done, value } = await reader.read();
|
|
187
|
+
if (done) break;
|
|
188
|
+
total += value.byteLength;
|
|
189
|
+
if (total > max) {
|
|
190
|
+
await reader.cancel().catch(() => undefined);
|
|
191
|
+
return { tooLarge: true };
|
|
192
|
+
}
|
|
193
|
+
chunks.push(value);
|
|
194
|
+
}
|
|
195
|
+
} catch {
|
|
196
|
+
if (signal?.aborted) throw signal.reason;
|
|
197
|
+
throw new LineError(
|
|
198
|
+
"content_interrupted",
|
|
199
|
+
"line: the content download was cut short",
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
const bytes = new Uint8Array(total);
|
|
203
|
+
let offset = 0;
|
|
204
|
+
for (const chunk of chunks) {
|
|
205
|
+
bytes.set(chunk, offset);
|
|
206
|
+
offset += chunk.byteLength;
|
|
207
|
+
}
|
|
208
|
+
return { bytes, contentType };
|
|
209
|
+
}
|
|
210
|
+
}
|