tsdmarc 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +201 -0
- package/NOTICE +61 -0
- package/README.md +137 -2
- package/dist/index.d.ts +2700 -0
- package/dist/index.js +3146 -0
- package/dist/index.js.map +1 -0
- package/dist/parallel-worker.d.ts +1 -0
- package/dist/parallel-worker.js +42 -0
- package/dist/parallel-worker.js.map +1 -0
- package/dist/report-file-C6u0I5I1.js +27271 -0
- package/dist/report-file-C6u0I5I1.js.map +1 -0
- package/package.json +92 -3
- package/resources/base_reverse_dns_map.csv +38629 -0
- package/resources/known_unknown_base_reverse_dns.txt +31882 -0
- package/resources/psl_overrides.txt +298 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3146 @@
|
|
|
1
|
+
import { $ as RFC_9990_NAMESPACE, $n as InvalidForensicReport, $t as camelToSnake, A as pyJsonDumps, An as Logger, Ar as createParserConfig, At as DNSTimeout, B as CalledProcessError, Bn as formatAsctime, Br as RECOMMENDED_DNS_NAMESERVERS, Bt as UnknownRdatatype, Cn as CRITICAL, Cr as DEFAULT_MAX_XML_DEPTH, Ct as queryDns, D as parseReportFileTask, Dn as FileHandler, Dr as IP_ADDRESS_CACHE, Dt as DNSException, E as parseReportEmailTask, En as ERROR, Er as ExpiringDict, Et as BadResponse, F as parseForensicReport, Fn as basicFormat, Fr as replaceParserConfig, Ft as NXDOMAIN, G as isMbox, Gn as DownloadError, Gt as InvalidURL, H as convertOutlookMsg, Hn as getLogger, Hr as VERSION, Ht as getBaseDomain, I as parsedFailureReportsToCsv, In as captureLogs, Ir as resolveParserConfig, It as NameTooLong, J as parseEmailAddress, Jn as EnrichmentError, Jt as Message, K as isOutlookMsg, Kn as EmailParserError, Kt as maskNameserverSecrets, L as parsedFailureReportsToCsvRows, Ln as configureLogging, Lr as DEFAULT_DNS_MAX_RETRIES, Lt as NoAnswer, M as EMAIL_SAMPLE_CONTENT_TYPES, Mn as NullHandler, Mr as newSeenAggregateReportIds, Mt as IDNAException, N as FAILURE_CSV_FIELDS, Nn as StreamHandler, Nr as parserConfigFromState, Nt as LabelTooLong, On as Handler, Or as REVERSE_DNS_MAP, Ot as DNSFormError, P as parseFailureReport, Pn as WARNING, Pr as parserConfigToState, Pt as LifetimeTimeout, Q as RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES, Qn as InvalidFailureReport, Qt as RecursionError, R as parsedForensicReportsToCsv, Rn as consoleFormat, Rr as DEFAULT_DNS_TIMEOUT, Rt as NoMetaqueries, Sn as PyDateTime, Sr as DEFAULT_MAX_WORKER_HEAP_MB, St as getReverseDns, T as parallelMap, Tn as DEBUG, Tr as DEFAULT_PARSER_OPTIONS, Tt as BadEscape, U as decodeBase64, Un as logger, Ut as loadPslOverrides, V as MSGCONVERT_TIMEOUT_MS, Vn as getLevelName, Vr as USER_AGENT, Vt as YXDOMAIN, W as getFilenameSafeString, Wn as sanitizeForLog, Wt as pslOverrides, X as AGGREGATE_CSV_FIELDS, Xn as InvalidAggregateReport, Xt as formatParam, Y as parseEmailText, Yn as IPDatabaseError, Yt as base64BodyEncode, Z as NormalizedRecordBudget, Zn as InvalidDMARCReport, Zt as MessageError, _ as JSONDecodeError, _n as humanTimestampToUnixTimestamp, _r as DEFAULT_MAX_LOOKUPS_PER_REPORT, _t as loadIpDb, a as aggregateReportKey, an as decompressGzipBounded, ar as CsvDictWriter, at as parseAggregateReportXml, b as JobTimeoutError, bn as timestampToDatetime, br as DEFAULT_MAX_NORMALIZED_RECORDS, bt as DNSSocketError, c as emailByteLength, cn as extractReportFromStream, cr as rowsToCsv, ct as parsedAggregateReportsToCsvRows, d as SMTP_TLS_CSV_FIELDS, dn as PyAttributeError, dr as pyStr, dt as configureIpinfoApi, en as snakeToCamel, er as InvalidIPinfoAPIKey, et as RFC_9990_POLICY_OVERRIDE_TYPES, f as parseSmtpTlsFailureDetails, fn as PyValueError, fr as DEFAULT_JOB_TIMEOUT, ft as getIpAddressCountry, g as parsedSmtpTlsReportsToCsvRows, gn as humanTimestampToDatetime, gr as DEFAULT_MAX_HEADER_LINES, gt as ipInfoState, h as parsedSmtpTlsReportsToCsv, hn as DateParseError, hr as DEFAULT_MAX_FAILED_LOOKUPS_PER_REPORT, ht as getServiceFromReverseDnsBaseDomain, i as TEXT_REPORT_REGEX, in as MAX_DECOMPRESSED_SLOW_CODEC_SIZE, ir as findEnrichmentError, it as parseAggregateReportFromFilePath, j as rawJsonIntegers, jn as NOTSET, jr as newIpAddressCache, jt as EmptyLabel, k as PyJsonNumber, kn as INFO, kr as SEEN_AGGREGATE_REPORT_IDS, kt as DNSSyntaxError, l as parseReportEmail, ln as pyDecode, lr as pyFloatRepr, lt as unwrapText, m as parseSmtpTlsReportPolicy, mn as pyInt, mr as DEFAULT_MAX_EXPANDED_EMAIL_SIZE, mt as getIpAddressInfo, n as parseReportFile, nn as toSnakeCaseKeys, nr as ParserError, nt as bucketIntervalByDay, o as checkEmailLimits, on as extractReport, or as csvFormulaEscape, ot as parseReportRecord, p as parseSmtpTlsReportJson, pn as errorText, pr as DEFAULT_MAX_EMAIL_SIZE, pt as getIpAddressDbRecord, q as parseEmail, qn as EmailTooLargeError, qt as redactUrlCredentials, r as parseReportFileJob, rn as MAX_DECOMPRESSED_REPORT_SIZE, rr as emailTooLargeError, rt as parseAggregateReportFile, s as classifyParsedEmail, sn as extractReportFromFilePath, sr as csvSafe, st as parsedAggregateReportsToCsv, t as getDmarcReportsFromMbox, tn as toCamelCaseKeys, tr as InvalidSMTPTLSReport, tt as appendParsedRecord, u as parseReportEmailJob, un as KeyError, ur as pyRepr, ut as ExpatError, v as pyJsonLoads, vn as parseDate, vr as DEFAULT_MAX_LOOKUP_TIME_PER_REPORT, vt as loadReverseDnsMap, w as initWorker, wn as CaptureHandler, wr as DEFAULT_MAX_XML_NODES, wt as reverseNameFromAddress, x as WorkerCrashedError, xn as timestampToHuman, xr as DEFAULT_MAX_SMTP_TLS_REPORT_SIZE, xt as InvalidNameserverError, y as BrokenProcessPool, yn as stripComments, yr as DEFAULT_MAX_MIME_PARTS, yt as DEFAULT_NAMESERVERS, z as parsedForensicReportsToCsvRows, zn as fileFormat, zr as PARSEDMARC_COMPAT_VERSION, zt as NoNameservers } from "./report-file-C6u0I5I1.js";
|
|
2
|
+
import { constants, mkdirSync, readFileSync, statSync } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { mkdir, mkdtemp, open, opendir, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
5
|
+
import { request } from "node:http";
|
|
6
|
+
import { randomBytes } from "node:crypto";
|
|
7
|
+
import { EOL, homedir, hostname, tmpdir } from "node:os";
|
|
8
|
+
import { createSocket } from "node:dgram";
|
|
9
|
+
import { lookup } from "node:dns/promises";
|
|
10
|
+
import { connect, isIP } from "node:net";
|
|
11
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
12
|
+
import { connect as connect$1, getCACertificates } from "node:tls";
|
|
13
|
+
import { zipSync } from "fflate";
|
|
14
|
+
import { deflateSync } from "node:zlib";
|
|
15
|
+
import { request as request$1 } from "node:https";
|
|
16
|
+
import { Buffer as Buffer$1 } from "node:buffer";
|
|
17
|
+
// src/mailbox/base.ts
|
|
18
|
+
/** Raised by mailbox methods a backend does not implement. */
|
|
19
|
+
var NotImplementedError = class extends Error {
|
|
20
|
+
name = "NotImplementedError";
|
|
21
|
+
};
|
|
22
|
+
/** A folder/label operation targets a name that is already taken. */
|
|
23
|
+
var FolderExistsError = class extends Error {
|
|
24
|
+
name = "FolderExistsError";
|
|
25
|
+
};
|
|
26
|
+
/** A folder/label referenced by an operation does not exist. */
|
|
27
|
+
var FolderNotFoundError = class extends Error {
|
|
28
|
+
name = "FolderNotFoundError";
|
|
29
|
+
};
|
|
30
|
+
function notImplemented(method) {
|
|
31
|
+
return Promise.reject(new NotImplementedError(`${method} is not implemented by this mailbox backend`));
|
|
32
|
+
}
|
|
33
|
+
var MailboxConnection = class {
|
|
34
|
+
/**
|
|
35
|
+
* Tells this mailbox from others whose message IDs may be the same, for
|
|
36
|
+
* state kept between calls (several connection objects to one mailbox
|
|
37
|
+
* share it). Backends whose IDs are unique on their own leave it empty.
|
|
38
|
+
*/
|
|
39
|
+
get mailboxIdentity() {
|
|
40
|
+
return "";
|
|
41
|
+
}
|
|
42
|
+
createFolder(_folderName) {
|
|
43
|
+
return notImplemented("createFolder");
|
|
44
|
+
}
|
|
45
|
+
deleteFolder(_folderName) {
|
|
46
|
+
return notImplemented("deleteFolder");
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Rename a folder. Implementations call `ensureNoFolderConflict()` first,
|
|
50
|
+
* so renaming onto an existing name throws `FolderExistsError`.
|
|
51
|
+
*/
|
|
52
|
+
renameFolder(_oldName, _newName) {
|
|
53
|
+
return notImplemented("renameFolder");
|
|
54
|
+
}
|
|
55
|
+
folderExists(_folderName) {
|
|
56
|
+
return notImplemented("folderExists");
|
|
57
|
+
}
|
|
58
|
+
/** Throw `FolderExistsError` if `folderName` already exists. */
|
|
59
|
+
async ensureNoFolderConflict(folderName) {
|
|
60
|
+
if (await this.folderExists(folderName)) throw new FolderExistsError(`cannot rename to ${JSON.stringify(folderName)}: a folder or label with that name already exists`);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Move a folder (and its contents). Give exactly one of `newPath` (the
|
|
64
|
+
* complete new path) or `newParent` (keep the leaf name).
|
|
65
|
+
*/
|
|
66
|
+
async moveFolder(source, options) {
|
|
67
|
+
const newPath = options.newPath ?? null;
|
|
68
|
+
const newParent = options.newParent ?? null;
|
|
69
|
+
if (newPath === null === (newParent === null)) throw new PyValueError("provide exactly one of newPath or newParent");
|
|
70
|
+
if (!await this.folderExists(source)) throw new FolderNotFoundError(`folder ${JSON.stringify(source)} not found`);
|
|
71
|
+
let targetParent;
|
|
72
|
+
let targetPath;
|
|
73
|
+
if (newParent === null) {
|
|
74
|
+
const path = newPath;
|
|
75
|
+
const slash = path.lastIndexOf("/");
|
|
76
|
+
targetParent = slash === -1 ? "" : path.slice(0, slash);
|
|
77
|
+
targetPath = path;
|
|
78
|
+
} else {
|
|
79
|
+
const sourceLeaf = source.slice(source.lastIndexOf("/") + 1);
|
|
80
|
+
targetParent = newParent;
|
|
81
|
+
targetPath = newParent ? `${newParent}/${sourceLeaf}` : sourceLeaf;
|
|
82
|
+
}
|
|
83
|
+
if (targetParent && !await this.folderExists(targetParent)) {
|
|
84
|
+
if (options.create) await this.createFolder(targetParent);
|
|
85
|
+
else throw new FolderNotFoundError(`destination parent ${JSON.stringify(targetParent)} not found (pass create: true to create it)`);
|
|
86
|
+
}
|
|
87
|
+
await this.ensureNoFolderConflict(targetPath);
|
|
88
|
+
await this.doMoveFolder(source, targetParent, targetPath);
|
|
89
|
+
}
|
|
90
|
+
/** Move every message of each source folder into `destination`. */
|
|
91
|
+
async mergeFolders(sources, destination, options = {}) {
|
|
92
|
+
const sourceList = typeof sources === "string" ? [sources] : sources;
|
|
93
|
+
if (!await this.folderExists(destination)) {
|
|
94
|
+
if (options.create) await this.createFolder(destination);
|
|
95
|
+
else throw new FolderNotFoundError(`destination ${JSON.stringify(destination)} not found (pass create: true to create it)`);
|
|
96
|
+
}
|
|
97
|
+
for (const source of sourceList) {
|
|
98
|
+
if (source === destination) continue;
|
|
99
|
+
if (!await this.folderExists(source)) throw new FolderNotFoundError(`folder ${JSON.stringify(source)} not found`);
|
|
100
|
+
for (const messageId of await this.fetchMessages(source)) await this.moveMessageFrom(messageId, source, destination);
|
|
101
|
+
if (!options.keepSourceFolders) await this.deleteFolder(source);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/** Backend-specific relocation, called by `moveFolder()` after validation. */
|
|
105
|
+
doMoveFolder(_source, _targetParent, _targetPath) {
|
|
106
|
+
return notImplemented("moveFolder");
|
|
107
|
+
}
|
|
108
|
+
/** Move one message during `mergeFolders()`. Defaults to `moveMessage()`. */
|
|
109
|
+
moveMessageFrom(messageId, _source, destination) {
|
|
110
|
+
return this.moveMessage(messageId, destination);
|
|
111
|
+
}
|
|
112
|
+
/** Identifiers of the messages in a folder. */
|
|
113
|
+
fetchMessages(_reportsFolder, _options) {
|
|
114
|
+
return notImplemented("fetchMessages");
|
|
115
|
+
}
|
|
116
|
+
/** Raw RFC 822 contents of a message. */
|
|
117
|
+
fetchMessage(_messageId, _options) {
|
|
118
|
+
return notImplemented("fetchMessage");
|
|
119
|
+
}
|
|
120
|
+
deleteMessage(_messageId) {
|
|
121
|
+
return notImplemented("deleteMessage");
|
|
122
|
+
}
|
|
123
|
+
moveMessage(_messageId, _folderName) {
|
|
124
|
+
return notImplemented("moveMessage");
|
|
125
|
+
}
|
|
126
|
+
/** Send a no-op to keep the connection alive (if applicable). */
|
|
127
|
+
keepalive() {
|
|
128
|
+
return notImplemented("keepalive");
|
|
129
|
+
}
|
|
130
|
+
/** Watch for new messages, calling `checkCallback` on new mail or each interval. */
|
|
131
|
+
watch(_options) {
|
|
132
|
+
return notImplemented("watch");
|
|
133
|
+
}
|
|
134
|
+
/** Send through the backend's native API. Returns a message ID when available. */
|
|
135
|
+
sendMessage(_options) {
|
|
136
|
+
return notImplemented("sendMessage");
|
|
137
|
+
}
|
|
138
|
+
/** Release network resources. No-op for backends without a connection. */
|
|
139
|
+
close() {
|
|
140
|
+
return Promise.resolve();
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
// src/mailbox/reports.ts
|
|
144
|
+
/**
|
|
145
|
+
* Per-message count of consecutive failed saves, keyed on
|
|
146
|
+
* `failedSaveKey(connection, reportsFolder, messageUid)` (parsedmarc's
|
|
147
|
+
* `_FAILED_SAVE_ATTEMPTS`). Populated when a `saveCallback` reports a batch
|
|
148
|
+
* unsaved, or a message runs past `jobTimeout` or loses its worker; a message
|
|
149
|
+
* whose count exceeds `maxUnsavedRetries` is moved to `{archiveFolder}/Unsaved` and its
|
|
150
|
+
* entry dropped. A successful save clears the entries of its batch.
|
|
151
|
+
*
|
|
152
|
+
* Process-local and deliberately not persisted: a restart re-attempts every
|
|
153
|
+
* message still in the reports folder (retry rather than shelve). Unlike
|
|
154
|
+
* parsedmarc's, the key carries `mailboxIdentity`: IMAP UIDs are small
|
|
155
|
+
* integers, so two mailboxes would share counters. Reconnecting to the same
|
|
156
|
+
* mailbox keeps them.
|
|
157
|
+
*/
|
|
158
|
+
const FAILED_SAVE_ATTEMPTS = /* @__PURE__ */ new Map();
|
|
159
|
+
/** The `FAILED_SAVE_ATTEMPTS` key: Python's `(reports_folder, str(uid))` plus the mailbox. */
|
|
160
|
+
function failedSaveKey(connection, reportsFolder, messageUid) {
|
|
161
|
+
return JSON.stringify([
|
|
162
|
+
connection.mailboxIdentity,
|
|
163
|
+
reportsFolder,
|
|
164
|
+
String(messageUid)
|
|
165
|
+
]);
|
|
166
|
+
}
|
|
167
|
+
/** Each connection's seen aggregate report IDs, for calls without `config`. */
|
|
168
|
+
const CONNECTION_SEEN_REPORT_IDS = /* @__PURE__ */ new WeakMap();
|
|
169
|
+
/**
|
|
170
|
+
* `resolveParserConfig()` for a mailbox: without an explicit `config`, dedup
|
|
171
|
+
* is per connection instead of per process (parsedmarc's module-wide
|
|
172
|
+
* cache), so a report seen in one mailbox isn't dropped, and its message
|
|
173
|
+
* archived unsaved, in another. Share an explicit `config` to dedup across
|
|
174
|
+
* connections.
|
|
175
|
+
*/
|
|
176
|
+
function resolveMailboxConfig(connection, input) {
|
|
177
|
+
const config = resolveParserConfig(input);
|
|
178
|
+
if (input.config) return config;
|
|
179
|
+
let seenAggregateReportIds = CONNECTION_SEEN_REPORT_IDS.get(connection);
|
|
180
|
+
if (seenAggregateReportIds === void 0) {
|
|
181
|
+
seenAggregateReportIds = newSeenAggregateReportIds();
|
|
182
|
+
CONNECTION_SEEN_REPORT_IDS.set(connection, seenAggregateReportIds);
|
|
183
|
+
}
|
|
184
|
+
return replaceParserConfig(config, { seenAggregateReportIds });
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Fetch one message by UID, casting the UID to what each backend expects
|
|
188
|
+
* (parsedmarc's `_fetch_mailbox_message`). Returns `[messageId, content]`;
|
|
189
|
+
* `messageId` is what later move/delete calls use.
|
|
190
|
+
*
|
|
191
|
+
* A content error raised while fetching (Python's `email.errors.MessageError`
|
|
192
|
+
* or `RecursionError`, from a backend that re-serializes the message) is
|
|
193
|
+
* returned, not thrown, as `content`: a `ParserError` the caller treats like
|
|
194
|
+
* a parse failure, so the message is disposed of as invalid instead of
|
|
195
|
+
* aborting the batch on every run (GHSA-qxwv-2vwj-v828). Transport and
|
|
196
|
+
* service errors still throw: a transient network failure must not move a
|
|
197
|
+
* legitimate report to the invalid folder.
|
|
198
|
+
*
|
|
199
|
+
* With `maxSize` (bytes, not in parsedmarc), a message over the limit is
|
|
200
|
+
* returned as an `EmailTooLargeError` the same way: backends that know a
|
|
201
|
+
* message's size refuse it before downloading it.
|
|
202
|
+
*/
|
|
203
|
+
async function fetchMailboxMessage(connection, msgUid, test, maxSize = null) {
|
|
204
|
+
const limit = maxSize !== null && maxSize > 0 ? { maxSize } : {};
|
|
205
|
+
const messageId = connection.kind === "imap" ? pyInt(msgUid) : String(msgUid);
|
|
206
|
+
let content;
|
|
207
|
+
try {
|
|
208
|
+
content = connection.kind === "msgraph" || connection.kind === "maildir" ? await connection.fetchMessage(messageId, {
|
|
209
|
+
markRead: !test,
|
|
210
|
+
...limit
|
|
211
|
+
}) : await connection.fetchMessage(messageId, limit);
|
|
212
|
+
} catch (error) {
|
|
213
|
+
if (error instanceof MessageError || error instanceof RecursionError) return [messageId, new ParserError(`Unable to fetch message UID ${messageId}: ${error.name}: ${error.message}`, { cause: error })];
|
|
214
|
+
if (error instanceof EmailTooLargeError) return [messageId, new EmailTooLargeError(`Message UID ${msgUid}: ${error.message}`, { cause: error })];
|
|
215
|
+
throw error;
|
|
216
|
+
}
|
|
217
|
+
if (limit.maxSize !== void 0) {
|
|
218
|
+
const size = emailByteLength(content);
|
|
219
|
+
if (size > limit.maxSize) {
|
|
220
|
+
const tooLarge = emailTooLargeError(size, limit.maxSize);
|
|
221
|
+
return [messageId, new EmailTooLargeError(`Message UID ${msgUid}: ${tooLarge.message}`, { cause: tooLarge })];
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return [messageId, content];
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* The warning for a message that couldn't be parsed: a {@link ParserError}'s
|
|
228
|
+
* own message (as parsedmarc logs it), or the error type and message of any
|
|
229
|
+
* other error. Untrusted text is escaped.
|
|
230
|
+
*/
|
|
231
|
+
function invalidMessageWarning(error, messageId) {
|
|
232
|
+
if (error instanceof ParserError) return sanitizeForLog(error.message);
|
|
233
|
+
const name = error instanceof Error ? error.name : "Error";
|
|
234
|
+
return sanitizeForLog(`Unexpected error parsing message UID ${messageId}, treating it as invalid: ${name}: ${errorText(error)}`);
|
|
235
|
+
}
|
|
236
|
+
/** The warning for a message that ran past `jobTimeout`. */
|
|
237
|
+
function timedOutMessageWarning(error, messageId) {
|
|
238
|
+
return `Timed out parsing message UID ${messageId}: ${error.message}`;
|
|
239
|
+
}
|
|
240
|
+
/** The warning for a message whose worker died (usually out of memory). */
|
|
241
|
+
function crashedMessageWarning(error, messageId) {
|
|
242
|
+
return `Worker failed parsing message UID ${messageId}: ${error.message}: ${errorText(error.cause)}`;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Delete or move an unparseable message (parsedmarc's
|
|
246
|
+
* `_dispose_invalid_message`). Callers pass their effective
|
|
247
|
+
* `deleteInvalid` as `del`.
|
|
248
|
+
*/
|
|
249
|
+
async function disposeInvalidMessage(connection, messageId, del, invalidReportsFolder) {
|
|
250
|
+
try {
|
|
251
|
+
const id = connection.kind === "imap" ? pyInt(messageId) : String(messageId);
|
|
252
|
+
if (del) {
|
|
253
|
+
logger.debug(`Deleting message UID ${messageId}`);
|
|
254
|
+
await connection.deleteMessage(id);
|
|
255
|
+
} else {
|
|
256
|
+
logger.debug(`Moving message UID ${messageId} to ${invalidReportsFolder}`);
|
|
257
|
+
await connection.moveMessage(id, invalidReportsFolder);
|
|
258
|
+
}
|
|
259
|
+
} catch (error) {
|
|
260
|
+
const action = del ? "deleting" : "moving";
|
|
261
|
+
const name = error instanceof Error ? error.name : "Error";
|
|
262
|
+
logger.error(`Mailbox error: Error ${action} invalid message UID ${messageId}: ${name}: ${errorText(error)}`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Consolidate a pre-rename `<archive>/Forensic` subfolder into
|
|
267
|
+
* `<archive>/Failure` (parsedmarc's `_migrate_forensic_archive_folder`).
|
|
268
|
+
* A no-op without a legacy folder; never throws (warn, don't crash).
|
|
269
|
+
*/
|
|
270
|
+
async function migrateForensicArchiveFolder(connection, archiveFolder) {
|
|
271
|
+
const oldFolder = `${archiveFolder}/Forensic`;
|
|
272
|
+
const newFolder = `${archiveFolder}/Failure`;
|
|
273
|
+
try {
|
|
274
|
+
if (!await connection.folderExists(oldFolder)) return;
|
|
275
|
+
if (await connection.folderExists(newFolder)) {
|
|
276
|
+
await connection.mergeFolders(oldFolder, newFolder);
|
|
277
|
+
logger.info(`Merged legacy archive folder ${oldFolder} into ${newFolder}`);
|
|
278
|
+
} else {
|
|
279
|
+
await connection.renameFolder(oldFolder, newFolder);
|
|
280
|
+
logger.info(`Renamed legacy archive folder ${oldFolder} to ${newFolder}`);
|
|
281
|
+
}
|
|
282
|
+
} catch (error) {
|
|
283
|
+
logger.warning(`Could not migrate legacy archive folder ${oldFolder} to ${newFolder}: ${errorText(error)}`);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Best-effort create `folder` if the backend says it is missing
|
|
288
|
+
* (parsedmarc's `_ensure_folder`). Never throws.
|
|
289
|
+
*/
|
|
290
|
+
async function ensureFolder(connection, folder) {
|
|
291
|
+
try {
|
|
292
|
+
if (!await connection.folderExists(folder)) await connection.createFolder(folder);
|
|
293
|
+
} catch (error) {
|
|
294
|
+
logger.warning(`Could not create folder ${folder}: ${errorText(error)}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const SINCE_PATTERN = /^(?<amount>\p{Nd}+)(?<unit>[mhdw]\n?)$/u;
|
|
298
|
+
const MINUTES_PER_UNIT = {
|
|
299
|
+
m: 1,
|
|
300
|
+
h: 60,
|
|
301
|
+
d: 1440,
|
|
302
|
+
w: 10080
|
|
303
|
+
};
|
|
304
|
+
const DEFAULT_SINCE_MINUTES = 1440;
|
|
305
|
+
const SECONDS_PER_MINUTE = 60;
|
|
306
|
+
const DECIMAL_BASE = 10;
|
|
307
|
+
const MS_PER_SECOND$3 = 1e3;
|
|
308
|
+
/** Python's `int()` of a run of Unicode decimal digits (`\d` matches them all). */
|
|
309
|
+
function decimalDigitsValue(digits) {
|
|
310
|
+
let value = 0;
|
|
311
|
+
for (const char of digits) {
|
|
312
|
+
let codePoint = char.codePointAt(0) ?? 0;
|
|
313
|
+
let start = codePoint;
|
|
314
|
+
while (/^\p{Nd}$/u.test(String.fromCodePoint(start - 1))) start -= 1;
|
|
315
|
+
codePoint = (codePoint - start) % DECIMAL_BASE;
|
|
316
|
+
value = value * DECIMAL_BASE + codePoint;
|
|
317
|
+
}
|
|
318
|
+
return value;
|
|
319
|
+
}
|
|
320
|
+
/** The minutes a relative `since` (`5m`, `3h`, `2d`, `1w`) stands for. */
|
|
321
|
+
function sinceMinutes(since) {
|
|
322
|
+
const match = SINCE_PATTERN.exec(since);
|
|
323
|
+
if (!match?.groups) {
|
|
324
|
+
logger.warning(`Incorrect format for 'since' option. Provided value: ${since}, expected values: (5m|3h|2d|1w). Ignoring option, fetching messages for last 24hrs. SMTP does not support a time or timezone in since. See https://www.rfc-editor.org/rfc/rfc3501#page-52`);
|
|
325
|
+
return DEFAULT_SINCE_MINUTES;
|
|
326
|
+
}
|
|
327
|
+
const perUnit = MINUTES_PER_UNIT[match.groups.unit ?? ""];
|
|
328
|
+
if (perUnit === void 0) return DEFAULT_SINCE_MINUTES;
|
|
329
|
+
return decimalDigitsValue(match.groups.amount ?? "") * perUnit;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* `strftime("%s")` of an aware UTC datetime, as CPython computes it on
|
|
333
|
+
* glibc: the tm fields go through mktime(), which reads them as local
|
|
334
|
+
* time, so the result is off by the local UTC offset (exact under TZ=UTC).
|
|
335
|
+
*/
|
|
336
|
+
function pyStrftimeEpoch(datetime) {
|
|
337
|
+
return String(Math.floor(datetime.withOffset(null).timestamp()));
|
|
338
|
+
}
|
|
339
|
+
/** Backend-specific `since` and `currentTime` strings. */
|
|
340
|
+
function resolveSince(connection, since) {
|
|
341
|
+
const minutes = sinceMinutes(since);
|
|
342
|
+
const sinceTime = () => PyDateTime.now(true).addSeconds(-minutes * SECONDS_PER_MINUTE);
|
|
343
|
+
switch (connection.kind) {
|
|
344
|
+
case "imap":
|
|
345
|
+
logger.debug("Only days and weeks values in 'since' option are considered for IMAP connections. Examples: 2d or 1w");
|
|
346
|
+
return {
|
|
347
|
+
since: sinceTime().strftime("%d-%b-%Y"),
|
|
348
|
+
currentTime: PyDateTime.now(true).strftime("%d-%b-%Y")
|
|
349
|
+
};
|
|
350
|
+
case "msgraph": return {
|
|
351
|
+
since: sinceTime().isoformat(),
|
|
352
|
+
currentTime: PyDateTime.now(true).isoformat()
|
|
353
|
+
};
|
|
354
|
+
case "gmail": return {
|
|
355
|
+
since: pyStrftimeEpoch(sinceTime()),
|
|
356
|
+
currentTime: pyStrftimeEpoch(PyDateTime.now(true))
|
|
357
|
+
};
|
|
358
|
+
default: return {
|
|
359
|
+
since,
|
|
360
|
+
currentTime: null
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
/** One batch's parsed reports and the message IDs behind them. */
|
|
365
|
+
var Batch = class {
|
|
366
|
+
config;
|
|
367
|
+
aggregateReports = [];
|
|
368
|
+
failureReports = [];
|
|
369
|
+
smtpTlsReports = [];
|
|
370
|
+
/** Aggregate dedup keys, staged until the batch is saved. */
|
|
371
|
+
pendingAggregateKeys = /* @__PURE__ */ new Set();
|
|
372
|
+
aggregateMsgUids = [];
|
|
373
|
+
failureMsgUids = [];
|
|
374
|
+
smtpTlsMsgUids = [];
|
|
375
|
+
constructor(config) {
|
|
376
|
+
this.config = config;
|
|
377
|
+
}
|
|
378
|
+
add(parsedEmail, messageId) {
|
|
379
|
+
const reportType = classifyParsedEmail(parsedEmail, this.aggregateReports, this.failureReports, this.smtpTlsReports, {
|
|
380
|
+
seenAggregateReportIds: this.config.seenAggregateReportIds,
|
|
381
|
+
pendingAggregateKeys: this.pendingAggregateKeys
|
|
382
|
+
});
|
|
383
|
+
if (reportType === "aggregate") this.aggregateMsgUids.push(messageId);
|
|
384
|
+
else if (reportType === "failure") this.failureMsgUids.push(messageId);
|
|
385
|
+
else if (reportType === "smtp_tls") this.smtpTlsMsgUids.push(messageId);
|
|
386
|
+
}
|
|
387
|
+
results() {
|
|
388
|
+
return {
|
|
389
|
+
aggregateReports: this.aggregateReports,
|
|
390
|
+
failureReports: this.failureReports,
|
|
391
|
+
smtpTlsReports: this.smtpTlsReports
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
msgUids() {
|
|
395
|
+
return [
|
|
396
|
+
...this.aggregateMsgUids,
|
|
397
|
+
...this.failureMsgUids,
|
|
398
|
+
...this.smtpTlsMsgUids
|
|
399
|
+
];
|
|
400
|
+
}
|
|
401
|
+
};
|
|
402
|
+
/**
|
|
403
|
+
* Put back messages that ran past `jobTimeout` or lost their worker. Neither
|
|
404
|
+
* says anything about the content (the same report can fit in on a healthy
|
|
405
|
+
* resolver, or in a worker with free memory), so unlike an invalid message
|
|
406
|
+
* one is never deleted: it is retried like an unsaved batch, up to
|
|
407
|
+
* `maxUnsavedRetries` times so it can't hold up every pass, then moved to
|
|
408
|
+
* `Unsaved`.
|
|
409
|
+
*/
|
|
410
|
+
async function holdBackUnfinishedMessages(context, messageIds) {
|
|
411
|
+
if (context.test || messageIds.length === 0) return;
|
|
412
|
+
await holdBackUnsavedBatch(context.connection, messageIds, context.reportsFolder, context.unsavedReportsFolder, context.maxUnsavedRetries);
|
|
413
|
+
}
|
|
414
|
+
async function parseInParallel(context, nProcs) {
|
|
415
|
+
const { connection, messages, messageLimit, test, batch } = context;
|
|
416
|
+
const fetchedIds = [];
|
|
417
|
+
const invalidMsgIds = [];
|
|
418
|
+
const unfinishedMsgIds = [];
|
|
419
|
+
async function* jobs() {
|
|
420
|
+
for (let index = 0; index < messageLimit; index += 1) {
|
|
421
|
+
const msgUid = messages[index];
|
|
422
|
+
logger.debug(`Processing message ${index + 1} of ${messageLimit}: UID ${msgUid}`);
|
|
423
|
+
const [messageId, msgContent] = await fetchMailboxMessage(connection, msgUid, test, context.maxSize);
|
|
424
|
+
if (msgContent instanceof ParserError) {
|
|
425
|
+
logger.warning(sanitizeForLog(msgContent.message));
|
|
426
|
+
invalidMsgIds.push(messageId);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
fetchedIds.push(messageId);
|
|
430
|
+
yield msgContent;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
for await (const result of parallelMap(parseReportEmailTask(context.config), jobs(), nProcs, {
|
|
434
|
+
heartbeat: () => connection.keepalive(),
|
|
435
|
+
onJobError: (error) => error instanceof Error ? error : new Error(String(error))
|
|
436
|
+
})) {
|
|
437
|
+
const messageId = fetchedIds.shift();
|
|
438
|
+
if (result instanceof JobTimeoutError) {
|
|
439
|
+
logger.warning(timedOutMessageWarning(result, messageId));
|
|
440
|
+
unfinishedMsgIds.push(messageId);
|
|
441
|
+
} else if (result instanceof WorkerCrashedError) {
|
|
442
|
+
logger.warning(crashedMessageWarning(result, messageId));
|
|
443
|
+
unfinishedMsgIds.push(messageId);
|
|
444
|
+
} else if (result instanceof Error) {
|
|
445
|
+
const enrichmentError = findEnrichmentError(result);
|
|
446
|
+
if (enrichmentError !== null) throw enrichmentError;
|
|
447
|
+
logger.warning(invalidMessageWarning(result, messageId));
|
|
448
|
+
invalidMsgIds.push(messageId);
|
|
449
|
+
} else batch.add(result, messageId);
|
|
450
|
+
}
|
|
451
|
+
if (!test) for (const invalidMessageId of invalidMsgIds) await disposeInvalidMessage(connection, invalidMessageId, context.deleteInvalid, context.invalidReportsFolder);
|
|
452
|
+
await holdBackUnfinishedMessages(context, unfinishedMsgIds);
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* `parseReportEmail()` on the calling thread, with the connection kept alive
|
|
456
|
+
* and `config.jobTimeout` checked at each keepalive. A failure of either
|
|
457
|
+
* reaches the caller as thrown (see `parseReportEmail()`): the message isn't
|
|
458
|
+
* at fault, and a `ParserError` here would dispose of a legitimate report.
|
|
459
|
+
*/
|
|
460
|
+
async function parseWithKeepalive(context, content) {
|
|
461
|
+
const { connection, config } = context;
|
|
462
|
+
const deadline = config.jobTimeout > 0 ? Date.now() + config.jobTimeout * MS_PER_SECOND$3 : Number.POSITIVE_INFINITY;
|
|
463
|
+
const keepAlive = async () => {
|
|
464
|
+
if (Date.now() >= deadline) throw new JobTimeoutError(`Job did not finish within ${config.jobTimeout} seconds`);
|
|
465
|
+
await connection.keepalive();
|
|
466
|
+
};
|
|
467
|
+
return parseReportEmail(content, {
|
|
468
|
+
config,
|
|
469
|
+
keepAlive
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
async function parseSequentially(context) {
|
|
473
|
+
const { connection, messages, messageLimit, test, batch } = context;
|
|
474
|
+
for (let index = 0; index < messageLimit; index += 1) {
|
|
475
|
+
const msgUid = messages[index];
|
|
476
|
+
logger.debug(`Processing message ${index + 1} of ${messageLimit}: UID ${msgUid}`);
|
|
477
|
+
const [messageId, msgContent] = await fetchMailboxMessage(connection, msgUid, test, context.maxSize);
|
|
478
|
+
try {
|
|
479
|
+
if (msgContent instanceof ParserError) throw msgContent;
|
|
480
|
+
batch.add(await parseWithKeepalive(context, msgContent), messageId);
|
|
481
|
+
} catch (error) {
|
|
482
|
+
if (error instanceof JobTimeoutError) {
|
|
483
|
+
logger.warning(timedOutMessageWarning(error, messageId));
|
|
484
|
+
await holdBackUnfinishedMessages(context, [messageId]);
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
if (!(error instanceof ParserError)) throw error;
|
|
488
|
+
const enrichmentError = findEnrichmentError(error);
|
|
489
|
+
if (enrichmentError !== null) throw enrichmentError;
|
|
490
|
+
logger.warning(invalidMessageWarning(error, messageId));
|
|
491
|
+
if (!test) await disposeInvalidMessage(connection, messageId, context.deleteInvalid, context.invalidReportsFolder);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Failure bookkeeping for an unsaved batch: count the attempt for each
|
|
497
|
+
* message, keep those under the cap in place and move the rest to
|
|
498
|
+
* `Unsaved` (never deleted).
|
|
499
|
+
*/
|
|
500
|
+
async function holdBackUnsavedBatch(connection, batchMsgUids, reportsFolder, unsavedReportsFolder, maxUnsavedRetries) {
|
|
501
|
+
const retainedUids = [];
|
|
502
|
+
const overCapUids = [];
|
|
503
|
+
let highestAttempt = 0;
|
|
504
|
+
for (const msgUid of batchMsgUids) {
|
|
505
|
+
const attemptsKey = failedSaveKey(connection, reportsFolder, msgUid);
|
|
506
|
+
const attempts = (FAILED_SAVE_ATTEMPTS.get(attemptsKey) ?? 0) + 1;
|
|
507
|
+
FAILED_SAVE_ATTEMPTS.set(attemptsKey, attempts);
|
|
508
|
+
if (attempts > maxUnsavedRetries) overCapUids.push(msgUid);
|
|
509
|
+
else {
|
|
510
|
+
retainedUids.push(msgUid);
|
|
511
|
+
highestAttempt = Math.max(highestAttempt, attempts);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
if (retainedUids.length > 0) logger.error(`Reports were not saved: leaving ${retainedUids.length} message(s) in ${reportsFolder} to retry on the next run or check (failed attempt ${highestAttempt} of ${maxUnsavedRetries + 1})`);
|
|
515
|
+
if (overCapUids.length > 0) {
|
|
516
|
+
logger.error(`Reports were not saved after ${maxUnsavedRetries + 1} attempt(s): moving ${overCapUids.length} message(s) from ${reportsFolder} to ${unsavedReportsFolder} instead of retrying them further. They are never deleted -- fix the output destination, then move them back to ${reportsFolder}`);
|
|
517
|
+
await ensureFolder(connection, unsavedReportsFolder);
|
|
518
|
+
for (const msgUid of overCapUids) {
|
|
519
|
+
try {
|
|
520
|
+
await connection.moveMessage(msgUid, unsavedReportsFolder);
|
|
521
|
+
} catch (error) {
|
|
522
|
+
logger.error(`Mailbox error: Error moving message UID ${msgUid}: ${errorText(error)}`);
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
FAILED_SAVE_ATTEMPTS.delete(failedSaveKey(connection, reportsFolder, msgUid));
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
/** Delete or archive each report type's messages per its delete flag. */
|
|
530
|
+
async function disposeSavedBatch(connection, batch, reportsFolder, folders, deleteFlags) {
|
|
531
|
+
const groups = [
|
|
532
|
+
[
|
|
533
|
+
batch.aggregateMsgUids,
|
|
534
|
+
deleteFlags.aggregate,
|
|
535
|
+
folders.aggregate,
|
|
536
|
+
"aggregate report"
|
|
537
|
+
],
|
|
538
|
+
[
|
|
539
|
+
batch.failureMsgUids,
|
|
540
|
+
deleteFlags.failure,
|
|
541
|
+
folders.failure,
|
|
542
|
+
"failure report"
|
|
543
|
+
],
|
|
544
|
+
[
|
|
545
|
+
batch.smtpTlsMsgUids,
|
|
546
|
+
deleteFlags.smtpTls,
|
|
547
|
+
folders.smtpTls,
|
|
548
|
+
"SMTP TLS report"
|
|
549
|
+
]
|
|
550
|
+
];
|
|
551
|
+
for (const [msgUids, deleteType, destinationFolder, label] of groups) {
|
|
552
|
+
const numberOfMsgs = msgUids.length;
|
|
553
|
+
if (numberOfMsgs === 0) continue;
|
|
554
|
+
if (!deleteType) logger.debug(`Moving ${label} messages from ${reportsFolder} to ${destinationFolder}`);
|
|
555
|
+
for (const [index, msgUid] of msgUids.entries()) if (deleteType) {
|
|
556
|
+
logger.debug(`Deleting message ${index + 1} of ${numberOfMsgs}: UID ${msgUid}`);
|
|
557
|
+
try {
|
|
558
|
+
await connection.deleteMessage(msgUid);
|
|
559
|
+
} catch (error) {
|
|
560
|
+
logger.error(`Mailbox error: Error deleting message UID ${msgUid}: ${errorText(error)}`);
|
|
561
|
+
}
|
|
562
|
+
} else {
|
|
563
|
+
logger.debug(`Moving message ${index + 1} of ${numberOfMsgs}: UID ${msgUid}`);
|
|
564
|
+
try {
|
|
565
|
+
await connection.moveMessage(msgUid, destinationFolder);
|
|
566
|
+
} catch (error) {
|
|
567
|
+
logger.error(`Mailbox error: Error moving message UID ${msgUid}: ${errorText(error)}`);
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Fetches and parses DMARC and SMTP TLS reports from a mailbox, then
|
|
574
|
+
* archives (or deletes) each message by report type: `<archive>/Aggregate`,
|
|
575
|
+
* `Failure`, `SMTP-TLS`, `Invalid`, and `Unsaved` for batches the
|
|
576
|
+
* `saveCallback` kept failing to save.
|
|
577
|
+
*
|
|
578
|
+
* Parsing options (or `config`) work as in every parsing function; the
|
|
579
|
+
* mailbox options always apply. With `batchSize: 0` the reports folder is
|
|
580
|
+
* re-checked after the run and new messages are processed recursively.
|
|
581
|
+
*
|
|
582
|
+
* Throws an `EnrichmentError` when enrichment fails for every report
|
|
583
|
+
* (a revoked IPinfo token, no readable IP database): the batch stays in
|
|
584
|
+
* `reportsFolder`, unsaved, for the next run. Unlike parsedmarc, which
|
|
585
|
+
* files those reports as invalid or empty, and deletes them with `delete`.
|
|
586
|
+
*
|
|
587
|
+
* Throws `ValueError` when a delete option is combined with `test`, when
|
|
588
|
+
* `maxUnsavedRetries` is negative, or without a connection.
|
|
589
|
+
*/
|
|
590
|
+
async function getDmarcReportsFromMailbox(connection, options = {}) {
|
|
591
|
+
const { reportsFolder = "INBOX", archiveFolder = "Archive", delete: del = false, deleteAggregate = null, deleteFailure = null, deleteSmtpTls = null, deleteInvalid = null, test = false, results = null, batchSize = 10, since: sinceOption = null, createFolders = true, nProcs = 1, saveCallback = null, maxUnsavedRetries = 2, ...configOptions } = options;
|
|
592
|
+
const deleteFlags = {
|
|
593
|
+
aggregate: deleteAggregate ?? del,
|
|
594
|
+
failure: deleteFailure ?? del,
|
|
595
|
+
smtpTls: deleteSmtpTls ?? del,
|
|
596
|
+
invalid: deleteInvalid ?? del
|
|
597
|
+
};
|
|
598
|
+
if (test && (deleteFlags.aggregate || deleteFlags.failure || deleteFlags.smtpTls || deleteFlags.invalid)) throw new PyValueError("delete options and test are mutually exclusive");
|
|
599
|
+
if (maxUnsavedRetries < 0) throw new PyValueError("max_unsaved_retries must be >= 0");
|
|
600
|
+
if (connection == null) throw new PyValueError("Must supply a connection");
|
|
601
|
+
const config = resolveMailboxConfig(connection, configOptions);
|
|
602
|
+
let currentTime = null;
|
|
603
|
+
let aggregateReports = [];
|
|
604
|
+
let failureReports = [];
|
|
605
|
+
let smtpTlsReports = [];
|
|
606
|
+
const batch = new Batch(config);
|
|
607
|
+
const folders = {
|
|
608
|
+
aggregate: `${archiveFolder}/Aggregate`,
|
|
609
|
+
failure: `${archiveFolder}/Failure`,
|
|
610
|
+
smtpTls: `${archiveFolder}/SMTP-TLS`,
|
|
611
|
+
invalid: `${archiveFolder}/Invalid`,
|
|
612
|
+
unsaved: `${archiveFolder}/Unsaved`
|
|
613
|
+
};
|
|
614
|
+
if (results) {
|
|
615
|
+
aggregateReports = [...results.aggregateReports];
|
|
616
|
+
failureReports = [...results.failureReports];
|
|
617
|
+
smtpTlsReports = [...results.smtpTlsReports];
|
|
618
|
+
}
|
|
619
|
+
if (!test && createFolders) {
|
|
620
|
+
await migrateForensicArchiveFolder(connection, archiveFolder);
|
|
621
|
+
await connection.createFolder(archiveFolder);
|
|
622
|
+
await connection.createFolder(folders.aggregate);
|
|
623
|
+
await connection.createFolder(folders.failure);
|
|
624
|
+
await connection.createFolder(folders.smtpTls);
|
|
625
|
+
await connection.createFolder(folders.invalid);
|
|
626
|
+
if (saveCallback !== null) await connection.createFolder(folders.unsaved);
|
|
627
|
+
}
|
|
628
|
+
let since = sinceOption;
|
|
629
|
+
if (since) ({since, currentTime} = resolveSince(connection, since));
|
|
630
|
+
const messages = await connection.fetchMessages(reportsFolder, {
|
|
631
|
+
batchSize,
|
|
632
|
+
since
|
|
633
|
+
});
|
|
634
|
+
let totalMessages = messages.length;
|
|
635
|
+
logger.debug(`Found ${messages.length} messages in ${reportsFolder}`);
|
|
636
|
+
const messageLimit = batchSize && !since ? Math.min(totalMessages, batchSize) : totalMessages;
|
|
637
|
+
logger.debug(`Processing ${messageLimit} messages`);
|
|
638
|
+
const context = {
|
|
639
|
+
connection,
|
|
640
|
+
maxSize: config.maxEmailSize > 0 ? config.maxEmailSize : null,
|
|
641
|
+
messages,
|
|
642
|
+
messageLimit,
|
|
643
|
+
test,
|
|
644
|
+
config,
|
|
645
|
+
batch,
|
|
646
|
+
deleteInvalid: deleteFlags.invalid,
|
|
647
|
+
invalidReportsFolder: folders.invalid,
|
|
648
|
+
reportsFolder,
|
|
649
|
+
unsavedReportsFolder: folders.unsaved,
|
|
650
|
+
maxUnsavedRetries
|
|
651
|
+
};
|
|
652
|
+
if (nProcs > 1 && messageLimit > 1) await parseInParallel(context, nProcs);
|
|
653
|
+
else await parseSequentially(context);
|
|
654
|
+
let persisted = true;
|
|
655
|
+
let callbackError = null;
|
|
656
|
+
if (saveCallback !== null) try {
|
|
657
|
+
persisted = await saveCallback(batch.results()) !== false;
|
|
658
|
+
} catch (error) {
|
|
659
|
+
persisted = false;
|
|
660
|
+
callbackError = { error };
|
|
661
|
+
}
|
|
662
|
+
const batchMsgUids = batch.msgUids();
|
|
663
|
+
if (persisted) {
|
|
664
|
+
for (const reportKey of batch.pendingAggregateKeys) config.seenAggregateReportIds.set(reportKey, true);
|
|
665
|
+
if (!test) for (const msgUid of batchMsgUids) FAILED_SAVE_ATTEMPTS.delete(failedSaveKey(connection, reportsFolder, msgUid));
|
|
666
|
+
} else if (!test) await holdBackUnsavedBatch(connection, batchMsgUids, reportsFolder, folders.unsaved, maxUnsavedRetries);
|
|
667
|
+
if (callbackError !== null) throw callbackError.error;
|
|
668
|
+
for (const report of batch.aggregateReports) aggregateReports.push(report);
|
|
669
|
+
for (const report of batch.failureReports) failureReports.push(report);
|
|
670
|
+
for (const report of batch.smtpTlsReports) smtpTlsReports.push(report);
|
|
671
|
+
if (persisted && !test) await disposeSavedBatch(connection, batch, reportsFolder, folders, deleteFlags);
|
|
672
|
+
let accumulated = {
|
|
673
|
+
aggregateReports,
|
|
674
|
+
failureReports,
|
|
675
|
+
smtpTlsReports
|
|
676
|
+
};
|
|
677
|
+
if (persisted && !test && !batchSize) totalMessages = currentTime ? (await connection.fetchMessages(reportsFolder, { since: currentTime })).length : (await connection.fetchMessages(reportsFolder)).length;
|
|
678
|
+
else totalMessages = 0;
|
|
679
|
+
if (totalMessages > 0) accumulated = await getDmarcReportsFromMailbox(connection, {
|
|
680
|
+
reportsFolder,
|
|
681
|
+
archiveFolder,
|
|
682
|
+
delete: del,
|
|
683
|
+
deleteAggregate: deleteFlags.aggregate,
|
|
684
|
+
deleteFailure: deleteFlags.failure,
|
|
685
|
+
deleteSmtpTls: deleteFlags.smtpTls,
|
|
686
|
+
deleteInvalid: deleteFlags.invalid,
|
|
687
|
+
test,
|
|
688
|
+
results: accumulated,
|
|
689
|
+
since: currentTime,
|
|
690
|
+
nProcs,
|
|
691
|
+
saveCallback,
|
|
692
|
+
maxUnsavedRetries,
|
|
693
|
+
config
|
|
694
|
+
});
|
|
695
|
+
return accumulated;
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Watches the mailbox for new messages and processes them with
|
|
699
|
+
* {@link getDmarcReportsFromMailbox} on every check, with `callback` as its
|
|
700
|
+
* `saveCallback`: it runs once per batch, before the batch's messages are
|
|
701
|
+
* archived or deleted, and returning `false` (or throwing) leaves them for
|
|
702
|
+
* the next check. What happens to a thrown error is backend-specific: the
|
|
703
|
+
* IMAP and Maildir watch loops log it and keep checking; Gmail and
|
|
704
|
+
* Microsoft Graph end the watch.
|
|
705
|
+
*
|
|
706
|
+
* Throws `ValueError` for a negative `maxUnsavedRetries` before watching.
|
|
707
|
+
*/
|
|
708
|
+
async function watchInbox(mailboxConnection, callback, options = {}) {
|
|
709
|
+
const { reportsFolder = "INBOX", archiveFolder = "Archive", delete: del = false, deleteAggregate = null, deleteFailure = null, deleteSmtpTls = null, deleteInvalid = null, test = false, checkTimeout = 30, batchSize = 10, since = null, configReloading = null, nProcs = 1, maxUnsavedRetries = 2, signal, ...configOptions } = options;
|
|
710
|
+
if (maxUnsavedRetries < 0) throw new PyValueError("max_unsaved_retries must be >= 0");
|
|
711
|
+
const config = resolveMailboxConfig(mailboxConnection, configOptions);
|
|
712
|
+
const checkCallback = async (connection) => {
|
|
713
|
+
await getDmarcReportsFromMailbox(connection, {
|
|
714
|
+
reportsFolder,
|
|
715
|
+
archiveFolder,
|
|
716
|
+
delete: del,
|
|
717
|
+
deleteAggregate,
|
|
718
|
+
deleteFailure,
|
|
719
|
+
deleteSmtpTls,
|
|
720
|
+
deleteInvalid,
|
|
721
|
+
test,
|
|
722
|
+
batchSize,
|
|
723
|
+
since,
|
|
724
|
+
createFolders: false,
|
|
725
|
+
nProcs,
|
|
726
|
+
saveCallback: callback,
|
|
727
|
+
maxUnsavedRetries,
|
|
728
|
+
config
|
|
729
|
+
});
|
|
730
|
+
};
|
|
731
|
+
await mailboxConnection.watch({
|
|
732
|
+
checkCallback,
|
|
733
|
+
checkTimeout,
|
|
734
|
+
...configReloading === null ? {} : { configReloading },
|
|
735
|
+
...signal === void 0 ? {} : { signal }
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
// src/output.ts
|
|
739
|
+
function errorCode(error) {
|
|
740
|
+
if (error instanceof Error && "code" in error) return typeof error.code === "string" ? error.code : void 0;
|
|
741
|
+
}
|
|
742
|
+
/** `os.stat()` that returns `null` where `os.path.exists()` is false. */
|
|
743
|
+
async function statOrNull(filePath) {
|
|
744
|
+
try {
|
|
745
|
+
return await stat(filePath);
|
|
746
|
+
} catch (error) {
|
|
747
|
+
if (errorCode(error) !== void 0) return null;
|
|
748
|
+
throw error;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
/** Python's `os.path.expanduser()` for `~` and `~/...`. */
|
|
752
|
+
function expandUser(filePath) {
|
|
753
|
+
if (filePath === "~" || filePath.startsWith("~/")) return path.join(homedir(), filePath.slice(1));
|
|
754
|
+
return filePath;
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Files up to this size are parsed and rewritten, like parsedmarc. A larger
|
|
758
|
+
* file that ends the way {@link appendJson} writes it only gets the new
|
|
759
|
+
* entries spliced in: parsing it would take seconds per batch.
|
|
760
|
+
*/
|
|
761
|
+
const MAX_REWRITTEN_JSON_BYTES = 8388608;
|
|
762
|
+
/** The largest file ever parsed; V8 strings stop at 512 MiB of characters. */
|
|
763
|
+
const MAX_PARSED_JSON_BYTES = 268435456;
|
|
764
|
+
/** How `json.dump(reports, indent=2)` starts and ends a list of objects. */
|
|
765
|
+
const JSON_ARRAY_HEAD = "[\n {";
|
|
766
|
+
const JSON_ARRAY_TAIL = "\n }\n]";
|
|
767
|
+
/** The closing `\n]`, replaced when entries are spliced in. */
|
|
768
|
+
const JSON_ARRAY_CLOSE_BYTES = 2;
|
|
769
|
+
const COPY_CHUNK_BYTES = 1048576;
|
|
770
|
+
const TEMP_SUFFIX_BYTES = 6;
|
|
771
|
+
const PERMISSION_BITS = 4095;
|
|
772
|
+
/** One array element, laid out as `json.dump(list, indent=2)` writes it. */
|
|
773
|
+
function encodeJsonEntry(entry) {
|
|
774
|
+
try {
|
|
775
|
+
return pyJsonDumps([entry], {
|
|
776
|
+
ensureAscii: false,
|
|
777
|
+
indent: 2
|
|
778
|
+
}).slice(2, -2);
|
|
779
|
+
} catch (error) {
|
|
780
|
+
if (error instanceof RangeError) throw new PyValueError(`Report too large for JSON output: ${error.message}`);
|
|
781
|
+
throw error;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
/** Whether a file looks like what `json.dump(reports, indent=2)` writes. */
|
|
785
|
+
async function isJsonArrayOfObjects(filename, size) {
|
|
786
|
+
const handle = await open(filename, "r");
|
|
787
|
+
try {
|
|
788
|
+
const head = Buffer.alloc(5);
|
|
789
|
+
const tail = Buffer.alloc(6);
|
|
790
|
+
await handle.read(head, 0, head.length, 0);
|
|
791
|
+
await handle.read(tail, 0, tail.length, size - tail.length);
|
|
792
|
+
return head.toString("latin1") === JSON_ARRAY_HEAD && tail.toString("latin1") === JSON_ARRAY_TAIL;
|
|
793
|
+
} finally {
|
|
794
|
+
await handle.close();
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
/** Copy the first `length` bytes of `filename`, without holding them all. */
|
|
798
|
+
async function copyFileStart(filename, length, output) {
|
|
799
|
+
const input = await open(filename, "r");
|
|
800
|
+
try {
|
|
801
|
+
const buffer = Buffer.allocUnsafe(COPY_CHUNK_BYTES);
|
|
802
|
+
let remaining = length;
|
|
803
|
+
while (remaining > 0) {
|
|
804
|
+
const { bytesRead } = await input.read(buffer, 0, Math.min(remaining, buffer.length));
|
|
805
|
+
if (bytesRead === 0) throw new PyValueError(`${filename} shrank while being copied`);
|
|
806
|
+
await output.writeFile(buffer.subarray(0, bytesRead));
|
|
807
|
+
remaining -= bytesRead;
|
|
808
|
+
}
|
|
809
|
+
} finally {
|
|
810
|
+
await input.close();
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
/**
|
|
814
|
+
* Replace `filename` with what `write` produces: a temporary file in the same
|
|
815
|
+
* directory, flushed, then renamed over it, so a crash or a full disk leaves
|
|
816
|
+
* the old content whole. Symbolic links, the mode and (when allowed) the
|
|
817
|
+
* owner are kept. Anything but a regular file (a FIFO, `/dev/stdout`) can't
|
|
818
|
+
* be replaced and is written directly, like Python.
|
|
819
|
+
*/
|
|
820
|
+
async function replaceFile(filename, info, write) {
|
|
821
|
+
if (info !== null && !info.isFile()) {
|
|
822
|
+
const output = await open(filename, "w");
|
|
823
|
+
try {
|
|
824
|
+
await write(output);
|
|
825
|
+
} finally {
|
|
826
|
+
await output.close();
|
|
827
|
+
}
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
830
|
+
const target = info === null ? filename : await realpath(filename);
|
|
831
|
+
const tempPath = `${target}.${randomBytes(TEMP_SUFFIX_BYTES).toString("hex")}.tmp`;
|
|
832
|
+
const output = await open(tempPath, "wx");
|
|
833
|
+
try {
|
|
834
|
+
await write(output);
|
|
835
|
+
if (info !== null) {
|
|
836
|
+
await output.chmod(info.mode & PERMISSION_BITS);
|
|
837
|
+
await output.chown(info.uid, info.gid).catch(() => void 0);
|
|
838
|
+
}
|
|
839
|
+
await output.sync();
|
|
840
|
+
await output.close();
|
|
841
|
+
await rename(tempPath, target);
|
|
842
|
+
} catch (error) {
|
|
843
|
+
await output.close().catch(() => void 0);
|
|
844
|
+
await rm(tempPath, { force: true });
|
|
845
|
+
throw error;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
/** Keep a file {@link appendJson} can't use next to it, and say so. */
|
|
849
|
+
async function setAsideJsonFile(filename, reason) {
|
|
850
|
+
const aside = `${filename}.corrupt-${Date.now()}`;
|
|
851
|
+
await rename(filename, aside);
|
|
852
|
+
logger.warning(`${sanitizeForLog(filename)} ${reason}: moved to ${sanitizeForLog(aside)}, starting a new file`);
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* The entries of an existing JSON array to carry over, or `null` when the
|
|
856
|
+
* file was moved aside (it holds no array that can be read).
|
|
857
|
+
*/
|
|
858
|
+
async function readJsonArray(filename, size) {
|
|
859
|
+
if (size > MAX_PARSED_JSON_BYTES) {
|
|
860
|
+
await setAsideJsonFile(filename, "is too large to parse");
|
|
861
|
+
return null;
|
|
862
|
+
}
|
|
863
|
+
let bytes;
|
|
864
|
+
try {
|
|
865
|
+
bytes = await readFile(filename);
|
|
866
|
+
} catch (error) {
|
|
867
|
+
if (errorCode(error) === void 0) throw error;
|
|
868
|
+
await setAsideJsonFile(filename, `is unreadable (${errorText(error)})`);
|
|
869
|
+
return null;
|
|
870
|
+
}
|
|
871
|
+
const text = pyDecode(bytes, "utf-8");
|
|
872
|
+
try {
|
|
873
|
+
const loaded = pyJsonLoads(text, { exactNumbers: true });
|
|
874
|
+
if (Array.isArray(loaded)) return loaded;
|
|
875
|
+
await setAsideJsonFile(filename, "does not hold a JSON array");
|
|
876
|
+
return null;
|
|
877
|
+
} catch (error) {
|
|
878
|
+
if (!(error instanceof JSONDecodeError)) throw error;
|
|
879
|
+
await setAsideJsonFile(filename, "is not valid JSON");
|
|
880
|
+
return null;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
async function appendJsonEntries(filename, reports) {
|
|
884
|
+
const entries = reports.map((report) => toSnakeCaseKeys(report));
|
|
885
|
+
const info = await statOrNull(filename);
|
|
886
|
+
let existing = [];
|
|
887
|
+
let keptBytes = 0;
|
|
888
|
+
if (info?.isFile() && info.size > 0) {
|
|
889
|
+
if (info.size > MAX_REWRITTEN_JSON_BYTES && await isJsonArrayOfObjects(filename, info.size)) keptBytes = info.size - JSON_ARRAY_CLOSE_BYTES;
|
|
890
|
+
else existing = await readJsonArray(filename, info.size) ?? [];
|
|
891
|
+
}
|
|
892
|
+
await replaceFile(filename, await statOrNull(filename), async (output) => {
|
|
893
|
+
let separator = "[\n";
|
|
894
|
+
if (keptBytes > 0) {
|
|
895
|
+
await copyFileStart(filename, keptBytes, output);
|
|
896
|
+
separator = ",\n";
|
|
897
|
+
}
|
|
898
|
+
for (const entry of [...existing, ...entries]) {
|
|
899
|
+
await output.writeFile(`${separator}${encodeJsonEntry(entry)}`, "utf-8");
|
|
900
|
+
separator = ",\n";
|
|
901
|
+
}
|
|
902
|
+
await output.writeFile("\n]", "utf-8");
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
/** Tails of the {@link appendJson} calls running, by absolute path. */
|
|
906
|
+
const jsonAppends = /* @__PURE__ */ new Map();
|
|
907
|
+
/**
|
|
908
|
+
* Append `reports` to a JSON array on disk, creating the file if needed.
|
|
909
|
+
*
|
|
910
|
+
* Reads the existing array (if the file exists and parses cleanly), merges
|
|
911
|
+
* the new reports onto the end, and replaces the file with a single valid
|
|
912
|
+
* JSON array (indented by 2, keys in snake_case like parsedmarc's output).
|
|
913
|
+
* A file above {@link MAX_REWRITTEN_JSON_BYTES} that already has that layout
|
|
914
|
+
* keeps its bytes and only gets the new entries, which gives the same result.
|
|
915
|
+
* A file that can't be used (corrupt, unreadable, a root that is not an
|
|
916
|
+
* array) is moved to `<filename>.corrupt-<timestamp>` with a warning, never
|
|
917
|
+
* discarded. An empty `reports` leaves the file alone (and creates nothing).
|
|
918
|
+
* Calls for one file run in turn; other processes are not coordinated with.
|
|
919
|
+
*
|
|
920
|
+
* Throws `ValueError` when a single report is too large to serialize.
|
|
921
|
+
*/
|
|
922
|
+
async function appendJson(filename, reports) {
|
|
923
|
+
if (reports.length === 0) return;
|
|
924
|
+
const key = path.resolve(filename);
|
|
925
|
+
const current = (jsonAppends.get(key) ?? Promise.resolve()).catch(() => void 0).then(() => appendJsonEntries(filename, reports));
|
|
926
|
+
jsonAppends.set(key, current);
|
|
927
|
+
try {
|
|
928
|
+
await current;
|
|
929
|
+
} finally {
|
|
930
|
+
if (jsonAppends.get(key) === current) jsonAppends.delete(key);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
/**
|
|
934
|
+
* Append CSV text to a file. When the file already has content, the header
|
|
935
|
+
* line of `csv` is dropped; a header-only `csv` then writes nothing.
|
|
936
|
+
*/
|
|
937
|
+
async function appendCsv(filename, csv) {
|
|
938
|
+
const handle = await open(filename, "a+");
|
|
939
|
+
try {
|
|
940
|
+
let text = csv;
|
|
941
|
+
const { size } = await handle.stat();
|
|
942
|
+
if (size !== 0) {
|
|
943
|
+
const newline = text.indexOf("\n");
|
|
944
|
+
if (newline === -1) throw new PyValueError("not enough values to unpack (expected 2, got 1)");
|
|
945
|
+
text = text.slice(newline + 1);
|
|
946
|
+
if (text.length === 0) return;
|
|
947
|
+
}
|
|
948
|
+
await handle.writeFile(text, "utf-8");
|
|
949
|
+
} finally {
|
|
950
|
+
await handle.close();
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
/** Numbered names tried for a sample before a random one is used. */
|
|
954
|
+
const MAX_SAMPLE_NAME_PROBES = 1e3;
|
|
955
|
+
/**
|
|
956
|
+
* Write `sample` to a file that doesn't exist yet, named `<subject>.eml`,
|
|
957
|
+
* then `<subject> (1).eml`, … A sender repeats a subject at will, so an
|
|
958
|
+
* existing sample is never reused, and a long run of one subject stops
|
|
959
|
+
* probing. Returns the name used, without the extension.
|
|
960
|
+
*/
|
|
961
|
+
async function writeNewSample(directory, subject, sample) {
|
|
962
|
+
for (let count = 0;; count += 1) {
|
|
963
|
+
let filename = subject;
|
|
964
|
+
if (count > MAX_SAMPLE_NAME_PROBES) filename = `${subject} (${randomBytes(TEMP_SUFFIX_BYTES).toString("hex")})`;
|
|
965
|
+
else if (count > 0) filename = `${subject} (${count})`;
|
|
966
|
+
try {
|
|
967
|
+
await writeFile(path.join(directory, `${filename}.eml`), sample, {
|
|
968
|
+
encoding: "utf-8",
|
|
969
|
+
flag: "wx"
|
|
970
|
+
});
|
|
971
|
+
return filename;
|
|
972
|
+
} catch (error) {
|
|
973
|
+
if (errorCode(error) !== "EEXIST") throw error;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
/**
|
|
978
|
+
* Save report data in the given directory.
|
|
979
|
+
*
|
|
980
|
+
* Writes the JSON and CSV files of each report type, then the message
|
|
981
|
+
* sample of each failure report to a `samples` subdirectory, named after the
|
|
982
|
+
* sample's own `subject` header run through {@link getFilenameSafeString},
|
|
983
|
+
* falling back to `sample` when sanitizing leaves nothing. A subject comes
|
|
984
|
+
* from an untrusted sender, so it is sanitized here, at write time; a
|
|
985
|
+
* caller-supplied `filenameSafeSubject` is not trusted and not used. Names
|
|
986
|
+
* that collide, in this call or with a file already there, get a ` (1)`,
|
|
987
|
+
* ` (2)`, … suffix.
|
|
988
|
+
*
|
|
989
|
+
* Throws `ValueError` if `outputDirectory` exists and is not a directory.
|
|
990
|
+
*/
|
|
991
|
+
async function saveOutput(results, options = {}) {
|
|
992
|
+
const { aggregateJsonFilename = "aggregate.json", failureJsonFilename = "failure.json", smtpTlsJsonFilename = "smtp_tls.json", aggregateCsvFilename = "aggregate.csv", failureCsvFilename = "failure.csv", smtpTlsCsvFilename = "smtp_tls.csv" } = options;
|
|
993
|
+
const { aggregateReports, failureReports, smtpTlsReports } = results;
|
|
994
|
+
const outputDirectory = expandUser(options.outputDirectory ?? "output");
|
|
995
|
+
const csvOptions = { formulaEscape: options.csvFormulaEscape !== false };
|
|
996
|
+
const outputInfo = await statOrNull(outputDirectory);
|
|
997
|
+
if (outputInfo === null) await mkdir(outputDirectory, { recursive: true });
|
|
998
|
+
else if (!outputInfo.isDirectory()) throw new PyValueError(`${outputDirectory} is not a directory`);
|
|
999
|
+
const aggregateCsv = parsedAggregateReportsToCsv(aggregateReports, csvOptions);
|
|
1000
|
+
const failureCsv = parsedFailureReportsToCsv(failureReports, csvOptions);
|
|
1001
|
+
const smtpTlsCsv = parsedSmtpTlsReportsToCsv(smtpTlsReports, csvOptions);
|
|
1002
|
+
await appendJson(path.join(outputDirectory, aggregateJsonFilename), aggregateReports);
|
|
1003
|
+
await appendCsv(path.join(outputDirectory, aggregateCsvFilename), aggregateCsv);
|
|
1004
|
+
await appendJson(path.join(outputDirectory, failureJsonFilename), failureReports);
|
|
1005
|
+
await appendCsv(path.join(outputDirectory, failureCsvFilename), failureCsv);
|
|
1006
|
+
await appendJson(path.join(outputDirectory, smtpTlsJsonFilename), smtpTlsReports);
|
|
1007
|
+
await appendCsv(path.join(outputDirectory, smtpTlsCsvFilename), smtpTlsCsv);
|
|
1008
|
+
const samplesDirectory = path.join(outputDirectory, "samples");
|
|
1009
|
+
if (await statOrNull(samplesDirectory) === null) await mkdir(samplesDirectory, { recursive: true });
|
|
1010
|
+
for (const failureReport of failureReports) {
|
|
1011
|
+
const { sample, parsedSample } = failureReport;
|
|
1012
|
+
const subject = getFilenameSafeString(parsedSample.subject ?? null) || "sample";
|
|
1013
|
+
try {
|
|
1014
|
+
await writeNewSample(samplesDirectory, subject, sample);
|
|
1015
|
+
} catch (error) {
|
|
1016
|
+
if (subject === "sample") throw error;
|
|
1017
|
+
const fallback = await writeNewSample(samplesDirectory, "sample", sample);
|
|
1018
|
+
logger.warning(`Could not write sample ${sanitizeForLog(JSON.stringify(subject))}.eml (${errorText(error)}); writing it as ${fallback}.eml instead`);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
/** Entries of a directory in on-disk order, like `os.scandir()`. */
|
|
1023
|
+
async function scanDirectory(directory) {
|
|
1024
|
+
const dirs = [];
|
|
1025
|
+
const files = [];
|
|
1026
|
+
for await (const entry of await opendir(directory)) if (entry.isDirectory()) dirs.push(entry.name);
|
|
1027
|
+
else files.push(entry.name);
|
|
1028
|
+
return {
|
|
1029
|
+
root: directory,
|
|
1030
|
+
dirs,
|
|
1031
|
+
files
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
/**
|
|
1035
|
+
* Python's `os.walk(top)`: top-down, a directory before its children.
|
|
1036
|
+
*
|
|
1037
|
+
* @yields {WalkEntry} Each directory with its subdirectory and file names.
|
|
1038
|
+
*/
|
|
1039
|
+
async function* walk(top) {
|
|
1040
|
+
let entry;
|
|
1041
|
+
try {
|
|
1042
|
+
entry = await scanDirectory(top);
|
|
1043
|
+
} catch {
|
|
1044
|
+
return;
|
|
1045
|
+
}
|
|
1046
|
+
yield entry;
|
|
1047
|
+
for (const directory of entry.dirs) yield* walk(path.join(top, directory));
|
|
1048
|
+
}
|
|
1049
|
+
/** The arcname `zipfile.ZipInfo.from_file()` stores (normalized, relative). */
|
|
1050
|
+
function zipArcname(arcname, isDirectory) {
|
|
1051
|
+
let name = path.normalize(arcname);
|
|
1052
|
+
while (name.startsWith("/")) name = name.slice(1);
|
|
1053
|
+
return isDirectory ? `${name}/` : name;
|
|
1054
|
+
}
|
|
1055
|
+
const ZIP_UNIX = 3;
|
|
1056
|
+
const MS_DOS_DIRECTORY = 16;
|
|
1057
|
+
const MODE_MASK = 65535;
|
|
1058
|
+
const ATTRS_SHIFT = 65536;
|
|
1059
|
+
/** Python's `ZipFile.write(filename, arcname)` with `ZIP_DEFLATED`. */
|
|
1060
|
+
async function addToZip(entries, filePath, arcname) {
|
|
1061
|
+
const info = await stat(filePath);
|
|
1062
|
+
const isDirectory = info.isDirectory();
|
|
1063
|
+
let attrs = (info.mode & MODE_MASK) * ATTRS_SHIFT;
|
|
1064
|
+
if (isDirectory) attrs += MS_DOS_DIRECTORY;
|
|
1065
|
+
const options = {
|
|
1066
|
+
os: ZIP_UNIX,
|
|
1067
|
+
attrs,
|
|
1068
|
+
mtime: info.mtime,
|
|
1069
|
+
level: isDirectory ? 0 : 6
|
|
1070
|
+
};
|
|
1071
|
+
const data = isDirectory ? /* @__PURE__ */ new Uint8Array(0) : await readFile(filePath);
|
|
1072
|
+
entries.push([zipArcname(arcname, isDirectory), [data, options]]);
|
|
1073
|
+
}
|
|
1074
|
+
async function isFile(filePath) {
|
|
1075
|
+
return (await statOrNull(filePath))?.isFile() ?? false;
|
|
1076
|
+
}
|
|
1077
|
+
async function isDirectory$1(filePath) {
|
|
1078
|
+
return (await statOrNull(filePath))?.isDirectory() ?? false;
|
|
1079
|
+
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Creates a zip file of parsed report output: {@link saveOutput} into a
|
|
1082
|
+
* temporary directory, then every file in it.
|
|
1083
|
+
*
|
|
1084
|
+
* The layout is parsedmarc's, except that parsedmarc also stores each file
|
|
1085
|
+
* of `samples/` a second time as `../<name>` (its `add_subdir()` swaps the
|
|
1086
|
+
* `relpath` arguments). Those entries point outside the extraction
|
|
1087
|
+
* directory, a path-traversal risk for whoever unzips the attachment, so
|
|
1088
|
+
* they are left out; every file is still in the zip once.
|
|
1089
|
+
*/
|
|
1090
|
+
async function getReportZip(results, options = {}) {
|
|
1091
|
+
const entries = [];
|
|
1092
|
+
const tmpDir = await mkdtemp(path.join(tmpdir(), "tmp"));
|
|
1093
|
+
try {
|
|
1094
|
+
await saveOutput(results, {
|
|
1095
|
+
outputDirectory: tmpDir,
|
|
1096
|
+
csvFormulaEscape: options.csvFormulaEscape !== false
|
|
1097
|
+
});
|
|
1098
|
+
for await (const { root, dirs, files } of walk(tmpDir)) {
|
|
1099
|
+
for (const file of files) {
|
|
1100
|
+
const filePath = path.join(root, file);
|
|
1101
|
+
if (await isFile(filePath)) await addToZip(entries, filePath, path.join(path.relative(tmpDir, root) || ".", file));
|
|
1102
|
+
}
|
|
1103
|
+
for (const directory of dirs) {
|
|
1104
|
+
const dirPath = path.join(root, directory);
|
|
1105
|
+
if (await isDirectory$1(dirPath)) await addToZip(entries, dirPath, directory);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
} finally {
|
|
1109
|
+
await rm(tmpDir, {
|
|
1110
|
+
recursive: true,
|
|
1111
|
+
force: true
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
return zipSync(Object.fromEntries(entries));
|
|
1115
|
+
}
|
|
1116
|
+
// src/outputs/socket-io.ts
|
|
1117
|
+
const KEEP_ALIVE_INTERVAL_MS = 2 ** 30;
|
|
1118
|
+
/** Python's `TimeoutError: timed out` for socket operations. */
|
|
1119
|
+
var SocketTimeoutError = class extends Error {
|
|
1120
|
+
name = "TimeoutError";
|
|
1121
|
+
constructor() {
|
|
1122
|
+
super("timed out");
|
|
1123
|
+
}
|
|
1124
|
+
};
|
|
1125
|
+
/**
|
|
1126
|
+
* Runs send tasks one after another. Sockets are unref'd, like Python's,
|
|
1127
|
+
* which never hold the interpreter open; while sends are queued a timer
|
|
1128
|
+
* keeps the event loop alive so they are not cut short.
|
|
1129
|
+
*/
|
|
1130
|
+
var SendQueue = class {
|
|
1131
|
+
#tail = Promise.resolve();
|
|
1132
|
+
#queued = 0;
|
|
1133
|
+
#keepAlive = null;
|
|
1134
|
+
/** Queue `task`; its error goes to `onError` (never rejects the queue). */
|
|
1135
|
+
push(task, onError) {
|
|
1136
|
+
this.#queued += 1;
|
|
1137
|
+
this.#keepAlive ??= setInterval(() => void 0, KEEP_ALIVE_INTERVAL_MS);
|
|
1138
|
+
this.#tail = this.#tail.then(() => this.#run(task, onError));
|
|
1139
|
+
}
|
|
1140
|
+
async #run(task, onError) {
|
|
1141
|
+
try {
|
|
1142
|
+
await task();
|
|
1143
|
+
} catch (error) {
|
|
1144
|
+
onError(error);
|
|
1145
|
+
} finally {
|
|
1146
|
+
this.#queued -= 1;
|
|
1147
|
+
if (this.#queued === 0 && this.#keepAlive !== null) {
|
|
1148
|
+
clearInterval(this.#keepAlive);
|
|
1149
|
+
this.#keepAlive = null;
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
/** Resolves once every queued send has finished. */
|
|
1154
|
+
async drain() {
|
|
1155
|
+
await this.#tail;
|
|
1156
|
+
}
|
|
1157
|
+
};
|
|
1158
|
+
/**
|
|
1159
|
+
* Messages a stream handler could not deliver. `emit()` can't report them
|
|
1160
|
+
* and `SendQueue` runs on, so the client's save method asks here after
|
|
1161
|
+
* draining and fails, which keeps the batch from counting as saved.
|
|
1162
|
+
*/
|
|
1163
|
+
var SendFailures = class {
|
|
1164
|
+
#count = 0;
|
|
1165
|
+
#first = null;
|
|
1166
|
+
record(error) {
|
|
1167
|
+
if (this.#count === 0) this.#first = error;
|
|
1168
|
+
this.#count += 1;
|
|
1169
|
+
}
|
|
1170
|
+
/** The failures since the last call, or `null` when there were none. */
|
|
1171
|
+
take() {
|
|
1172
|
+
if (this.#count === 0) return null;
|
|
1173
|
+
const failures = {
|
|
1174
|
+
count: this.#count,
|
|
1175
|
+
error: this.#first
|
|
1176
|
+
};
|
|
1177
|
+
this.#count = 0;
|
|
1178
|
+
this.#first = null;
|
|
1179
|
+
return failures;
|
|
1180
|
+
}
|
|
1181
|
+
};
|
|
1182
|
+
/** Run `start` and settle with it, or reject after `timeoutMs`. */
|
|
1183
|
+
async function withTimeout(timeoutMs, start, onTimeout = () => void 0) {
|
|
1184
|
+
return await new Promise((resolve, reject) => {
|
|
1185
|
+
const timer = timeoutMs === null ? null : setTimeout(() => {
|
|
1186
|
+
onTimeout();
|
|
1187
|
+
reject(new SocketTimeoutError());
|
|
1188
|
+
}, timeoutMs);
|
|
1189
|
+
const done = () => {
|
|
1190
|
+
if (timer !== null) clearTimeout(timer);
|
|
1191
|
+
};
|
|
1192
|
+
start((value) => {
|
|
1193
|
+
done();
|
|
1194
|
+
resolve(value);
|
|
1195
|
+
}, (error) => {
|
|
1196
|
+
done();
|
|
1197
|
+
reject(error);
|
|
1198
|
+
});
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
/** Connect a TCP socket, failing after `timeoutMs`. */
|
|
1202
|
+
async function connectTcp(options, timeoutMs) {
|
|
1203
|
+
const socket = connect(options);
|
|
1204
|
+
try {
|
|
1205
|
+
await withTimeout(timeoutMs, (resolve, reject) => {
|
|
1206
|
+
socket.once("connect", () => resolve());
|
|
1207
|
+
socket.once("error", reject);
|
|
1208
|
+
}, () => socket.destroy());
|
|
1209
|
+
} catch (error) {
|
|
1210
|
+
socket.destroy();
|
|
1211
|
+
throw error;
|
|
1212
|
+
}
|
|
1213
|
+
socket.removeAllListeners("error");
|
|
1214
|
+
socket.on("error", () => void 0);
|
|
1215
|
+
return socket;
|
|
1216
|
+
}
|
|
1217
|
+
/** Start TLS (over `options.socket` or a new connection), with a timeout. */
|
|
1218
|
+
async function connectTls(options, timeoutMs) {
|
|
1219
|
+
const socket = connect$1(options);
|
|
1220
|
+
try {
|
|
1221
|
+
await withTimeout(timeoutMs, (resolve, reject) => {
|
|
1222
|
+
socket.once("secureConnect", () => resolve());
|
|
1223
|
+
socket.once("error", reject);
|
|
1224
|
+
}, () => socket.destroy());
|
|
1225
|
+
} catch (error) {
|
|
1226
|
+
socket.destroy();
|
|
1227
|
+
throw error;
|
|
1228
|
+
}
|
|
1229
|
+
socket.removeAllListeners("error");
|
|
1230
|
+
socket.on("error", () => void 0);
|
|
1231
|
+
return socket;
|
|
1232
|
+
}
|
|
1233
|
+
/** Python's `sock.sendall(data)`, with the socket's timeout. */
|
|
1234
|
+
async function sendAll(socket, data, timeoutMs) {
|
|
1235
|
+
await withTimeout(timeoutMs, (resolve, reject) => {
|
|
1236
|
+
if (socket.destroyed || !socket.writable) {
|
|
1237
|
+
reject(Object.assign(/* @__PURE__ */ new Error("Broken pipe"), { code: "EPIPE" }));
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
socket.write(data, (error) => {
|
|
1241
|
+
if (error) reject(error);
|
|
1242
|
+
else resolve();
|
|
1243
|
+
});
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
/** Python's `sock.sendto(data, (host, port))`. */
|
|
1247
|
+
async function sendTo(socket, data, port, host) {
|
|
1248
|
+
await new Promise((resolve, reject) => {
|
|
1249
|
+
socket.send(data, port, host, (error) => {
|
|
1250
|
+
if (error) reject(error);
|
|
1251
|
+
else resolve();
|
|
1252
|
+
});
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
/** Close a stream socket after its pending writes, without waiting forever. */
|
|
1256
|
+
async function closeStream(socket, timeoutMs) {
|
|
1257
|
+
if (socket.destroyed) return;
|
|
1258
|
+
await new Promise((resolve) => {
|
|
1259
|
+
const timer = setTimeout(() => {
|
|
1260
|
+
socket.destroy();
|
|
1261
|
+
}, timeoutMs);
|
|
1262
|
+
timer.unref();
|
|
1263
|
+
socket.once("close", () => {
|
|
1264
|
+
clearTimeout(timer);
|
|
1265
|
+
resolve();
|
|
1266
|
+
});
|
|
1267
|
+
socket.end(() => socket.destroy());
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
/** Close a datagram socket. */
|
|
1271
|
+
async function closeDatagram(socket) {
|
|
1272
|
+
await new Promise((resolve) => {
|
|
1273
|
+
try {
|
|
1274
|
+
socket.close(() => resolve());
|
|
1275
|
+
} catch {
|
|
1276
|
+
resolve();
|
|
1277
|
+
}
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
// src/outputs/gelf.ts
|
|
1281
|
+
const MS_PER_SECOND$2 = 1e3;
|
|
1282
|
+
/** How long `close()` waits for queued bytes to flush. */
|
|
1283
|
+
const CLOSE_TIMEOUT_MS$1 = 5e3;
|
|
1284
|
+
/** pygelf's `LEVELS`: logging level → syslog severity. */
|
|
1285
|
+
const LEVELS = /* @__PURE__ */ new Map([
|
|
1286
|
+
[10, 7],
|
|
1287
|
+
[20, 6],
|
|
1288
|
+
[30, 4],
|
|
1289
|
+
[40, 3],
|
|
1290
|
+
[50, 2]
|
|
1291
|
+
]);
|
|
1292
|
+
/**
|
|
1293
|
+
* pygelf's `SKIP_LIST`: LogRecord attributes never sent as extra fields
|
|
1294
|
+
* (plus `id`, which GELF forbids).
|
|
1295
|
+
*/
|
|
1296
|
+
const SKIP_LIST = /* @__PURE__ */ new Set([
|
|
1297
|
+
"args",
|
|
1298
|
+
"asctime",
|
|
1299
|
+
"created",
|
|
1300
|
+
"exc_info",
|
|
1301
|
+
"exc_text",
|
|
1302
|
+
"filename",
|
|
1303
|
+
"funcName",
|
|
1304
|
+
"id",
|
|
1305
|
+
"levelname",
|
|
1306
|
+
"levelno",
|
|
1307
|
+
"lineno",
|
|
1308
|
+
"module",
|
|
1309
|
+
"msecs",
|
|
1310
|
+
"message",
|
|
1311
|
+
"msg",
|
|
1312
|
+
"name",
|
|
1313
|
+
"pathname",
|
|
1314
|
+
"process",
|
|
1315
|
+
"processName",
|
|
1316
|
+
"relativeCreated",
|
|
1317
|
+
"thread",
|
|
1318
|
+
"threadName"
|
|
1319
|
+
]);
|
|
1320
|
+
/**
|
|
1321
|
+
* LogRecord attributes of Python 3.12+ that pygelf's `SKIP_LIST` misses, so
|
|
1322
|
+
* they always reach the packet as `_stack_info` / `_taskName` (both `None`
|
|
1323
|
+
* for parsedmarc's records).
|
|
1324
|
+
*/
|
|
1325
|
+
const UNSKIPPED_RECORD_ATTRIBUTES = ["stack_info", "taskName"];
|
|
1326
|
+
/** pygelf's `object_to_json`: datetimes as ISO 8601, anything else `str()`. */
|
|
1327
|
+
function objectToJson(value) {
|
|
1328
|
+
if (value instanceof PyDateTime) return value.isoformat();
|
|
1329
|
+
if (value instanceof Date) return value.toISOString();
|
|
1330
|
+
return String(value);
|
|
1331
|
+
}
|
|
1332
|
+
function tracebackText(error) {
|
|
1333
|
+
return error instanceof Error ? error.stack ?? errorText(error) : String(error);
|
|
1334
|
+
}
|
|
1335
|
+
/** pygelf's `gelf.make()` with `debug=False` and no static fields. */
|
|
1336
|
+
function makeGelf(record, domain, version, includeExtraFields) {
|
|
1337
|
+
const level = LEVELS.get(record.level);
|
|
1338
|
+
if (level === void 0) throw new KeyError(record.level);
|
|
1339
|
+
const gelf = {
|
|
1340
|
+
version,
|
|
1341
|
+
short_message: record.message,
|
|
1342
|
+
timestamp: record.created / MS_PER_SECOND$2,
|
|
1343
|
+
level,
|
|
1344
|
+
host: domain
|
|
1345
|
+
};
|
|
1346
|
+
if (record.error !== void 0) gelf.full_message = tracebackText(record.error);
|
|
1347
|
+
if (includeExtraFields) {
|
|
1348
|
+
for (const key of UNSKIPPED_RECORD_ATTRIBUTES) gelf[`_${key}`] = null;
|
|
1349
|
+
for (const [key, value] of Object.entries(record.extra)) if (!(SKIP_LIST.has(key) || key.startsWith("_"))) gelf[`_${key}`] = value;
|
|
1350
|
+
}
|
|
1351
|
+
return gelf;
|
|
1352
|
+
}
|
|
1353
|
+
/** pygelf's `gelf.pack()`: compact JSON, UTF-8, zlib when compressing. */
|
|
1354
|
+
function packGelf(gelf, compress) {
|
|
1355
|
+
const packed = Buffer.from(pyJsonDumps(gelf, {
|
|
1356
|
+
separators: [",", ":"],
|
|
1357
|
+
default: objectToJson
|
|
1358
|
+
}), "utf-8");
|
|
1359
|
+
return compress ? deflateSync(packed) : packed;
|
|
1360
|
+
}
|
|
1361
|
+
const CHUNK_MAGIC = [30, 15];
|
|
1362
|
+
const MESSAGE_ID_LENGTH = 8;
|
|
1363
|
+
const MAX_CHUNKS = 127;
|
|
1364
|
+
/**
|
|
1365
|
+
* pygelf's `gelf.split()`: GELF chunks of `chunkSize` bytes, each with the
|
|
1366
|
+
* magic bytes, a random 8-byte message id, the sequence number and count.
|
|
1367
|
+
*/
|
|
1368
|
+
function splitGelf(gelf, chunkSize) {
|
|
1369
|
+
const messageId = randomBytes(MESSAGE_ID_LENGTH);
|
|
1370
|
+
const count = Math.ceil(gelf.length / chunkSize);
|
|
1371
|
+
if (count > MAX_CHUNKS) throw new Error(`'b' format requires -128 <= number <= ${MAX_CHUNKS}`);
|
|
1372
|
+
const chunks = [];
|
|
1373
|
+
for (let index = 0; index < count; index += 1) {
|
|
1374
|
+
const body = gelf.subarray(index * chunkSize, (index + 1) * chunkSize);
|
|
1375
|
+
chunks.push(Buffer.concat([
|
|
1376
|
+
Buffer.from(CHUNK_MAGIC),
|
|
1377
|
+
messageId,
|
|
1378
|
+
Buffer.from([index, count]),
|
|
1379
|
+
body
|
|
1380
|
+
]));
|
|
1381
|
+
}
|
|
1382
|
+
return chunks;
|
|
1383
|
+
}
|
|
1384
|
+
/** pygelf's `BaseHandler`: turns records into GELF. */
|
|
1385
|
+
var BaseGelfHandler = class extends Handler {
|
|
1386
|
+
host;
|
|
1387
|
+
port;
|
|
1388
|
+
version;
|
|
1389
|
+
includeExtraFields;
|
|
1390
|
+
compress;
|
|
1391
|
+
/** The `host` field: this machine's hostname. */
|
|
1392
|
+
domain = hostname();
|
|
1393
|
+
queue = new SendQueue();
|
|
1394
|
+
failures = new SendFailures();
|
|
1395
|
+
constructor(host, port, options, compressDefault) {
|
|
1396
|
+
super();
|
|
1397
|
+
this.host = host;
|
|
1398
|
+
this.port = port;
|
|
1399
|
+
this.version = options.version ?? "1.1";
|
|
1400
|
+
this.includeExtraFields = options.includeExtraFields ?? false;
|
|
1401
|
+
this.compress = options.compress ?? compressDefault;
|
|
1402
|
+
}
|
|
1403
|
+
convertRecordToGelf(record) {
|
|
1404
|
+
return packGelf(makeGelf(record, this.domain, this.version, this.includeExtraFields), this.compress);
|
|
1405
|
+
}
|
|
1406
|
+
/** The bytes sent for one record. */
|
|
1407
|
+
makePickle(record) {
|
|
1408
|
+
return this.convertRecordToGelf(record);
|
|
1409
|
+
}
|
|
1410
|
+
emit(record) {
|
|
1411
|
+
const data = this.makePickle(record);
|
|
1412
|
+
this.queue.push(() => this.send(data), (error) => this.handleError(record, error));
|
|
1413
|
+
}
|
|
1414
|
+
/** Resolves once every record emitted so far has been sent (or dropped). */
|
|
1415
|
+
async drain() {
|
|
1416
|
+
await this.queue.drain();
|
|
1417
|
+
}
|
|
1418
|
+
/** The messages dropped since the last call (always none over UDP). */
|
|
1419
|
+
takeFailures() {
|
|
1420
|
+
return this.failures.take();
|
|
1421
|
+
}
|
|
1422
|
+
close() {
|
|
1423
|
+
this.closeAsync();
|
|
1424
|
+
}
|
|
1425
|
+
};
|
|
1426
|
+
/** `SocketHandler.makeSocket()`'s timeout, in seconds. */
|
|
1427
|
+
const SOCKET_TIMEOUT = 1;
|
|
1428
|
+
/**
|
|
1429
|
+
* pygelf's `GelfTcpHandler` over Python's `SocketHandler`: connects on the
|
|
1430
|
+
* first record, sends each message NUL-terminated, and on a connection or
|
|
1431
|
+
* send error closes the socket and retries later with exponential back-off
|
|
1432
|
+
* (1 s doubling up to 30 s). Unlike Python, a message that fails on a stale
|
|
1433
|
+
* connection is sent again once over a new one, and one that is still lost
|
|
1434
|
+
* is recorded in `failures` (`SocketHandler` drops it without a trace).
|
|
1435
|
+
*/
|
|
1436
|
+
var GelfTcpHandler = class extends BaseGelfHandler {
|
|
1437
|
+
sock = null;
|
|
1438
|
+
/** Epoch ms before which no reconnect is attempted. */
|
|
1439
|
+
retryTime = null;
|
|
1440
|
+
retryPeriod = 1;
|
|
1441
|
+
#connectError = null;
|
|
1442
|
+
constructor(host, port, options = {}) {
|
|
1443
|
+
super(host, port, options, false);
|
|
1444
|
+
}
|
|
1445
|
+
/** NUL-terminated: "if you send the message over tcp, it should always be null terminated". */
|
|
1446
|
+
makePickle(record) {
|
|
1447
|
+
return Buffer.concat([this.convertRecordToGelf(record), Buffer.from([0])]);
|
|
1448
|
+
}
|
|
1449
|
+
/** Open the connection (`SocketHandler.makeSocket`). */
|
|
1450
|
+
async makeSocket(timeout = SOCKET_TIMEOUT) {
|
|
1451
|
+
return await connectTcp({
|
|
1452
|
+
host: this.host,
|
|
1453
|
+
port: this.port
|
|
1454
|
+
}, timeout * MS_PER_SECOND$2);
|
|
1455
|
+
}
|
|
1456
|
+
/** `SocketHandler.createSocket()`: connect unless backing off. */
|
|
1457
|
+
async createSocket() {
|
|
1458
|
+
const now = Date.now();
|
|
1459
|
+
if (this.retryTime !== null && now < this.retryTime) return;
|
|
1460
|
+
try {
|
|
1461
|
+
this.sock = await this.makeSocket();
|
|
1462
|
+
this.sock.unref();
|
|
1463
|
+
this.retryTime = null;
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
this.#connectError = error;
|
|
1466
|
+
if (this.retryTime === null) this.retryPeriod = 1;
|
|
1467
|
+
else this.retryPeriod = Math.min(this.retryPeriod * 2, 30);
|
|
1468
|
+
this.retryTime = now + this.retryPeriod * MS_PER_SECOND$2;
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
/** Send over the open connection, or a new one; throws if that fails. */
|
|
1472
|
+
async #sendOnce(data) {
|
|
1473
|
+
if (this.sock === null) await this.createSocket();
|
|
1474
|
+
const { sock } = this;
|
|
1475
|
+
if (sock === null) throw this.#connectError;
|
|
1476
|
+
try {
|
|
1477
|
+
await sendAll(sock, data, SOCKET_TIMEOUT * MS_PER_SECOND$2);
|
|
1478
|
+
} catch (error) {
|
|
1479
|
+
sock.destroy();
|
|
1480
|
+
this.sock = null;
|
|
1481
|
+
throw error;
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
async send(data) {
|
|
1485
|
+
try {
|
|
1486
|
+
await this.#sendOnce(data);
|
|
1487
|
+
} catch {
|
|
1488
|
+
try {
|
|
1489
|
+
await this.#sendOnce(data);
|
|
1490
|
+
} catch (error) {
|
|
1491
|
+
this.failures.record(error);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
async closeAsync() {
|
|
1496
|
+
await this.drain();
|
|
1497
|
+
const { sock } = this;
|
|
1498
|
+
this.sock = null;
|
|
1499
|
+
if (sock !== null) await closeStream(sock, CLOSE_TIMEOUT_MS$1);
|
|
1500
|
+
}
|
|
1501
|
+
};
|
|
1502
|
+
/**
|
|
1503
|
+
* pygelf's `GelfUdpHandler`: zlib-compressed datagrams (IPv4, like Python's
|
|
1504
|
+
* `DatagramHandler`), split into GELF chunks above `chunkSize` bytes.
|
|
1505
|
+
*/
|
|
1506
|
+
var GelfUdpHandler = class extends BaseGelfHandler {
|
|
1507
|
+
chunkSize;
|
|
1508
|
+
sock = null;
|
|
1509
|
+
constructor(host, port, options = {}) {
|
|
1510
|
+
super(host, port, options, true);
|
|
1511
|
+
this.chunkSize = options.chunkSize ?? 1300;
|
|
1512
|
+
}
|
|
1513
|
+
async send(data) {
|
|
1514
|
+
if (this.sock === null) {
|
|
1515
|
+
const sock = createSocket("udp4");
|
|
1516
|
+
sock.on("error", () => void 0);
|
|
1517
|
+
sock.unref();
|
|
1518
|
+
this.sock = sock;
|
|
1519
|
+
}
|
|
1520
|
+
const chunks = data.length <= this.chunkSize ? [data] : splitGelf(data, this.chunkSize);
|
|
1521
|
+
for (const chunk of chunks) await sendTo(this.sock, chunk, this.port, this.host);
|
|
1522
|
+
}
|
|
1523
|
+
async closeAsync() {
|
|
1524
|
+
await this.drain();
|
|
1525
|
+
const { sock } = this;
|
|
1526
|
+
this.sock = null;
|
|
1527
|
+
if (sock !== null) await closeDatagram(sock);
|
|
1528
|
+
}
|
|
1529
|
+
};
|
|
1530
|
+
/** pygelf's `GelfTlsHandler`: {@link GelfTcpHandler} over TLS (IPv4). */
|
|
1531
|
+
var GelfTlsHandler = class extends GelfTcpHandler {
|
|
1532
|
+
tlsOptions;
|
|
1533
|
+
constructor(host, port, options = {}) {
|
|
1534
|
+
const { validate = true, caCerts, certfile, keyfile } = options;
|
|
1535
|
+
if (keyfile && !certfile) throw new PyValueError("certfile must be specified");
|
|
1536
|
+
super(host, port, options);
|
|
1537
|
+
const tlsOptions = { rejectUnauthorized: validate };
|
|
1538
|
+
if (caCerts) tlsOptions.ca = [...getCACertificates("default"), readFileSync(caCerts, "utf-8")];
|
|
1539
|
+
if (certfile) {
|
|
1540
|
+
tlsOptions.cert = readFileSync(certfile);
|
|
1541
|
+
tlsOptions.key = readFileSync(keyfile || certfile);
|
|
1542
|
+
}
|
|
1543
|
+
this.tlsOptions = tlsOptions;
|
|
1544
|
+
}
|
|
1545
|
+
async makeSocket(timeout = SOCKET_TIMEOUT) {
|
|
1546
|
+
const timeoutMs = timeout * MS_PER_SECOND$2;
|
|
1547
|
+
const plain = await connectTcp({
|
|
1548
|
+
host: this.host,
|
|
1549
|
+
port: this.port,
|
|
1550
|
+
family: 4
|
|
1551
|
+
}, timeoutMs);
|
|
1552
|
+
plain.unref();
|
|
1553
|
+
try {
|
|
1554
|
+
return await connectTls({
|
|
1555
|
+
...this.tlsOptions,
|
|
1556
|
+
socket: plain,
|
|
1557
|
+
host: this.host,
|
|
1558
|
+
...isIP(this.host) === 0 && { servername: this.host }
|
|
1559
|
+
}, timeoutMs);
|
|
1560
|
+
} catch (error) {
|
|
1561
|
+
plain.destroy();
|
|
1562
|
+
throw error;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
};
|
|
1566
|
+
/** parsedmarc's thread-local `log_context_data`: the row being logged. */
|
|
1567
|
+
const logContextData = {};
|
|
1568
|
+
/** Copies `logContextData.parsedmarc` onto each record as `parsedmarc`. */
|
|
1569
|
+
var ContextFilter = class {
|
|
1570
|
+
filter = (record) => {
|
|
1571
|
+
if (!("parsedmarc" in logContextData)) throw new PyAttributeError("'_thread._local' object has no attribute 'parsedmarc'");
|
|
1572
|
+
record.extra.parsedmarc = logContextData.parsedmarc;
|
|
1573
|
+
return true;
|
|
1574
|
+
};
|
|
1575
|
+
};
|
|
1576
|
+
/** Raised when messages could not be delivered over TCP or TLS. */
|
|
1577
|
+
var GelfError = class extends Error {
|
|
1578
|
+
name = "GelfError";
|
|
1579
|
+
};
|
|
1580
|
+
/** A client for the Graylog Extended Log Format. */
|
|
1581
|
+
var GelfClient = class {
|
|
1582
|
+
host;
|
|
1583
|
+
port;
|
|
1584
|
+
logger;
|
|
1585
|
+
gelfMode;
|
|
1586
|
+
/** The GELF handler attached to {@link logger}. */
|
|
1587
|
+
handler;
|
|
1588
|
+
/**
|
|
1589
|
+
* @param host - The GELF host
|
|
1590
|
+
* @param port - The GELF port
|
|
1591
|
+
* @param mode - The GELF transport mode: `udp`, `tcp` or `tls`
|
|
1592
|
+
*/
|
|
1593
|
+
constructor(host, port, mode, options = {}) {
|
|
1594
|
+
this.host = host;
|
|
1595
|
+
this.port = port;
|
|
1596
|
+
this.logger = new Logger("tsdmarc_gelf");
|
|
1597
|
+
this.logger.setLevel(20);
|
|
1598
|
+
this.logger.addFilter(new ContextFilter().filter);
|
|
1599
|
+
this.gelfMode = {
|
|
1600
|
+
udp: (h, p) => new GelfUdpHandler(h, p, { includeExtraFields: true }),
|
|
1601
|
+
tcp: (h, p) => new GelfTcpHandler(h, p, { includeExtraFields: true }),
|
|
1602
|
+
tls: (h, p) => new GelfTlsHandler(h, p, {
|
|
1603
|
+
...options.tls,
|
|
1604
|
+
includeExtraFields: true
|
|
1605
|
+
})
|
|
1606
|
+
};
|
|
1607
|
+
if (!Object.hasOwn(this.gelfMode, mode)) throw new KeyError(mode);
|
|
1608
|
+
this.handler = this.gelfMode[mode](this.host, this.port);
|
|
1609
|
+
this.logger.addHandler(this.handler);
|
|
1610
|
+
}
|
|
1611
|
+
async saveAggregateReportToGelf(aggregateReports) {
|
|
1612
|
+
const rows = parsedAggregateReportsToCsvRows(aggregateReports);
|
|
1613
|
+
for (const row of rows) {
|
|
1614
|
+
logContextData.parsedmarc = row;
|
|
1615
|
+
this.logger.info("parsedmarc aggregate report");
|
|
1616
|
+
}
|
|
1617
|
+
logContextData.parsedmarc = null;
|
|
1618
|
+
await this.#drain();
|
|
1619
|
+
}
|
|
1620
|
+
async saveFailureReportToGelf(failureReports) {
|
|
1621
|
+
const rows = parsedFailureReportsToCsvRows(failureReports);
|
|
1622
|
+
for (const row of rows) {
|
|
1623
|
+
logContextData.parsedmarc = row;
|
|
1624
|
+
this.logger.info("parsedmarc failure report");
|
|
1625
|
+
}
|
|
1626
|
+
await this.#drain();
|
|
1627
|
+
}
|
|
1628
|
+
/** Backward-compatible alias of {@link saveFailureReportToGelf}. */
|
|
1629
|
+
async saveForensicReportToGelf(failureReports) {
|
|
1630
|
+
await this.saveFailureReportToGelf(failureReports);
|
|
1631
|
+
}
|
|
1632
|
+
async saveSmtpTlsReportToGelf(smtpTlsReports) {
|
|
1633
|
+
const rows = parsedSmtpTlsReportsToCsvRows(smtpTlsReports);
|
|
1634
|
+
for (const row of rows) {
|
|
1635
|
+
logContextData.parsedmarc = row;
|
|
1636
|
+
this.logger.info("parsedmarc smtptls report");
|
|
1637
|
+
}
|
|
1638
|
+
await this.#drain();
|
|
1639
|
+
}
|
|
1640
|
+
/**
|
|
1641
|
+
* Wait until the messages logged so far have been sent.
|
|
1642
|
+
*
|
|
1643
|
+
* @throws {GelfError} When some could not be delivered over TCP or TLS.
|
|
1644
|
+
*/
|
|
1645
|
+
async #drain() {
|
|
1646
|
+
if (this.handler instanceof BaseGelfHandler) {
|
|
1647
|
+
await this.handler.drain();
|
|
1648
|
+
const failures = this.handler.takeFailures();
|
|
1649
|
+
if (failures !== null) throw new GelfError(`${failures.count} message(s) not sent to ${this.host}:${this.port}: ${errorText(failures.error)}`, { cause: failures.error });
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
/** Remove and close the GELF handler, releasing its connection. */
|
|
1653
|
+
async close() {
|
|
1654
|
+
this.logger.removeHandler(this.handler);
|
|
1655
|
+
if (this.handler instanceof BaseGelfHandler) await this.handler.closeAsync();
|
|
1656
|
+
else this.handler.close();
|
|
1657
|
+
}
|
|
1658
|
+
};
|
|
1659
|
+
// src/outputs/http.ts
|
|
1660
|
+
/**
|
|
1661
|
+
* The largest response body read (Splunk's HEC answers with a few bytes of
|
|
1662
|
+
* JSON). httpx reads any size; a hostile or broken server could otherwise
|
|
1663
|
+
* make the client buffer without limit.
|
|
1664
|
+
*/
|
|
1665
|
+
const MAX_RESPONSE_BYTES = 1048576;
|
|
1666
|
+
/** httpx's default `max_redirects`. */
|
|
1667
|
+
const MAX_REDIRECTS = 20;
|
|
1668
|
+
const MS_PER_SECOND$1 = 1e3;
|
|
1669
|
+
const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/iu;
|
|
1670
|
+
const REDIRECT_STATUSES = /* @__PURE__ */ new Set([
|
|
1671
|
+
301,
|
|
1672
|
+
302,
|
|
1673
|
+
303,
|
|
1674
|
+
307,
|
|
1675
|
+
308
|
|
1676
|
+
]);
|
|
1677
|
+
/** `scheme://`, then `user:password@`, the host, and the rest of the URL. */
|
|
1678
|
+
/** What `fetch` refuses in a header value (it quotes the value in its error). */
|
|
1679
|
+
const INVALID_HEADER_VALUE = /[\0\n\r\u0100-\u{10ffff}]/u;
|
|
1680
|
+
const URL_IN_TEXT = /\b([a-z][a-z0-9+.-]*:\/\/)(?:[^\s/@"'<>)]*@)?([^\s/?#"'<>)]*)[^\s"'<>)]*/giu;
|
|
1681
|
+
function bodyTooLarge() {
|
|
1682
|
+
return /* @__PURE__ */ new Error(`Response body exceeds ${MAX_RESPONSE_BYTES} bytes`);
|
|
1683
|
+
}
|
|
1684
|
+
/** Read a fetch body up to {@link MAX_RESPONSE_BYTES}. */
|
|
1685
|
+
async function readFetchBody(response) {
|
|
1686
|
+
const declared = Number(response.headers.get("content-length"));
|
|
1687
|
+
if (Number.isFinite(declared) && declared > 1048576) {
|
|
1688
|
+
await response.body?.cancel();
|
|
1689
|
+
throw bodyTooLarge();
|
|
1690
|
+
}
|
|
1691
|
+
if (response.body === null) return "";
|
|
1692
|
+
const reader = response.body.getReader();
|
|
1693
|
+
const chunks = [];
|
|
1694
|
+
let total = 0;
|
|
1695
|
+
for (;;) {
|
|
1696
|
+
const { done, value } = await reader.read();
|
|
1697
|
+
if (done) break;
|
|
1698
|
+
total += value.length;
|
|
1699
|
+
if (total > 1048576) {
|
|
1700
|
+
await reader.cancel();
|
|
1701
|
+
throw bodyTooLarge();
|
|
1702
|
+
}
|
|
1703
|
+
chunks.push(value);
|
|
1704
|
+
}
|
|
1705
|
+
return Buffer.concat(chunks, total).toString("utf8");
|
|
1706
|
+
}
|
|
1707
|
+
function fetchResponse(response) {
|
|
1708
|
+
return {
|
|
1709
|
+
status: response.status,
|
|
1710
|
+
text: () => readFetchBody(response),
|
|
1711
|
+
discard: async () => {
|
|
1712
|
+
try {
|
|
1713
|
+
await response.body?.cancel();
|
|
1714
|
+
} catch {}
|
|
1715
|
+
}
|
|
1716
|
+
};
|
|
1717
|
+
}
|
|
1718
|
+
function requestSignal(options) {
|
|
1719
|
+
const signals = [];
|
|
1720
|
+
if (options.timeout !== null) signals.push(AbortSignal.timeout(options.timeout * MS_PER_SECOND$1));
|
|
1721
|
+
if (options.signal) signals.push(options.signal);
|
|
1722
|
+
if (signals.length === 0) return;
|
|
1723
|
+
return signals.length === 1 ? signals[0] : AbortSignal.any(signals);
|
|
1724
|
+
}
|
|
1725
|
+
/**
|
|
1726
|
+
* The URL to request, like httpx refusing what it can't send (no URL
|
|
1727
|
+
* quoted). An unparsable URL with a scheme throws {@link InvalidURL}: a
|
|
1728
|
+
* password with an unencoded `/`, `?` or `#` ends the authority early, and
|
|
1729
|
+
* httpx's `InvalidURL` then quotes part of it as the port.
|
|
1730
|
+
*/
|
|
1731
|
+
function parseRequestUrl(url) {
|
|
1732
|
+
if (!URL.canParse(url)) {
|
|
1733
|
+
if (URL_SCHEME.test(url)) throw new InvalidURL("Invalid URL");
|
|
1734
|
+
throw new Error("Request URL is missing an 'http://' or 'https://' protocol.");
|
|
1735
|
+
}
|
|
1736
|
+
const target = new URL(url);
|
|
1737
|
+
if (target.protocol !== "http:" && target.protocol !== "https:") throw new Error(`Request URL has an unsupported protocol '${target.protocol}//'.`);
|
|
1738
|
+
return target;
|
|
1739
|
+
}
|
|
1740
|
+
/** httpx's `unquote()`: invalid percent-encoding stays as it is. */
|
|
1741
|
+
function unquote(text) {
|
|
1742
|
+
try {
|
|
1743
|
+
return decodeURIComponent(text);
|
|
1744
|
+
} catch {
|
|
1745
|
+
return text;
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
/** Basic auth from a URL's userinfo; it replaces any `Authorization`, like httpx. */
|
|
1749
|
+
function withUrlCredentials(target, headers) {
|
|
1750
|
+
const credentials = `${unquote(target.username)}:${unquote(target.password)}`;
|
|
1751
|
+
return {
|
|
1752
|
+
...withoutHeader(headers, ["Authorization"]),
|
|
1753
|
+
Authorization: `Basic ${Buffer.from(credentials).toString("base64")}`
|
|
1754
|
+
};
|
|
1755
|
+
}
|
|
1756
|
+
/** `text` with each URL cut down to its scheme and host. */
|
|
1757
|
+
function redactUrls(text) {
|
|
1758
|
+
return text.replaceAll(URL_IN_TEXT, "$1$2");
|
|
1759
|
+
}
|
|
1760
|
+
/** `error`, or a copy of it (and its causes) without the URLs it quotes. */
|
|
1761
|
+
function withoutUrlSecrets(error) {
|
|
1762
|
+
if (!(error instanceof Error)) return error;
|
|
1763
|
+
const { cause } = error;
|
|
1764
|
+
const safeCause = withoutUrlSecrets(cause);
|
|
1765
|
+
const message = redactUrls(error.message);
|
|
1766
|
+
if (message === error.message && safeCause === cause) return error;
|
|
1767
|
+
const safe = new Error(message, { cause: safeCause });
|
|
1768
|
+
safe.name = error.name;
|
|
1769
|
+
return safe;
|
|
1770
|
+
}
|
|
1771
|
+
/** POST `body` to `url`, following redirects unless `followRedirects` is `false`. */
|
|
1772
|
+
async function httpPost(url, options) {
|
|
1773
|
+
const target = parseRequestUrl(url);
|
|
1774
|
+
let request = options;
|
|
1775
|
+
let requestUrl = url;
|
|
1776
|
+
if (target.username || target.password) {
|
|
1777
|
+
request = {
|
|
1778
|
+
...options,
|
|
1779
|
+
headers: withUrlCredentials(target, options.headers)
|
|
1780
|
+
};
|
|
1781
|
+
target.username = "";
|
|
1782
|
+
target.password = "";
|
|
1783
|
+
requestUrl = target.href;
|
|
1784
|
+
}
|
|
1785
|
+
for (const [name, value] of Object.entries(request.headers)) if (INVALID_HEADER_VALUE.test(value)) throw new Error(`Invalid value for the ${name} header`);
|
|
1786
|
+
const signal = requestSignal(request);
|
|
1787
|
+
try {
|
|
1788
|
+
if (request.verify === false && target.protocol === "https:") return await insecurePost(requestUrl, request, signal);
|
|
1789
|
+
return fetchResponse(await fetch(requestUrl, {
|
|
1790
|
+
method: "POST",
|
|
1791
|
+
headers: request.headers,
|
|
1792
|
+
body: request.body,
|
|
1793
|
+
redirect: request.followRedirects === false ? "manual" : "follow",
|
|
1794
|
+
signal
|
|
1795
|
+
}));
|
|
1796
|
+
} catch (error) {
|
|
1797
|
+
throw withoutUrlSecrets(error);
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
/** Read a node:http body up to {@link MAX_RESPONSE_BYTES}. */
|
|
1801
|
+
function readNodeBody(response) {
|
|
1802
|
+
return new Promise((resolve, reject) => {
|
|
1803
|
+
const declared = Number(response.headers["content-length"]);
|
|
1804
|
+
if (Number.isFinite(declared) && declared > 1048576) {
|
|
1805
|
+
response.destroy();
|
|
1806
|
+
reject(bodyTooLarge());
|
|
1807
|
+
return;
|
|
1808
|
+
}
|
|
1809
|
+
const chunks = [];
|
|
1810
|
+
let total = 0;
|
|
1811
|
+
response.on("data", (chunk) => {
|
|
1812
|
+
total += chunk.length;
|
|
1813
|
+
if (total > 1048576) {
|
|
1814
|
+
response.destroy();
|
|
1815
|
+
reject(bodyTooLarge());
|
|
1816
|
+
return;
|
|
1817
|
+
}
|
|
1818
|
+
chunks.push(chunk);
|
|
1819
|
+
});
|
|
1820
|
+
response.on("end", () => resolve(Buffer.concat(chunks, total).toString("utf8")));
|
|
1821
|
+
response.on("error", reject);
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
function nodeRequest(url, method, headers, body, signal) {
|
|
1825
|
+
return new Promise((resolve, reject) => {
|
|
1826
|
+
const outgoing = (url.protocol === "https:" ? request$1 : request)(url, {
|
|
1827
|
+
method,
|
|
1828
|
+
headers,
|
|
1829
|
+
rejectUnauthorized: false,
|
|
1830
|
+
signal
|
|
1831
|
+
}, resolve);
|
|
1832
|
+
outgoing.on("error", reject);
|
|
1833
|
+
outgoing.end(body);
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
function encodeBody(body) {
|
|
1837
|
+
if (body instanceof Uint8Array) return body;
|
|
1838
|
+
return new TextEncoder().encode(body.toString());
|
|
1839
|
+
}
|
|
1840
|
+
function withoutHeader(headers, names) {
|
|
1841
|
+
const lowered = new Set(names.map((name) => name.toLowerCase()));
|
|
1842
|
+
return Object.fromEntries(Object.entries(headers).filter(([key]) => !lowered.has(key.toLowerCase())));
|
|
1843
|
+
}
|
|
1844
|
+
async function insecurePost(url, options, signal) {
|
|
1845
|
+
let target = new URL(url);
|
|
1846
|
+
let method = "POST";
|
|
1847
|
+
let body = encodeBody(options.body);
|
|
1848
|
+
let headers = { ...options.headers };
|
|
1849
|
+
if (options.body instanceof URLSearchParams && !hasHeader(headers)) headers["Content-Type"] = "application/x-www-form-urlencoded";
|
|
1850
|
+
for (let redirects = 0;; redirects += 1) {
|
|
1851
|
+
const response = await nodeRequest(target, method, headers, body, signal);
|
|
1852
|
+
const status = response.statusCode ?? 0;
|
|
1853
|
+
const { location } = response.headers;
|
|
1854
|
+
if (options.followRedirects === false || !REDIRECT_STATUSES.has(status) || location === void 0) return {
|
|
1855
|
+
status,
|
|
1856
|
+
text: () => readNodeBody(response),
|
|
1857
|
+
discard: () => {
|
|
1858
|
+
response.destroy();
|
|
1859
|
+
return Promise.resolve();
|
|
1860
|
+
}
|
|
1861
|
+
};
|
|
1862
|
+
response.destroy();
|
|
1863
|
+
if (redirects >= MAX_REDIRECTS) throw new Error("Exceeded maximum allowed redirects.");
|
|
1864
|
+
const next = new URL(location, target);
|
|
1865
|
+
if (status !== 307 && status !== 308) {
|
|
1866
|
+
method = "GET";
|
|
1867
|
+
body = void 0;
|
|
1868
|
+
headers = withoutHeader(headers, ["Content-Type", "Content-Length"]);
|
|
1869
|
+
}
|
|
1870
|
+
if (next.origin !== target.origin) headers = withoutHeader(headers, ["Authorization"]);
|
|
1871
|
+
target = next;
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
function hasHeader(headers) {
|
|
1875
|
+
return Object.keys(headers).some((key) => key.toLowerCase() === "content-type");
|
|
1876
|
+
}
|
|
1877
|
+
/** The message of an error, with the low-level cause `fetch` wraps. */
|
|
1878
|
+
function describeError(error) {
|
|
1879
|
+
if (!(error instanceof Error)) return String(error);
|
|
1880
|
+
const { cause } = error;
|
|
1881
|
+
if (cause instanceof Error && cause.message && cause.message !== error.message) return `${error.message}: ${cause.message}`;
|
|
1882
|
+
return error.message;
|
|
1883
|
+
}
|
|
1884
|
+
// src/outputs/splunk.ts
|
|
1885
|
+
const DEFAULT_TIMEOUT$2 = 60;
|
|
1886
|
+
const CLOSED_MESSAGE$2 = "Cannot send a request, as the client has been closed.";
|
|
1887
|
+
const TOKEN_PREFIX_CHARS = "Splunk ";
|
|
1888
|
+
const URL_SCHEME_AND_NETLOC = /^([A-Za-z][A-Za-z0-9+.-]*):(?:\/\/([^/?#]*))?/u;
|
|
1889
|
+
const INVALID_URL_MESSAGE$2 = "Invalid Splunk HEC URL (details withheld because they may include credentials)";
|
|
1890
|
+
/** Characters of a sender's date quoted in a warning. */
|
|
1891
|
+
const MAX_LOGGED_DATE = 100;
|
|
1892
|
+
/** Raised when a Splunk API error occurs. */
|
|
1893
|
+
var SplunkError = class extends Error {
|
|
1894
|
+
name = "SplunkError";
|
|
1895
|
+
};
|
|
1896
|
+
const HEC_JSON = { nonFiniteAsNull: true };
|
|
1897
|
+
/** Python's `str.lstrip(chars)`: drop leading characters found in `chars`. */
|
|
1898
|
+
function lstripChars(value, chars) {
|
|
1899
|
+
let start = 0;
|
|
1900
|
+
while (start < value.length && chars.includes(value.charAt(start))) start += 1;
|
|
1901
|
+
return value.slice(start);
|
|
1902
|
+
}
|
|
1903
|
+
/** `{scheme}://{netloc}` as Python's `urlparse()` splits them. */
|
|
1904
|
+
function schemeAndNetloc(url) {
|
|
1905
|
+
const match = URL_SCHEME_AND_NETLOC.exec(url);
|
|
1906
|
+
return `${match?.[1]?.toLowerCase() ?? ""}://${match?.[2] ?? ""}`;
|
|
1907
|
+
}
|
|
1908
|
+
/**
|
|
1909
|
+
* The event time of an SMTP TLS report, or `null` when its `start-datetime`
|
|
1910
|
+
* is no date: the parser keeps that value as the sender wrote it (any JSON
|
|
1911
|
+
* type), and one such report must not cost the batch its Splunk events.
|
|
1912
|
+
*/
|
|
1913
|
+
function smtpTlsEventTime(report) {
|
|
1914
|
+
try {
|
|
1915
|
+
return humanTimestampToUnixTimestamp(report.beginDate);
|
|
1916
|
+
} catch (error) {
|
|
1917
|
+
if (!(error instanceof Error)) throw error;
|
|
1918
|
+
const value = sanitizeForLog(String(report.beginDate).slice(0, MAX_LOGGED_DATE));
|
|
1919
|
+
logger.warning(`Sending SMTP TLS report to Splunk without an event time: invalid begin_date ${value}`);
|
|
1920
|
+
return null;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
function snake(value) {
|
|
1924
|
+
return toSnakeCaseKeys(value);
|
|
1925
|
+
}
|
|
1926
|
+
/** A client for a Splunk HTTP Events Collector (HEC). */
|
|
1927
|
+
var HECClient = class {
|
|
1928
|
+
url;
|
|
1929
|
+
accessToken;
|
|
1930
|
+
index;
|
|
1931
|
+
host;
|
|
1932
|
+
source;
|
|
1933
|
+
timeout;
|
|
1934
|
+
verify;
|
|
1935
|
+
/** Headers sent with every request. */
|
|
1936
|
+
headers;
|
|
1937
|
+
#commonData;
|
|
1938
|
+
#closed = new AbortController();
|
|
1939
|
+
/**
|
|
1940
|
+
* @param url - The URL of the HEC; only its scheme and host are used
|
|
1941
|
+
* @param accessToken - The HEC access token
|
|
1942
|
+
* @param index - The name of the index
|
|
1943
|
+
*/
|
|
1944
|
+
constructor(url, accessToken, index, options = {}) {
|
|
1945
|
+
this.url = `${schemeAndNetloc(url)}/services/collector/event/1.0`;
|
|
1946
|
+
this.accessToken = lstripChars(accessToken, TOKEN_PREFIX_CHARS);
|
|
1947
|
+
this.index = index;
|
|
1948
|
+
this.host = hostname();
|
|
1949
|
+
this.source = options.source ?? "tsdmarc";
|
|
1950
|
+
this.timeout = options.timeout ?? DEFAULT_TIMEOUT$2;
|
|
1951
|
+
this.verify = options.verify ?? true;
|
|
1952
|
+
this.#commonData = {
|
|
1953
|
+
host: this.host,
|
|
1954
|
+
source: this.source,
|
|
1955
|
+
index: this.index
|
|
1956
|
+
};
|
|
1957
|
+
this.headers = {
|
|
1958
|
+
"User-Agent": USER_AGENT,
|
|
1959
|
+
Authorization: `Splunk ${this.accessToken}`
|
|
1960
|
+
};
|
|
1961
|
+
}
|
|
1962
|
+
/** The `host`, `source` and `index` fields every event carries. */
|
|
1963
|
+
get commonData() {
|
|
1964
|
+
return { ...this.#commonData };
|
|
1965
|
+
}
|
|
1966
|
+
/** Saves aggregate DMARC reports to Splunk, one event per record. */
|
|
1967
|
+
async saveAggregateReportsToSplunk(aggregateReports) {
|
|
1968
|
+
logger.debug("Saving aggregate reports to Splunk");
|
|
1969
|
+
const reports = Array.isArray(aggregateReports) ? aggregateReports : [aggregateReports];
|
|
1970
|
+
if (reports.length < 1) return;
|
|
1971
|
+
const data = { ...this.#commonData };
|
|
1972
|
+
const jsonLines = [];
|
|
1973
|
+
for (const report of reports) {
|
|
1974
|
+
const metadata = snake(report.reportMetadata);
|
|
1975
|
+
const policyPublished = snake(report.policyPublished);
|
|
1976
|
+
for (const record of report.records) {
|
|
1977
|
+
const snakeRecord = snake(record);
|
|
1978
|
+
const { source, alignment, identifiers } = record;
|
|
1979
|
+
const newReport = { ...metadata };
|
|
1980
|
+
newReport.interval_begin = record.intervalBegin;
|
|
1981
|
+
newReport.interval_end = record.intervalEnd;
|
|
1982
|
+
newReport.normalized_timespan = snakeRecord.normalized_timespan;
|
|
1983
|
+
newReport.published_policy = policyPublished;
|
|
1984
|
+
newReport.source_ip_address = source.ipAddress;
|
|
1985
|
+
newReport.source_country = source.country;
|
|
1986
|
+
newReport.source_reverse_dns = source.reverseDns;
|
|
1987
|
+
newReport.source_base_domain = source.baseDomain;
|
|
1988
|
+
newReport.source_type = source.type;
|
|
1989
|
+
newReport.source_name = source.name;
|
|
1990
|
+
newReport.source_asn = source.asn;
|
|
1991
|
+
newReport.source_as_name = source.asName;
|
|
1992
|
+
newReport.source_as_domain = source.asDomain;
|
|
1993
|
+
newReport.message_count = record.count;
|
|
1994
|
+
newReport.disposition = record.policyEvaluated.disposition;
|
|
1995
|
+
newReport.spf_aligned = alignment.spf;
|
|
1996
|
+
newReport.dkim_aligned = alignment.dkim;
|
|
1997
|
+
newReport.passed_dmarc = alignment.dmarc;
|
|
1998
|
+
newReport.header_from = identifiers.headerFrom;
|
|
1999
|
+
newReport.envelope_from = identifiers.envelopeFrom;
|
|
2000
|
+
const authResults = snakeRecord.auth_results;
|
|
2001
|
+
if ("dkim" in authResults) newReport.dkim_results = authResults.dkim;
|
|
2002
|
+
if ("spf" in authResults) newReport.spf_results = authResults.spf;
|
|
2003
|
+
data.sourcetype = "dmarc:aggregate";
|
|
2004
|
+
data.time = humanTimestampToUnixTimestamp(record.intervalBegin, { assumeUtc: true });
|
|
2005
|
+
data.event = newReport;
|
|
2006
|
+
jsonLines.push(`${pyJsonDumps(data, HEC_JSON)}\n`);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
await this.#post(jsonLines);
|
|
2010
|
+
}
|
|
2011
|
+
/** Saves failure DMARC reports to Splunk, one event per report. */
|
|
2012
|
+
async saveFailureReportsToSplunk(failureReports) {
|
|
2013
|
+
logger.debug("Saving failure reports to Splunk");
|
|
2014
|
+
const reports = Array.isArray(failureReports) ? failureReports : [failureReports];
|
|
2015
|
+
if (reports.length < 1) return;
|
|
2016
|
+
const jsonLines = [];
|
|
2017
|
+
for (const report of reports) {
|
|
2018
|
+
const data = { ...this.#commonData };
|
|
2019
|
+
data.sourcetype = "dmarc:failure";
|
|
2020
|
+
data.time = humanTimestampToUnixTimestamp(report.arrivalDateUtc, { assumeUtc: true });
|
|
2021
|
+
data.event = snake(report);
|
|
2022
|
+
jsonLines.push(`${pyJsonDumps(data, HEC_JSON)}\n`);
|
|
2023
|
+
}
|
|
2024
|
+
await this.#post(jsonLines);
|
|
2025
|
+
}
|
|
2026
|
+
/** Backward-compatible alias of {@link saveFailureReportsToSplunk}. */
|
|
2027
|
+
async saveForensicReportsToSplunk(failureReports) {
|
|
2028
|
+
await this.saveFailureReportsToSplunk(failureReports);
|
|
2029
|
+
}
|
|
2030
|
+
/** Saves SMTP TLS reports to Splunk, one event per report. */
|
|
2031
|
+
async saveSmtpTlsReportsToSplunk(reports) {
|
|
2032
|
+
logger.debug("Saving SMTP TLS reports to Splunk");
|
|
2033
|
+
const reportList = Array.isArray(reports) ? reports : [reports];
|
|
2034
|
+
if (reportList.length < 1) return;
|
|
2035
|
+
const jsonLines = [];
|
|
2036
|
+
for (const report of reportList) {
|
|
2037
|
+
const data = {
|
|
2038
|
+
...this.#commonData,
|
|
2039
|
+
sourcetype: "smtp:tls"
|
|
2040
|
+
};
|
|
2041
|
+
const time = smtpTlsEventTime(report);
|
|
2042
|
+
if (time !== null) data.time = time;
|
|
2043
|
+
data.event = snake(report);
|
|
2044
|
+
jsonLines.push(`${pyJsonDumps(data, HEC_JSON)}\n`);
|
|
2045
|
+
}
|
|
2046
|
+
await this.#post(jsonLines);
|
|
2047
|
+
}
|
|
2048
|
+
async #post(jsonLines) {
|
|
2049
|
+
if (!this.verify) logger.debug("Skipping certificate verification for Splunk HEC");
|
|
2050
|
+
let response;
|
|
2051
|
+
try {
|
|
2052
|
+
if (this.#closed.signal.aborted) throw new Error(CLOSED_MESSAGE$2);
|
|
2053
|
+
const result = await httpPost(this.url, {
|
|
2054
|
+
headers: { ...this.headers },
|
|
2055
|
+
body: new TextEncoder().encode(jsonLines.join("")),
|
|
2056
|
+
timeout: this.timeout,
|
|
2057
|
+
verify: this.verify,
|
|
2058
|
+
signal: this.#closed.signal
|
|
2059
|
+
});
|
|
2060
|
+
response = JSON.parse(await result.text());
|
|
2061
|
+
} catch (error) {
|
|
2062
|
+
if (error instanceof InvalidURL) throw new SplunkError(INVALID_URL_MESSAGE$2);
|
|
2063
|
+
throw new SplunkError(describeError(error), { cause: error });
|
|
2064
|
+
}
|
|
2065
|
+
const { code, text } = response ?? {};
|
|
2066
|
+
if (code !== 0) throw new SplunkError(String(text));
|
|
2067
|
+
}
|
|
2068
|
+
/** Close the client: abort in-flight requests and refuse new ones. */
|
|
2069
|
+
close() {
|
|
2070
|
+
this.#closed.abort(/* @__PURE__ */ new Error(CLOSED_MESSAGE$2));
|
|
2071
|
+
}
|
|
2072
|
+
};
|
|
2073
|
+
// src/outputs/loki.ts
|
|
2074
|
+
const LOKI_PUSH_PATH = "/loki/api/v1/push";
|
|
2075
|
+
const LOKI_TIMESTAMP_MODES = ["ingest", "report_begin"];
|
|
2076
|
+
const LOKI_DEFAULT_LABEL_FIELDS = ["report_type", "domain"];
|
|
2077
|
+
const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/u;
|
|
2078
|
+
const DEFAULT_TIMEOUT$1 = 60;
|
|
2079
|
+
const DEFAULT_BATCH_SIZE = 1e3;
|
|
2080
|
+
const NS_PER_MS = 1000000n;
|
|
2081
|
+
const NS_PER_SECOND = 1000000000n;
|
|
2082
|
+
/** Characters of the response body quoted in a failed push's error. */
|
|
2083
|
+
const MAX_ERROR_BODY = 500;
|
|
2084
|
+
/**
|
|
2085
|
+
* Loki answers 204 when it stored the push; 200 is also accepted. Other
|
|
2086
|
+
* 2xx codes are not success: with `block_ingestion_until` set it returns a
|
|
2087
|
+
* custom 260 and discards the logs (`block_ingestion_status_code`).
|
|
2088
|
+
*/
|
|
2089
|
+
const SUCCESS_STATUSES = /* @__PURE__ */ new Set([200, 204]);
|
|
2090
|
+
const CLOSED_MESSAGE$1 = "Cannot send a request, as the client has been closed.";
|
|
2091
|
+
const INVALID_URL_MESSAGE$1 = "Invalid Loki URL (details withheld because they may include credentials)";
|
|
2092
|
+
/**
|
|
2093
|
+
* Per report type: the row field holding the domain the report is about,
|
|
2094
|
+
* the row field with its begin time, and whether that value is UTC without
|
|
2095
|
+
* an offset (aggregate rows carry the record's interval_begin, which is
|
|
2096
|
+
* naive UTC wall-clock time).
|
|
2097
|
+
*/
|
|
2098
|
+
const REPORT_TYPES = {
|
|
2099
|
+
aggregate: [
|
|
2100
|
+
"domain",
|
|
2101
|
+
"begin_date",
|
|
2102
|
+
true
|
|
2103
|
+
],
|
|
2104
|
+
failure: [
|
|
2105
|
+
"reported_domain",
|
|
2106
|
+
"arrival_date_utc",
|
|
2107
|
+
true
|
|
2108
|
+
],
|
|
2109
|
+
smtp_tls: [
|
|
2110
|
+
"policy_domain",
|
|
2111
|
+
"begin_date",
|
|
2112
|
+
false
|
|
2113
|
+
]
|
|
2114
|
+
};
|
|
2115
|
+
/** Raised when pushing to Loki fails. */
|
|
2116
|
+
var LokiError = class extends Error {
|
|
2117
|
+
name = "LokiError";
|
|
2118
|
+
};
|
|
2119
|
+
/**
|
|
2120
|
+
* The push URL: `url` with {@link LOKI_PUSH_PATH} as its path when it has none
|
|
2121
|
+
* (keeping its query string, userinfo and port), else `url` unchanged.
|
|
2122
|
+
*/
|
|
2123
|
+
function pushUrl(url) {
|
|
2124
|
+
if (!URL.canParse(url)) throw new PyValueError(INVALID_URL_MESSAGE$1);
|
|
2125
|
+
const target = new URL(url);
|
|
2126
|
+
if (target.pathname !== "" && target.pathname !== "/") return url;
|
|
2127
|
+
target.pathname = LOKI_PUSH_PATH;
|
|
2128
|
+
return target.href;
|
|
2129
|
+
}
|
|
2130
|
+
/** `url` without its userinfo. */
|
|
2131
|
+
function withoutUserinfo(url) {
|
|
2132
|
+
const target = new URL(url);
|
|
2133
|
+
if (!target.username && !target.password) return url;
|
|
2134
|
+
target.username = "";
|
|
2135
|
+
target.password = "";
|
|
2136
|
+
return target.href;
|
|
2137
|
+
}
|
|
2138
|
+
/** The first `length` characters (code points) of `text`, like Python. */
|
|
2139
|
+
function headChars(text, length) {
|
|
2140
|
+
return Array.from(text).slice(0, length).join("");
|
|
2141
|
+
}
|
|
2142
|
+
/** A client for the Grafana Loki push API. */
|
|
2143
|
+
var LokiClient = class {
|
|
2144
|
+
/** The push URL (with its userinfo, if any). */
|
|
2145
|
+
url;
|
|
2146
|
+
labels;
|
|
2147
|
+
labelFields;
|
|
2148
|
+
timestamp;
|
|
2149
|
+
timeout;
|
|
2150
|
+
batchSize;
|
|
2151
|
+
verify;
|
|
2152
|
+
/** Headers sent with every request. */
|
|
2153
|
+
headers;
|
|
2154
|
+
#requestUrl;
|
|
2155
|
+
#closed = new AbortController();
|
|
2156
|
+
/**
|
|
2157
|
+
* Every report record becomes one JSON log line.
|
|
2158
|
+
*
|
|
2159
|
+
* @param url - Loki base URL (`/loki/api/v1/push` is appended when the URL
|
|
2160
|
+
* has no path) or the full push URL
|
|
2161
|
+
*/
|
|
2162
|
+
constructor(url, options = {}) {
|
|
2163
|
+
const timestamp = options.timestamp ?? "ingest";
|
|
2164
|
+
if (!LOKI_TIMESTAMP_MODES.includes(timestamp)) throw new PyValueError(`Invalid Loki timestamp '${pyStr(timestamp)}'. Must be one of: ${LOKI_TIMESTAMP_MODES.join(", ")}`);
|
|
2165
|
+
const { username, password, token, tenantId } = options;
|
|
2166
|
+
if (token && (username || password)) throw new PyValueError("Loki token and username/password are exclusive");
|
|
2167
|
+
const labels = {
|
|
2168
|
+
job: "parsedmarc",
|
|
2169
|
+
host: hostname()
|
|
2170
|
+
};
|
|
2171
|
+
for (const [name, value] of Object.entries(options.labels ?? {})) labels[name] = value;
|
|
2172
|
+
this.labels = labels;
|
|
2173
|
+
this.labelFields = [...options.labelFields ?? LOKI_DEFAULT_LABEL_FIELDS];
|
|
2174
|
+
for (const name of [...Object.keys(labels), ...this.labelFields]) if (!LABEL_NAME.test(name)) throw new PyValueError(`Invalid Loki label name '${name}'`);
|
|
2175
|
+
this.url = pushUrl(url);
|
|
2176
|
+
this.timestamp = timestamp;
|
|
2177
|
+
this.timeout = options.timeout ?? DEFAULT_TIMEOUT$1;
|
|
2178
|
+
this.batchSize = Math.max(1, Math.trunc(options.batchSize ?? DEFAULT_BATCH_SIZE));
|
|
2179
|
+
if (Number.isNaN(this.batchSize)) throw new PyValueError("Invalid Loki batch size");
|
|
2180
|
+
this.verify = options.verify ?? true;
|
|
2181
|
+
const headers = {
|
|
2182
|
+
"User-Agent": USER_AGENT,
|
|
2183
|
+
"Content-Type": "application/json"
|
|
2184
|
+
};
|
|
2185
|
+
if (token) headers.Authorization = `Bearer ${token}`;
|
|
2186
|
+
if (tenantId) headers["X-Scope-OrgID"] = tenantId;
|
|
2187
|
+
this.#requestUrl = this.url;
|
|
2188
|
+
if (username) {
|
|
2189
|
+
headers.Authorization = `Basic ${Buffer$1.from(`${username}:${password ?? ""}`, "utf8").toString("base64")}`;
|
|
2190
|
+
this.#requestUrl = withoutUserinfo(this.url);
|
|
2191
|
+
}
|
|
2192
|
+
this.headers = headers;
|
|
2193
|
+
}
|
|
2194
|
+
async saveAggregateReportToLoki(aggregateReports) {
|
|
2195
|
+
await this.#save("aggregate", parsedAggregateReportsToCsvRows(aggregateReports));
|
|
2196
|
+
}
|
|
2197
|
+
async saveFailureReportToLoki(failureReports) {
|
|
2198
|
+
await this.#save("failure", parsedFailureReportsToCsvRows(failureReports));
|
|
2199
|
+
}
|
|
2200
|
+
/** Backward-compatible alias of {@link saveFailureReportToLoki}. */
|
|
2201
|
+
async saveForensicReportToLoki(failureReports) {
|
|
2202
|
+
await this.saveFailureReportToLoki(failureReports);
|
|
2203
|
+
}
|
|
2204
|
+
async saveSmtpTlsReportToLoki(smtpTlsReports) {
|
|
2205
|
+
await this.#save("smtp_tls", parsedSmtpTlsReportsToCsvRows(smtpTlsReports));
|
|
2206
|
+
}
|
|
2207
|
+
async #save(reportType, rows) {
|
|
2208
|
+
if (rows.length === 0) return;
|
|
2209
|
+
logger.debug(`Saving ${rows.length} ${reportType} records to Loki`);
|
|
2210
|
+
const [domainField, beginField, beginIsUtc] = REPORT_TYPES[reportType];
|
|
2211
|
+
const now = BigInt(Date.now()) * NS_PER_MS;
|
|
2212
|
+
const sorted = rows.map((row, index) => {
|
|
2213
|
+
const line = {
|
|
2214
|
+
...row,
|
|
2215
|
+
report_type: reportType
|
|
2216
|
+
};
|
|
2217
|
+
if (!Object.hasOwn(line, "domain")) line.domain = row[domainField];
|
|
2218
|
+
let timestamp = now;
|
|
2219
|
+
if (this.timestamp === "report_begin") try {
|
|
2220
|
+
if (!Object.hasOwn(row, beginField)) throw new PyValueError(`'${beginField}'`);
|
|
2221
|
+
const begin = humanTimestampToUnixTimestamp(pyStr(row[beginField]), { assumeUtc: beginIsUtc });
|
|
2222
|
+
timestamp = BigInt(begin) * NS_PER_SECOND;
|
|
2223
|
+
} catch {
|
|
2224
|
+
logger.debug(`Unparsable ${beginField}, using ingest time`);
|
|
2225
|
+
}
|
|
2226
|
+
return {
|
|
2227
|
+
labels: this.#streamLabels(line),
|
|
2228
|
+
timestamp: timestamp + BigInt(index),
|
|
2229
|
+
line
|
|
2230
|
+
};
|
|
2231
|
+
}).toSorted((a, b) => {
|
|
2232
|
+
if (a.timestamp === b.timestamp) return 0;
|
|
2233
|
+
return a.timestamp < b.timestamp ? -1 : 1;
|
|
2234
|
+
});
|
|
2235
|
+
for (let start = 0; start < sorted.length; start += this.batchSize) await this.#push(sorted.slice(start, start + this.batchSize));
|
|
2236
|
+
}
|
|
2237
|
+
#streamLabels(line) {
|
|
2238
|
+
const labels = { ...this.labels };
|
|
2239
|
+
for (const field of this.labelFields) {
|
|
2240
|
+
const value = Object.hasOwn(line, field) ? line[field] : void 0;
|
|
2241
|
+
if (value !== null && value !== void 0 && value !== "") labels[field] = pyStr(value);
|
|
2242
|
+
}
|
|
2243
|
+
return Object.entries(labels);
|
|
2244
|
+
}
|
|
2245
|
+
async #push(entries) {
|
|
2246
|
+
const streams = /* @__PURE__ */ new Map();
|
|
2247
|
+
for (const { labels, timestamp, line } of entries) {
|
|
2248
|
+
const key = JSON.stringify(labels);
|
|
2249
|
+
let stream = streams.get(key);
|
|
2250
|
+
if (stream === void 0) {
|
|
2251
|
+
stream = {
|
|
2252
|
+
labels,
|
|
2253
|
+
values: []
|
|
2254
|
+
};
|
|
2255
|
+
streams.set(key, stream);
|
|
2256
|
+
}
|
|
2257
|
+
stream.values.push([timestamp.toString(), pyJsonDumps(line, {
|
|
2258
|
+
ensureAscii: false,
|
|
2259
|
+
default: pyStr
|
|
2260
|
+
})]);
|
|
2261
|
+
}
|
|
2262
|
+
const body = { streams: [...streams.values()].map(({ labels, values }) => ({
|
|
2263
|
+
stream: Object.fromEntries(labels),
|
|
2264
|
+
values
|
|
2265
|
+
})) };
|
|
2266
|
+
const redactedUrl = redactUrlCredentials(this.url);
|
|
2267
|
+
let status;
|
|
2268
|
+
let text;
|
|
2269
|
+
try {
|
|
2270
|
+
if (this.#closed.signal.aborted) throw new Error(CLOSED_MESSAGE$1);
|
|
2271
|
+
const response = await httpPost(this.#requestUrl, {
|
|
2272
|
+
headers: { ...this.headers },
|
|
2273
|
+
body: pyJsonDumps(body),
|
|
2274
|
+
timeout: this.timeout,
|
|
2275
|
+
verify: this.verify,
|
|
2276
|
+
followRedirects: false,
|
|
2277
|
+
signal: this.#closed.signal
|
|
2278
|
+
});
|
|
2279
|
+
({status} = response);
|
|
2280
|
+
if (SUCCESS_STATUSES.has(status)) {
|
|
2281
|
+
await response.discard();
|
|
2282
|
+
return;
|
|
2283
|
+
}
|
|
2284
|
+
try {
|
|
2285
|
+
text = await response.text();
|
|
2286
|
+
} catch {
|
|
2287
|
+
text = "";
|
|
2288
|
+
}
|
|
2289
|
+
} catch (error) {
|
|
2290
|
+
if (this.#closed.signal.aborted) throw new Error(CLOSED_MESSAGE$1, { cause: error });
|
|
2291
|
+
throw new LokiError(`Loki push to ${redactedUrl} failed: ${describeError(error)}`, { cause: error });
|
|
2292
|
+
}
|
|
2293
|
+
throw new LokiError(`Loki push to ${redactedUrl} failed with HTTP ${status}: ${headChars(text, MAX_ERROR_BODY)}`);
|
|
2294
|
+
}
|
|
2295
|
+
/** Close the client: abort in-flight requests and refuse new ones. */
|
|
2296
|
+
close() {
|
|
2297
|
+
this.#closed.abort(/* @__PURE__ */ new Error(CLOSED_MESSAGE$1));
|
|
2298
|
+
}
|
|
2299
|
+
};
|
|
2300
|
+
// src/outputs/syslog.ts
|
|
2301
|
+
/** Connection problems, shared; report rows go through each client's logger. */
|
|
2302
|
+
const connectionLogger = getLogger("tsdmarc_syslog");
|
|
2303
|
+
const MS_PER_SECOND = 1e3;
|
|
2304
|
+
/** How long `close()` waits for queued bytes to flush. */
|
|
2305
|
+
const CLOSE_TIMEOUT_MS = 5e3;
|
|
2306
|
+
/** `LOG_USER`, SysLogHandler's default facility. */
|
|
2307
|
+
const LOG_USER = 1;
|
|
2308
|
+
/** SysLogHandler's `priority_names` for the levels its `priority_map` uses. */
|
|
2309
|
+
const PRIORITIES = {
|
|
2310
|
+
DEBUG: 7,
|
|
2311
|
+
INFO: 6,
|
|
2312
|
+
WARNING: 4,
|
|
2313
|
+
ERROR: 3,
|
|
2314
|
+
CRITICAL: 2
|
|
2315
|
+
};
|
|
2316
|
+
const DEFAULT_PRIORITY = PRIORITIES.WARNING ?? 4;
|
|
2317
|
+
/**
|
|
2318
|
+
* Port of `logging.handlers.SysLogHandler` for a `(host, port)` address:
|
|
2319
|
+
* sends `<PRI>message\0` per record, as one datagram (UDP) or on one
|
|
2320
|
+
* connection (TCP, no octet counting, like Python). Create it with
|
|
2321
|
+
* {@link SysLogHandler.create}, which resolves the host and, for TCP,
|
|
2322
|
+
* connects, like Python's constructor. Unlike Python, which keeps using a
|
|
2323
|
+
* dead TCP socket forever, a failed send reconnects (with `SocketHandler`'s
|
|
2324
|
+
* back-off) and tries once more; what is still lost is in {@link takeFailures}.
|
|
2325
|
+
*/
|
|
2326
|
+
var SysLogHandler = class SysLogHandler extends Handler {
|
|
2327
|
+
address;
|
|
2328
|
+
facility = 1;
|
|
2329
|
+
socktype;
|
|
2330
|
+
/** Seconds for each send on a stream socket; `null` waits forever. */
|
|
2331
|
+
timeout;
|
|
2332
|
+
/** The datagram socket, or the (possibly TLS-wrapped) stream socket. */
|
|
2333
|
+
socket = null;
|
|
2334
|
+
#queue = new SendQueue();
|
|
2335
|
+
#failures = new SendFailures();
|
|
2336
|
+
#wrapSocket;
|
|
2337
|
+
/** Epoch ms before which no reconnect is attempted. */
|
|
2338
|
+
#retryTime = null;
|
|
2339
|
+
#retryPeriod = 1;
|
|
2340
|
+
#connectError = null;
|
|
2341
|
+
constructor(address, options) {
|
|
2342
|
+
super();
|
|
2343
|
+
this.address = address;
|
|
2344
|
+
this.socktype = options.socktype ?? "dgram";
|
|
2345
|
+
this.timeout = options.timeout ?? null;
|
|
2346
|
+
this.#wrapSocket = options.wrapSocket;
|
|
2347
|
+
this.formatter = (record) => record.error === void 0 ? record.message : `${record.message}\n${record.error instanceof Error ? record.error.stack ?? errorText(record.error) : String(record.error)}`;
|
|
2348
|
+
}
|
|
2349
|
+
/** Create the handler and its socket (throws if that fails). */
|
|
2350
|
+
static async create(address, options = {}) {
|
|
2351
|
+
const handler = new SysLogHandler(address, options);
|
|
2352
|
+
await handler.createSocket();
|
|
2353
|
+
return handler;
|
|
2354
|
+
}
|
|
2355
|
+
/**
|
|
2356
|
+
* Resolve the address and create the socket; for TCP, connect to the
|
|
2357
|
+
* first address that accepts, like Python's `createSocket()`.
|
|
2358
|
+
*/
|
|
2359
|
+
async createSocket() {
|
|
2360
|
+
const [host, port] = this.address;
|
|
2361
|
+
const addresses = await lookup(host, {
|
|
2362
|
+
all: true,
|
|
2363
|
+
order: "verbatim"
|
|
2364
|
+
});
|
|
2365
|
+
if (addresses.length === 0) throw new Error("getaddrinfo returns an empty list");
|
|
2366
|
+
const timeoutMs = this.timeout ? this.timeout * MS_PER_SECOND : null;
|
|
2367
|
+
let lastError = null;
|
|
2368
|
+
for (const { address, family } of addresses) {
|
|
2369
|
+
if (this.socktype === "dgram") {
|
|
2370
|
+
const socket = createSocket(family === 6 ? "udp6" : "udp4");
|
|
2371
|
+
socket.on("error", () => void 0);
|
|
2372
|
+
socket.unref();
|
|
2373
|
+
this.socket = socket;
|
|
2374
|
+
return;
|
|
2375
|
+
}
|
|
2376
|
+
try {
|
|
2377
|
+
const plain = await connectTcp({
|
|
2378
|
+
host: address,
|
|
2379
|
+
port,
|
|
2380
|
+
family
|
|
2381
|
+
}, timeoutMs);
|
|
2382
|
+
plain.unref();
|
|
2383
|
+
const socket = await this.#wrapSocket?.(plain) ?? plain;
|
|
2384
|
+
socket.unref();
|
|
2385
|
+
this.socket = socket;
|
|
2386
|
+
return;
|
|
2387
|
+
} catch (error) {
|
|
2388
|
+
lastError = error;
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
throw lastError;
|
|
2392
|
+
}
|
|
2393
|
+
/** `SysLogHandler.encodePriority()` with `mapPriority()`. */
|
|
2394
|
+
encodePriority(levelName) {
|
|
2395
|
+
return this.facility * 8 + (PRIORITIES[levelName] ?? DEFAULT_PRIORITY);
|
|
2396
|
+
}
|
|
2397
|
+
emit(record) {
|
|
2398
|
+
const message = `<${this.encodePriority(record.levelName)}>${this.format(record)}\0`;
|
|
2399
|
+
const data = Buffer.from(message, "utf-8");
|
|
2400
|
+
this.#queue.push(() => this.#send(data), (error) => {
|
|
2401
|
+
this.handleError(record, error);
|
|
2402
|
+
if (this.socktype === "stream") this.#failures.record(error);
|
|
2403
|
+
});
|
|
2404
|
+
}
|
|
2405
|
+
async #send(data) {
|
|
2406
|
+
if (this.socktype === "stream") {
|
|
2407
|
+
try {
|
|
2408
|
+
await this.#sendOverStream(data);
|
|
2409
|
+
} catch {
|
|
2410
|
+
await this.#sendOverStream(data);
|
|
2411
|
+
}
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
if (this.socket === null) await this.createSocket();
|
|
2415
|
+
const [host, port] = this.address;
|
|
2416
|
+
await sendTo(this.socket, data, port, host);
|
|
2417
|
+
}
|
|
2418
|
+
/** Send over the open connection, or a new one; throws if that fails. */
|
|
2419
|
+
async #sendOverStream(data) {
|
|
2420
|
+
if (this.socket === null) await this.#reconnect();
|
|
2421
|
+
const socket = this.socket;
|
|
2422
|
+
const timeoutMs = this.timeout ? this.timeout * MS_PER_SECOND : null;
|
|
2423
|
+
try {
|
|
2424
|
+
await sendAll(socket, data, timeoutMs);
|
|
2425
|
+
} catch (error) {
|
|
2426
|
+
socket.destroy();
|
|
2427
|
+
this.socket = null;
|
|
2428
|
+
throw error;
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
/** `createSocket()` with the back-off of Python's `SocketHandler`. */
|
|
2432
|
+
async #reconnect() {
|
|
2433
|
+
const now = Date.now();
|
|
2434
|
+
if (this.#retryTime !== null && now < this.#retryTime) throw this.#connectError;
|
|
2435
|
+
try {
|
|
2436
|
+
await this.createSocket();
|
|
2437
|
+
this.#retryTime = null;
|
|
2438
|
+
} catch (error) {
|
|
2439
|
+
this.#retryPeriod = this.#retryTime === null ? 1 : Math.min(this.#retryPeriod * 2, 30);
|
|
2440
|
+
this.#retryTime = now + this.#retryPeriod * MS_PER_SECOND;
|
|
2441
|
+
this.#connectError = error;
|
|
2442
|
+
throw error;
|
|
2443
|
+
}
|
|
2444
|
+
}
|
|
2445
|
+
/** Resolves once every record emitted so far has been sent (or failed). */
|
|
2446
|
+
async drain() {
|
|
2447
|
+
await this.#queue.drain();
|
|
2448
|
+
}
|
|
2449
|
+
/** The messages lost over TCP/TLS since the last call (UDP has no record). */
|
|
2450
|
+
takeFailures() {
|
|
2451
|
+
return this.#failures.take();
|
|
2452
|
+
}
|
|
2453
|
+
/** Close the socket (after the queued sends). Use {@link closeAsync} to wait. */
|
|
2454
|
+
close() {
|
|
2455
|
+
this.closeAsync();
|
|
2456
|
+
}
|
|
2457
|
+
/** Send what is queued, then close the socket. */
|
|
2458
|
+
async closeAsync() {
|
|
2459
|
+
await this.drain();
|
|
2460
|
+
const { socket } = this;
|
|
2461
|
+
this.socket = null;
|
|
2462
|
+
if (socket === null) return;
|
|
2463
|
+
if (this.socktype === "dgram") await closeDatagram(socket);
|
|
2464
|
+
else await closeStream(socket, CLOSE_TIMEOUT_MS);
|
|
2465
|
+
}
|
|
2466
|
+
};
|
|
2467
|
+
/** Hooks for tests: the delay between connection attempts. */
|
|
2468
|
+
const syslogTimers = { async sleep(seconds) {
|
|
2469
|
+
await new Promise((resolve) => setTimeout(resolve, seconds * MS_PER_SECOND));
|
|
2470
|
+
} };
|
|
2471
|
+
async function readOptionalFile(filePath) {
|
|
2472
|
+
return filePath ? await readFile(filePath) : null;
|
|
2473
|
+
}
|
|
2474
|
+
/** Raised when messages could not be delivered over TCP or TLS. */
|
|
2475
|
+
var SyslogError = class extends Error {
|
|
2476
|
+
name = "SyslogError";
|
|
2477
|
+
};
|
|
2478
|
+
/** A client for Syslog. */
|
|
2479
|
+
var SyslogClient = class SyslogClient {
|
|
2480
|
+
serverName;
|
|
2481
|
+
serverPort;
|
|
2482
|
+
protocol;
|
|
2483
|
+
timeout;
|
|
2484
|
+
retryAttempts;
|
|
2485
|
+
retryDelay;
|
|
2486
|
+
logger;
|
|
2487
|
+
/** The syslog handler attached to {@link logger}. */
|
|
2488
|
+
logHandler;
|
|
2489
|
+
constructor(serverName, serverPort, options) {
|
|
2490
|
+
this.serverName = serverName;
|
|
2491
|
+
this.serverPort = serverPort;
|
|
2492
|
+
this.protocol = (options.protocol ?? "udp").toLowerCase();
|
|
2493
|
+
this.timeout = options.timeout ?? 5;
|
|
2494
|
+
this.retryAttempts = options.retryAttempts ?? 3;
|
|
2495
|
+
this.retryDelay = options.retryDelay ?? 5;
|
|
2496
|
+
this.logger = new Logger("tsdmarc_syslog");
|
|
2497
|
+
this.logger.setLevel(20);
|
|
2498
|
+
}
|
|
2499
|
+
/**
|
|
2500
|
+
* Create a SyslogClient: resolves the server and, for TCP/TLS, connects
|
|
2501
|
+
* (retrying `retryAttempts` times, `retryDelay` seconds apart), so a bad
|
|
2502
|
+
* address fails here as Python's constructor does.
|
|
2503
|
+
*
|
|
2504
|
+
* @param serverName - The Syslog server
|
|
2505
|
+
* @param serverPort - The Syslog port
|
|
2506
|
+
*/
|
|
2507
|
+
static async create(serverName, serverPort, options = {}) {
|
|
2508
|
+
const client = new SyslogClient(serverName, serverPort, options);
|
|
2509
|
+
client.logHandler = await client.createSyslogHandler(options);
|
|
2510
|
+
client.logger.addHandler(client.logHandler);
|
|
2511
|
+
return client;
|
|
2512
|
+
}
|
|
2513
|
+
/** Creates a SysLogHandler with the specified protocol and TLS settings. */
|
|
2514
|
+
async createSyslogHandler(options) {
|
|
2515
|
+
const { protocol, retryAttempts, retryDelay, timeout } = this;
|
|
2516
|
+
const address = [this.serverName, this.serverPort];
|
|
2517
|
+
if (protocol === "udp") return await SysLogHandler.create(address, { socktype: "dgram" });
|
|
2518
|
+
if (protocol !== "tcp" && protocol !== "tls") throw new PyValueError(`Invalid protocol '${protocol}'. Must be 'udp', 'tcp', or 'tls'.`);
|
|
2519
|
+
for (let attempt = 1; attempt <= retryAttempts; attempt += 1) try {
|
|
2520
|
+
if (protocol === "tcp") return await SysLogHandler.create(address, {
|
|
2521
|
+
socktype: "stream",
|
|
2522
|
+
timeout
|
|
2523
|
+
});
|
|
2524
|
+
return await this.createTlsHandler(options);
|
|
2525
|
+
} catch (error) {
|
|
2526
|
+
if (attempt < retryAttempts) {
|
|
2527
|
+
connectionLogger.warning(`Syslog connection attempt ${attempt}/${retryAttempts} failed: ${errorText(error)}. Retrying in ${retryDelay} seconds...`);
|
|
2528
|
+
await syslogTimers.sleep(retryDelay);
|
|
2529
|
+
} else {
|
|
2530
|
+
connectionLogger.error(`Syslog connection failed after ${retryAttempts} attempts: ${errorText(error)}`);
|
|
2531
|
+
throw error;
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
throw new PyValueError("retry_attempts must be at least 1");
|
|
2535
|
+
}
|
|
2536
|
+
/** TCP connection wrapped in TLS ≥ 1.2, verifying the server. */
|
|
2537
|
+
async createTlsHandler(options) {
|
|
2538
|
+
const { cafilePath, certfilePath, keyfilePath } = options;
|
|
2539
|
+
const tlsOptions = {
|
|
2540
|
+
minVersion: "TLSv1.2",
|
|
2541
|
+
rejectUnauthorized: true
|
|
2542
|
+
};
|
|
2543
|
+
const ca = await readOptionalFile(cafilePath);
|
|
2544
|
+
if (ca !== null) tlsOptions.ca = [...getCACertificates("default"), ca.toString("utf-8")];
|
|
2545
|
+
if (certfilePath && keyfilePath) {
|
|
2546
|
+
tlsOptions.cert = await readFile(certfilePath);
|
|
2547
|
+
tlsOptions.key = await readFile(keyfilePath);
|
|
2548
|
+
} else if (certfilePath || keyfilePath) connectionLogger.warning("Both certfile_path and keyfile_path are required for client certificate authentication. Client authentication will not be used.");
|
|
2549
|
+
const timeoutMs = this.timeout * MS_PER_SECOND;
|
|
2550
|
+
const wrapSocket = async (plain) => {
|
|
2551
|
+
try {
|
|
2552
|
+
return await connectTls({
|
|
2553
|
+
...tlsOptions,
|
|
2554
|
+
socket: plain,
|
|
2555
|
+
host: this.serverName,
|
|
2556
|
+
...isIP(this.serverName) === 0 && { servername: this.serverName }
|
|
2557
|
+
}, timeoutMs);
|
|
2558
|
+
} catch (error) {
|
|
2559
|
+
plain.destroy();
|
|
2560
|
+
throw error;
|
|
2561
|
+
}
|
|
2562
|
+
};
|
|
2563
|
+
return await SysLogHandler.create([this.serverName, this.serverPort], {
|
|
2564
|
+
socktype: "stream",
|
|
2565
|
+
timeout: this.timeout,
|
|
2566
|
+
wrapSocket
|
|
2567
|
+
});
|
|
2568
|
+
}
|
|
2569
|
+
async saveAggregateReportToSyslog(aggregateReports) {
|
|
2570
|
+
const rows = parsedAggregateReportsToCsvRows(aggregateReports);
|
|
2571
|
+
for (const row of rows) this.logger.info(pyJsonDumps(row));
|
|
2572
|
+
await this.#drain();
|
|
2573
|
+
}
|
|
2574
|
+
async saveFailureReportToSyslog(failureReports) {
|
|
2575
|
+
const rows = parsedFailureReportsToCsvRows(failureReports);
|
|
2576
|
+
for (const row of rows) this.logger.info(pyJsonDumps(row));
|
|
2577
|
+
await this.#drain();
|
|
2578
|
+
}
|
|
2579
|
+
/** Backward-compatible alias of {@link saveFailureReportToSyslog}. */
|
|
2580
|
+
async saveForensicReportToSyslog(failureReports) {
|
|
2581
|
+
await this.saveFailureReportToSyslog(failureReports);
|
|
2582
|
+
}
|
|
2583
|
+
async saveSmtpTlsReportToSyslog(smtpTlsReports) {
|
|
2584
|
+
const rows = parsedSmtpTlsReportsToCsvRows(smtpTlsReports);
|
|
2585
|
+
for (const row of rows) this.logger.info(pyJsonDumps(row));
|
|
2586
|
+
await this.#drain();
|
|
2587
|
+
}
|
|
2588
|
+
/**
|
|
2589
|
+
* Wait until the messages logged so far have been sent.
|
|
2590
|
+
*
|
|
2591
|
+
* @throws {SyslogError} When some could not be delivered over TCP or TLS.
|
|
2592
|
+
*/
|
|
2593
|
+
async #drain() {
|
|
2594
|
+
if (this.logHandler instanceof SysLogHandler) {
|
|
2595
|
+
await this.logHandler.drain();
|
|
2596
|
+
const failures = this.logHandler.takeFailures();
|
|
2597
|
+
if (failures !== null) throw new SyslogError(`${failures.count} message(s) not sent to ${this.serverName}:${this.serverPort}: ${errorText(failures.error)}`, { cause: failures.error });
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
/** Remove and close the syslog handler, releasing its socket. */
|
|
2601
|
+
async close() {
|
|
2602
|
+
this.logger.removeHandler(this.logHandler);
|
|
2603
|
+
if (this.logHandler instanceof SysLogHandler) await this.logHandler.closeAsync();
|
|
2604
|
+
else this.logHandler.close();
|
|
2605
|
+
}
|
|
2606
|
+
};
|
|
2607
|
+
// src/outputs/webhook.ts
|
|
2608
|
+
const DEFAULT_TIMEOUT = 60;
|
|
2609
|
+
/** Raised (and logged) when a request is made after {@link WebhookClient.close}. */
|
|
2610
|
+
const CLOSED_MESSAGE = "Cannot send a request, as the client has been closed.";
|
|
2611
|
+
/** Logged (or thrown) for a URL that can't be parsed; see {@link InvalidURL}. */
|
|
2612
|
+
const INVALID_URL_MESSAGE = "invalid webhook URL (details withheld because they may include credentials)";
|
|
2613
|
+
const HTTP_SUCCESS_MIN = 200;
|
|
2614
|
+
const HTTP_SUCCESS_MAX = 299;
|
|
2615
|
+
function isSuccess(status) {
|
|
2616
|
+
return status >= HTTP_SUCCESS_MIN && status <= HTTP_SUCCESS_MAX;
|
|
2617
|
+
}
|
|
2618
|
+
/** A client for webhooks. */
|
|
2619
|
+
var WebhookClient = class {
|
|
2620
|
+
aggregateUrl;
|
|
2621
|
+
failureUrl;
|
|
2622
|
+
smtpTlsUrl;
|
|
2623
|
+
timeout;
|
|
2624
|
+
throwOnError;
|
|
2625
|
+
/** Headers sent with every request. */
|
|
2626
|
+
headers;
|
|
2627
|
+
#closed = new AbortController();
|
|
2628
|
+
/**
|
|
2629
|
+
* @param aggregateUrl - The aggregate report webhook URL
|
|
2630
|
+
* @param failureUrl - The failure report webhook URL
|
|
2631
|
+
* @param smtpTlsUrl - The SMTP TLS report webhook URL
|
|
2632
|
+
*/
|
|
2633
|
+
constructor(aggregateUrl, failureUrl, smtpTlsUrl, options = {}) {
|
|
2634
|
+
this.aggregateUrl = aggregateUrl;
|
|
2635
|
+
this.failureUrl = failureUrl;
|
|
2636
|
+
this.smtpTlsUrl = smtpTlsUrl;
|
|
2637
|
+
this.timeout = options.timeout === void 0 ? DEFAULT_TIMEOUT : options.timeout;
|
|
2638
|
+
this.throwOnError = options.throwOnError ?? false;
|
|
2639
|
+
this.headers = {
|
|
2640
|
+
"User-Agent": USER_AGENT,
|
|
2641
|
+
"Content-Type": "application/json"
|
|
2642
|
+
};
|
|
2643
|
+
}
|
|
2644
|
+
async saveFailureReportToWebhook(report) {
|
|
2645
|
+
await this.sendToWebhook(this.failureUrl, report);
|
|
2646
|
+
}
|
|
2647
|
+
/** Backward-compatible alias of {@link saveFailureReportToWebhook}. */
|
|
2648
|
+
async saveForensicReportToWebhook(report) {
|
|
2649
|
+
await this.saveFailureReportToWebhook(report);
|
|
2650
|
+
}
|
|
2651
|
+
async saveSmtpTlsReportToWebhook(report) {
|
|
2652
|
+
await this.sendToWebhook(this.smtpTlsUrl, report);
|
|
2653
|
+
}
|
|
2654
|
+
async saveAggregateReportToWebhook(report) {
|
|
2655
|
+
await this.sendToWebhook(this.aggregateUrl, report);
|
|
2656
|
+
}
|
|
2657
|
+
/**
|
|
2658
|
+
* POST `payload` to `webhookUrl`. A string or bytes payload is sent as
|
|
2659
|
+
* is; an object is form-encoded. Network errors are logged and swallowed
|
|
2660
|
+
* and HTTP error statuses ignored, so a failing webhook never aborts the
|
|
2661
|
+
* surrounding batch; with `throwOnError`, both are thrown instead (and
|
|
2662
|
+
* not logged here).
|
|
2663
|
+
*/
|
|
2664
|
+
async sendToWebhook(webhookUrl, payload) {
|
|
2665
|
+
let status;
|
|
2666
|
+
try {
|
|
2667
|
+
if (this.#closed.signal.aborted) throw new Error(CLOSED_MESSAGE);
|
|
2668
|
+
const body = typeof payload === "string" || payload instanceof Uint8Array ? payload : new URLSearchParams(payload);
|
|
2669
|
+
const response = await httpPost(webhookUrl, {
|
|
2670
|
+
headers: { ...this.headers },
|
|
2671
|
+
body,
|
|
2672
|
+
timeout: this.timeout,
|
|
2673
|
+
signal: this.#closed.signal
|
|
2674
|
+
});
|
|
2675
|
+
await response.discard();
|
|
2676
|
+
({status} = response);
|
|
2677
|
+
} catch (error) {
|
|
2678
|
+
if (error instanceof InvalidURL) {
|
|
2679
|
+
if (this.throwOnError) throw new Error(INVALID_URL_MESSAGE);
|
|
2680
|
+
logger.error(`Webhook Error: ${INVALID_URL_MESSAGE}`);
|
|
2681
|
+
return;
|
|
2682
|
+
}
|
|
2683
|
+
if (this.throwOnError) throw new Error(describeError(error), { cause: error });
|
|
2684
|
+
logger.error(`Webhook Error: ${describeError(error)}`);
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
if (this.throwOnError && !isSuccess(status)) throw new Error(`Webhook responded with HTTP ${status}`);
|
|
2688
|
+
}
|
|
2689
|
+
/** Close the client: abort in-flight requests and refuse new ones. */
|
|
2690
|
+
close() {
|
|
2691
|
+
this.#closed.abort(/* @__PURE__ */ new Error(CLOSED_MESSAGE));
|
|
2692
|
+
}
|
|
2693
|
+
};
|
|
2694
|
+
// src/mailbox/maildir.ts
|
|
2695
|
+
/** The Maildir directory does not exist and `maildirCreate` was not set. */
|
|
2696
|
+
var NoSuchMailboxError = class extends Error {
|
|
2697
|
+
name = "NoSuchMailboxError";
|
|
2698
|
+
};
|
|
2699
|
+
/** A folder can't be removed because it still holds messages or subfolders. */
|
|
2700
|
+
var NotEmptyError = class extends Error {
|
|
2701
|
+
name = "NotEmptyError";
|
|
2702
|
+
};
|
|
2703
|
+
/** A move would overwrite a message with the same key (`mailbox.ExternalClashError`). */
|
|
2704
|
+
var ExternalClashError = class extends Error {
|
|
2705
|
+
name = "ExternalClashError";
|
|
2706
|
+
};
|
|
2707
|
+
/**
|
|
2708
|
+
* Running as root, the process could not fully drop to the Maildir's owner
|
|
2709
|
+
* (uid, gid and supplementary groups). mailsuite only warns and stays root.
|
|
2710
|
+
*/
|
|
2711
|
+
var PrivilegeDropError = class extends Error {
|
|
2712
|
+
name = "PrivilegeDropError";
|
|
2713
|
+
};
|
|
2714
|
+
/** No message has the given key (Python's `KeyError`). */
|
|
2715
|
+
var MaildirKeyError = class extends Error {
|
|
2716
|
+
name = "KeyError";
|
|
2717
|
+
};
|
|
2718
|
+
/** Separator between a message's unique name and its info (`:2,S`). */
|
|
2719
|
+
const COLON = ":";
|
|
2720
|
+
const SUBDIRS = [
|
|
2721
|
+
"tmp",
|
|
2722
|
+
"new",
|
|
2723
|
+
"cur"
|
|
2724
|
+
];
|
|
2725
|
+
const TOC_SUBDIRS = ["new", "cur"];
|
|
2726
|
+
const FOLDER_MODE = 448;
|
|
2727
|
+
const FILE_MODE = 438;
|
|
2728
|
+
const MILLISECONDS_PER_SECOND = 1e3;
|
|
2729
|
+
const MAILDIR_INFO_PREFIX = "2,";
|
|
2730
|
+
const SEEN_FLAG = "S";
|
|
2731
|
+
const utf8Decoder = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
2732
|
+
/** Python's `str(exception)`: the message without the class name. */
|
|
2733
|
+
function errorMessage(error) {
|
|
2734
|
+
return error instanceof Error ? error.message : String(error);
|
|
2735
|
+
}
|
|
2736
|
+
/** Python's `os.path.isdir`: false for anything missing or unreadable. */
|
|
2737
|
+
async function isDirectory(target) {
|
|
2738
|
+
try {
|
|
2739
|
+
return (await stat(target)).isDirectory();
|
|
2740
|
+
} catch {
|
|
2741
|
+
return false;
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
/** The flags of an info suffix (`2,RS` → `RS`), like `MaildirMessage.get_flags()`. */
|
|
2745
|
+
function flagsOf(info) {
|
|
2746
|
+
return info.startsWith(MAILDIR_INFO_PREFIX) ? info.slice(2) : "";
|
|
2747
|
+
}
|
|
2748
|
+
/** `MaildirMessage.add_flag()`: merge, deduplicate and sort the flags. */
|
|
2749
|
+
function addFlag(info, flag) {
|
|
2750
|
+
const flags = /* @__PURE__ */ new Set([...flagsOf(info), ...flag]);
|
|
2751
|
+
return `${MAILDIR_INFO_PREFIX}${[...flags].sort().join("")}`;
|
|
2752
|
+
}
|
|
2753
|
+
/**
|
|
2754
|
+
* The parts of Python's `mailbox.Maildir` that `MaildirConnection` uses.
|
|
2755
|
+
*
|
|
2756
|
+
* Keys are the unique part of a file name (before the colon). The table of
|
|
2757
|
+
* contents is rebuilt on every lookup miss and every `keys()` call, which is
|
|
2758
|
+
* what Python's `_refresh()` amounts to when files change underneath it.
|
|
2759
|
+
*/
|
|
2760
|
+
var Maildir = class Maildir {
|
|
2761
|
+
root;
|
|
2762
|
+
toc = /* @__PURE__ */ new Map();
|
|
2763
|
+
constructor(root) {
|
|
2764
|
+
this.root = root;
|
|
2765
|
+
}
|
|
2766
|
+
/** Open (and with `create`, make) a Maildir, like `mailbox.Maildir(path, create=...)`. */
|
|
2767
|
+
static openSync(dirname, create) {
|
|
2768
|
+
const root = path.resolve(dirname);
|
|
2769
|
+
let exists = true;
|
|
2770
|
+
try {
|
|
2771
|
+
statSync(root);
|
|
2772
|
+
} catch {
|
|
2773
|
+
exists = false;
|
|
2774
|
+
}
|
|
2775
|
+
if (!exists) {
|
|
2776
|
+
if (!create) throw new NoSuchMailboxError(root);
|
|
2777
|
+
mkdirSync(root, { mode: FOLDER_MODE });
|
|
2778
|
+
for (const subdir of SUBDIRS) mkdirSync(path.join(root, subdir), { mode: FOLDER_MODE });
|
|
2779
|
+
}
|
|
2780
|
+
return new Maildir(root);
|
|
2781
|
+
}
|
|
2782
|
+
/** `Maildir.add_folder()`: open or create `.<folder>` and its `maildirfolder` marker. */
|
|
2783
|
+
async addFolder(folder) {
|
|
2784
|
+
const folderPath = path.join(this.root, `.${folder}`);
|
|
2785
|
+
const result = Maildir.openSync(folderPath, true);
|
|
2786
|
+
const markerPath = path.join(folderPath, "maildirfolder");
|
|
2787
|
+
await (await open(markerPath, constants.O_CREAT | constants.O_WRONLY, FILE_MODE)).close();
|
|
2788
|
+
return result;
|
|
2789
|
+
}
|
|
2790
|
+
/** `Maildir.remove_folder()`: delete an empty folder. */
|
|
2791
|
+
async removeFolder(folder) {
|
|
2792
|
+
const folderPath = path.join(this.root, `.${folder}`);
|
|
2793
|
+
const entries = [...await readdir(path.join(folderPath, "new")), ...await readdir(path.join(folderPath, "cur"))];
|
|
2794
|
+
for (const entry of entries) if (entry.length < 1 || !entry.startsWith(".")) throw new NotEmptyError(`Folder contains message(s): ${folder}`);
|
|
2795
|
+
for (const entry of await readdir(folderPath)) if (entry !== "new" && entry !== "cur" && entry !== "tmp" && await isDirectory(path.join(folderPath, entry))) throw new NotEmptyError(`Folder contains subdirectory '${folder}': ${entry}`);
|
|
2796
|
+
for (const entry of await readdir(folderPath)) await rm(path.join(folderPath, entry), { recursive: true });
|
|
2797
|
+
await rmdir(folderPath);
|
|
2798
|
+
}
|
|
2799
|
+
/** `Maildir._refresh()`: map each key to its `new/...` or `cur/...` subpath. */
|
|
2800
|
+
async refresh() {
|
|
2801
|
+
const toc = /* @__PURE__ */ new Map();
|
|
2802
|
+
for (const subdir of TOC_SUBDIRS) {
|
|
2803
|
+
const subdirPath = path.join(this.root, subdir);
|
|
2804
|
+
for (const entry of await readdir(subdirPath)) {
|
|
2805
|
+
if (entry.startsWith(".")) continue;
|
|
2806
|
+
if (await isDirectory(path.join(subdirPath, entry))) continue;
|
|
2807
|
+
const key = entry.split(COLON)[0] ?? entry;
|
|
2808
|
+
toc.set(key, path.join(subdir, entry));
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
this.toc = toc;
|
|
2812
|
+
}
|
|
2813
|
+
/** `Maildir._lookup()`: the subpath of a key, or `MaildirKeyError`. */
|
|
2814
|
+
async lookup(key) {
|
|
2815
|
+
const cached = this.toc.get(key);
|
|
2816
|
+
if (cached !== void 0) try {
|
|
2817
|
+
await stat(path.join(this.root, cached));
|
|
2818
|
+
return cached;
|
|
2819
|
+
} catch {}
|
|
2820
|
+
await this.refresh();
|
|
2821
|
+
const subpath = this.toc.get(key);
|
|
2822
|
+
if (subpath === void 0) throw new MaildirKeyError(`No message with key: ${key}`);
|
|
2823
|
+
return subpath;
|
|
2824
|
+
}
|
|
2825
|
+
async keys() {
|
|
2826
|
+
await this.refresh();
|
|
2827
|
+
return [...this.toc.keys()];
|
|
2828
|
+
}
|
|
2829
|
+
/** The size of a message file in bytes, or `null` when the key is unknown. */
|
|
2830
|
+
async size(key) {
|
|
2831
|
+
let subpath;
|
|
2832
|
+
try {
|
|
2833
|
+
subpath = await this.lookup(key);
|
|
2834
|
+
} catch (error) {
|
|
2835
|
+
if (error instanceof MaildirKeyError) return null;
|
|
2836
|
+
throw error;
|
|
2837
|
+
}
|
|
2838
|
+
return (await stat(path.join(this.root, subpath))).size;
|
|
2839
|
+
}
|
|
2840
|
+
/**
|
|
2841
|
+
* `Maildir.get_bytes()`: the stored bytes, or `null` when the key is
|
|
2842
|
+
* unknown. Python converts `os.linesep` to `\n` (a no-op on POSIX).
|
|
2843
|
+
*/
|
|
2844
|
+
async getBytes(key) {
|
|
2845
|
+
let subpath;
|
|
2846
|
+
try {
|
|
2847
|
+
subpath = await this.lookup(key);
|
|
2848
|
+
} catch (error) {
|
|
2849
|
+
if (error instanceof MaildirKeyError) return null;
|
|
2850
|
+
throw error;
|
|
2851
|
+
}
|
|
2852
|
+
const content = await readFile(path.join(this.root, subpath));
|
|
2853
|
+
return EOL === "\n" ? content : Buffer.from(content.toString("latin1").replaceAll(EOL, "\n"), "latin1");
|
|
2854
|
+
}
|
|
2855
|
+
/** `key in maildir`. */
|
|
2856
|
+
async has(key) {
|
|
2857
|
+
await this.refresh();
|
|
2858
|
+
return this.toc.has(key);
|
|
2859
|
+
}
|
|
2860
|
+
/**
|
|
2861
|
+
* Move a message's file into the same subdirectory (`new`/`cur`) of
|
|
2862
|
+
* `destination`, keeping its name; `MaildirKeyError` for an unknown key.
|
|
2863
|
+
*/
|
|
2864
|
+
async moveTo(key, destination) {
|
|
2865
|
+
const subpath = await this.lookup(key);
|
|
2866
|
+
await rename(path.join(this.root, subpath), path.join(destination.root, subpath));
|
|
2867
|
+
this.toc.delete(key);
|
|
2868
|
+
destination.toc.set(key, subpath);
|
|
2869
|
+
}
|
|
2870
|
+
/** `Maildir.remove()`: delete a message, or `MaildirKeyError`. */
|
|
2871
|
+
async remove(key) {
|
|
2872
|
+
await unlink(path.join(this.root, await this.lookup(key)));
|
|
2873
|
+
}
|
|
2874
|
+
/**
|
|
2875
|
+
* Mark a message read like mailsuite 2.3.3: rename its file into `cur/`
|
|
2876
|
+
* with the `S` (Seen) flag added to its info suffix, which is what the
|
|
2877
|
+
* Maildir spec defines a flag change to be; the content is untouched.
|
|
2878
|
+
*/
|
|
2879
|
+
async markRead(key) {
|
|
2880
|
+
const subpath = await this.lookup(key);
|
|
2881
|
+
const name = path.basename(subpath);
|
|
2882
|
+
const info = name.includes(COLON) ? name.split(COLON).at(-1) ?? "" : "";
|
|
2883
|
+
const newSubpath = path.join("cur", `${key}${COLON}${addFlag(info, SEEN_FLAG)}`);
|
|
2884
|
+
await rename(path.join(this.root, subpath), path.join(this.root, newSubpath));
|
|
2885
|
+
this.toc.set(key, newSubpath);
|
|
2886
|
+
}
|
|
2887
|
+
};
|
|
2888
|
+
/**
|
|
2889
|
+
* Drop root to the Maildir's owner: its group as the only group, then its uid.
|
|
2890
|
+
*
|
|
2891
|
+
* Unlike mailsuite, which keeps root's groups and, if `setuid` fails, only
|
|
2892
|
+
* warns and goes on as root, any failure throws `PrivilegeDropError`, so the
|
|
2893
|
+
* process never keeps working as root (or half-dropped) by accident.
|
|
2894
|
+
*/
|
|
2895
|
+
function dropPrivileges(maildirPath, uid, gid) {
|
|
2896
|
+
const { setuid, setgid, setgroups } = process;
|
|
2897
|
+
const fix = `Run tsdmarc as the Maildir's owner (uid ${uid}), or chown ${maildirPath} to the user tsdmarc runs as.`;
|
|
2898
|
+
if (setgroups === void 0 || setgid === void 0 || setuid === void 0) throw new PrivilegeDropError(`Running as root over ${maildirPath} (owner uid ${uid}), but this runtime can't drop root's groups. ${fix}`);
|
|
2899
|
+
try {
|
|
2900
|
+
setgroups([gid]);
|
|
2901
|
+
setgid(gid);
|
|
2902
|
+
setuid(uid);
|
|
2903
|
+
} catch (error) {
|
|
2904
|
+
throw new PrivilegeDropError(`Failed to switch to uid ${uid} and gid ${gid} to access Maildir ${maildirPath}: ${errorMessage(error)}. ${fix}`, { cause: error });
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
/**
|
|
2908
|
+
* A `MailboxConnection` backed by an on-disk Maildir.
|
|
2909
|
+
*
|
|
2910
|
+
* Useful for local processing of messages dropped into a Maildir by an MTA
|
|
2911
|
+
* (e.g. postfix delivering DMARC reports). Maildir has no concept of
|
|
2912
|
+
* sending: `sendMessage()` throws `NotImplementedError`.
|
|
2913
|
+
*
|
|
2914
|
+
* The constructor does its filesystem checks synchronously, like
|
|
2915
|
+
* mailsuite's: uid checks (warn, don't crash, but throw `PrivilegeDropError`
|
|
2916
|
+
* if root can't drop to the Maildir's owner), the optional `cur`/`new`/`tmp`
|
|
2917
|
+
* creation, and opening the Maildir.
|
|
2918
|
+
*/
|
|
2919
|
+
var MaildirConnection = class extends MailboxConnection {
|
|
2920
|
+
kind = "maildir";
|
|
2921
|
+
maildirPath;
|
|
2922
|
+
maildirCreate;
|
|
2923
|
+
client;
|
|
2924
|
+
activeFolder;
|
|
2925
|
+
subfolderClient = /* @__PURE__ */ new Map();
|
|
2926
|
+
constructor(maildirPath, options = {}) {
|
|
2927
|
+
super();
|
|
2928
|
+
this.maildirPath = maildirPath;
|
|
2929
|
+
this.maildirCreate = options.maildirCreate ?? false;
|
|
2930
|
+
const { getuid, setuid } = process;
|
|
2931
|
+
if (getuid !== void 0 && setuid !== void 0) {
|
|
2932
|
+
let maildirOwner = null;
|
|
2933
|
+
let maildirGroup = 0;
|
|
2934
|
+
try {
|
|
2935
|
+
({uid: maildirOwner, gid: maildirGroup} = statSync(maildirPath));
|
|
2936
|
+
} catch {
|
|
2937
|
+
maildirOwner = null;
|
|
2938
|
+
}
|
|
2939
|
+
const currentUid = getuid();
|
|
2940
|
+
if (maildirOwner !== null && currentUid !== maildirOwner) {
|
|
2941
|
+
if (currentUid === 0) {
|
|
2942
|
+
logger.warning(`Switching uid to ${maildirOwner} to access Maildir`);
|
|
2943
|
+
dropPrivileges(maildirPath, maildirOwner, maildirGroup);
|
|
2944
|
+
} else logger.warning(`Runtime uid ${currentUid} differs from maildir ${maildirPath} owner ${maildirOwner}. Access may fail if permissions are insufficient.`);
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
if (this.maildirCreate) for (const subdir of [
|
|
2948
|
+
"cur",
|
|
2949
|
+
"new",
|
|
2950
|
+
"tmp"
|
|
2951
|
+
]) mkdirSync(path.join(maildirPath, subdir), { recursive: true });
|
|
2952
|
+
this.client = Maildir.openSync(maildirPath, this.maildirCreate);
|
|
2953
|
+
this.activeFolder = this.client;
|
|
2954
|
+
}
|
|
2955
|
+
async getFolder(folderName) {
|
|
2956
|
+
let folder = this.subfolderClient.get(folderName);
|
|
2957
|
+
if (folder === void 0) {
|
|
2958
|
+
folder = await this.client.addFolder(folderName);
|
|
2959
|
+
this.subfolderClient.set(folderName, folder);
|
|
2960
|
+
}
|
|
2961
|
+
return folder;
|
|
2962
|
+
}
|
|
2963
|
+
folderPath(folderName) {
|
|
2964
|
+
return path.join(this.maildirPath, `.${folderName}`);
|
|
2965
|
+
}
|
|
2966
|
+
async createFolder(folderName) {
|
|
2967
|
+
await this.getFolder(folderName);
|
|
2968
|
+
}
|
|
2969
|
+
async renameFolder(oldName, newName) {
|
|
2970
|
+
await this.ensureNoFolderConflict(newName);
|
|
2971
|
+
await rename(this.folderPath(oldName), this.folderPath(newName));
|
|
2972
|
+
this.subfolderClient.delete(oldName);
|
|
2973
|
+
this.subfolderClient.delete(newName);
|
|
2974
|
+
}
|
|
2975
|
+
folderExists(folderName) {
|
|
2976
|
+
return isDirectory(this.folderPath(folderName));
|
|
2977
|
+
}
|
|
2978
|
+
async deleteFolder(folderName) {
|
|
2979
|
+
await this.client.removeFolder(folderName);
|
|
2980
|
+
this.subfolderClient.delete(folderName);
|
|
2981
|
+
}
|
|
2982
|
+
async doMoveFolder(source, _targetParent, targetPath) {
|
|
2983
|
+
await rename(this.folderPath(source), this.folderPath(targetPath));
|
|
2984
|
+
this.subfolderClient.delete(source);
|
|
2985
|
+
this.subfolderClient.delete(targetPath);
|
|
2986
|
+
}
|
|
2987
|
+
async fetchMessages(reportsFolder, _options) {
|
|
2988
|
+
this.activeFolder = reportsFolder && reportsFolder !== "INBOX" ? await this.getFolder(reportsFolder) : this.client;
|
|
2989
|
+
return this.activeFolder.keys();
|
|
2990
|
+
}
|
|
2991
|
+
/**
|
|
2992
|
+
* The message's stored bytes decoded as UTF-8 (undecodable bytes
|
|
2993
|
+
* replaced), like mailsuite 2.3.3 and its IMAP backend's raw fetch, or
|
|
2994
|
+
* `""` for an unknown key. The file is not parsed or re-serialized, so a
|
|
2995
|
+
* message Python's `email` package can parse but not generate (a `\x1c`
|
|
2996
|
+
* in a nested header, deep multipart nesting) is returned as is.
|
|
2997
|
+
*
|
|
2998
|
+
* With `markRead` the file is renamed into `cur/` with the `S` flag (see
|
|
2999
|
+
* `Maildir.markRead`). With `maxSize`, a larger file throws
|
|
3000
|
+
* `EmailTooLargeError` without being read (not in mailsuite).
|
|
3001
|
+
*/
|
|
3002
|
+
async fetchMessage(messageId, options = {}) {
|
|
3003
|
+
const key = String(messageId);
|
|
3004
|
+
const maxSize = options.maxSize ?? 0;
|
|
3005
|
+
if (maxSize > 0) {
|
|
3006
|
+
const size = await this.activeFolder.size(key);
|
|
3007
|
+
if (size !== null && size > maxSize) throw emailTooLargeError(size, maxSize);
|
|
3008
|
+
}
|
|
3009
|
+
const content = await this.activeFolder.getBytes(key);
|
|
3010
|
+
if (content === null) return "";
|
|
3011
|
+
const text = utf8Decoder.decode(content);
|
|
3012
|
+
if (options.markRead) await this.activeFolder.markRead(key);
|
|
3013
|
+
return text;
|
|
3014
|
+
}
|
|
3015
|
+
async deleteMessage(messageId) {
|
|
3016
|
+
await this.activeFolder.remove(String(messageId));
|
|
3017
|
+
}
|
|
3018
|
+
/**
|
|
3019
|
+
* Move the message's file into the same subdirectory (`new`/`cur`) of
|
|
3020
|
+
* `folderName`, keeping its file name, flags and bytes, like mailsuite
|
|
3021
|
+
* 2.3.3 (it no longer re-serializes the message). Maildir++ folders live
|
|
3022
|
+
* under the same root, so this is a same-filesystem rename; a folder on
|
|
3023
|
+
* another mount fails with `EXDEV`. Throws {@link ExternalClashError}
|
|
3024
|
+
* rather than overwriting when the destination already holds a message
|
|
3025
|
+
* with the same key (including a move into the active folder itself).
|
|
3026
|
+
*/
|
|
3027
|
+
async moveMessage(messageId, folderName) {
|
|
3028
|
+
const key = String(messageId);
|
|
3029
|
+
if (!await this.activeFolder.has(key)) return;
|
|
3030
|
+
const destination = await this.getFolder(folderName);
|
|
3031
|
+
if (await destination.has(key)) throw new ExternalClashError(`Folder ${folderName} already contains message ${key}`);
|
|
3032
|
+
await this.activeFolder.moveTo(key, destination);
|
|
3033
|
+
}
|
|
3034
|
+
keepalive() {
|
|
3035
|
+
return Promise.resolve();
|
|
3036
|
+
}
|
|
3037
|
+
async watch(options) {
|
|
3038
|
+
const { checkCallback, checkTimeout, configReloading, signal } = options;
|
|
3039
|
+
while (true) {
|
|
3040
|
+
if (signal?.aborted || configReloading?.()) return;
|
|
3041
|
+
try {
|
|
3042
|
+
await checkCallback(this);
|
|
3043
|
+
} catch (error) {
|
|
3044
|
+
logger.warning(`Maildir init error. ${errorMessage(error)}`);
|
|
3045
|
+
}
|
|
3046
|
+
if (configReloading?.()) return;
|
|
3047
|
+
try {
|
|
3048
|
+
await setTimeout$1(checkTimeout * MILLISECONDS_PER_SECOND, void 0, { signal });
|
|
3049
|
+
} catch {
|
|
3050
|
+
return;
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
sendMessage(_options) {
|
|
3055
|
+
return Promise.reject(new NotImplementedError("Maildir cannot send mail; use sendEmail from @tsdmarc/smtp"));
|
|
3056
|
+
}
|
|
3057
|
+
};
|
|
3058
|
+
// src/utils/create-email.ts
|
|
3059
|
+
const NON_ASCII = /\P{ASCII}/u;
|
|
3060
|
+
const DAY_NAMES = [
|
|
3061
|
+
"Sun",
|
|
3062
|
+
"Mon",
|
|
3063
|
+
"Tue",
|
|
3064
|
+
"Wed",
|
|
3065
|
+
"Thu",
|
|
3066
|
+
"Fri",
|
|
3067
|
+
"Sat"
|
|
3068
|
+
];
|
|
3069
|
+
const MONTH_NAMES = [
|
|
3070
|
+
"Jan",
|
|
3071
|
+
"Feb",
|
|
3072
|
+
"Mar",
|
|
3073
|
+
"Apr",
|
|
3074
|
+
"May",
|
|
3075
|
+
"Jun",
|
|
3076
|
+
"Jul",
|
|
3077
|
+
"Aug",
|
|
3078
|
+
"Sep",
|
|
3079
|
+
"Oct",
|
|
3080
|
+
"Nov",
|
|
3081
|
+
"Dec"
|
|
3082
|
+
];
|
|
3083
|
+
function pad2(value) {
|
|
3084
|
+
return String(value).padStart(2, "0");
|
|
3085
|
+
}
|
|
3086
|
+
/** `email.utils.formatdate(localtime=True)`: RFC 2822 date in local time. */
|
|
3087
|
+
function formatDate(date = /* @__PURE__ */ new Date()) {
|
|
3088
|
+
const offsetMinutes = -date.getTimezoneOffset();
|
|
3089
|
+
const sign = offsetMinutes < 0 ? "-" : "+";
|
|
3090
|
+
const absolute = Math.abs(offsetMinutes);
|
|
3091
|
+
const offset = `${sign}${pad2(Math.floor(absolute / 60))}${pad2(absolute % 60)}`;
|
|
3092
|
+
const day = DAY_NAMES[date.getDay()] ?? "";
|
|
3093
|
+
const month = MONTH_NAMES[date.getMonth()] ?? "";
|
|
3094
|
+
return `${day}, ${pad2(date.getDate())} ${month} ${date.getFullYear()} ${pad2(date.getHours())}:${pad2(date.getMinutes())}:${pad2(date.getSeconds())} ${offset}`;
|
|
3095
|
+
}
|
|
3096
|
+
/** `MIMEBase.__init__`: Content-Type (with `add_header` params) and MIME-Version. */
|
|
3097
|
+
function mimeBase(maintype, subtype, params) {
|
|
3098
|
+
const message = new Message();
|
|
3099
|
+
const parts = [`${maintype}/${subtype}`];
|
|
3100
|
+
for (const [name, value] of params) parts.push(formatParam(name, value));
|
|
3101
|
+
message.set("Content-Type", parts.join("; "));
|
|
3102
|
+
message.set("MIME-Version", "1.0");
|
|
3103
|
+
return message;
|
|
3104
|
+
}
|
|
3105
|
+
/** `MIMEText(text, subtype)`: us-ascii when it encodes, else utf-8 (base64). */
|
|
3106
|
+
function mimeText(text, subtype) {
|
|
3107
|
+
const charset = NON_ASCII.test(text) ? "utf-8" : "us-ascii";
|
|
3108
|
+
const message = mimeBase("text", subtype, [["charset", charset]]);
|
|
3109
|
+
message.setPayload(text, charset);
|
|
3110
|
+
return message;
|
|
3111
|
+
}
|
|
3112
|
+
/** `MIMEApplication(data, Name=filename)` with `encode_base64`. */
|
|
3113
|
+
function mimeApplication(data, filename) {
|
|
3114
|
+
const message = mimeBase("application", "octet-stream", [["Name", filename]]);
|
|
3115
|
+
message.setPayload(base64BodyEncode(data));
|
|
3116
|
+
message.set("Content-Transfer-Encoding", "base64");
|
|
3117
|
+
return message;
|
|
3118
|
+
}
|
|
3119
|
+
/**
|
|
3120
|
+
* Create an RFC 822 email message and return it as a string.
|
|
3121
|
+
*
|
|
3122
|
+
* Port of `mailsuite.utils.create_email`: a `multipart/mixed` message with
|
|
3123
|
+
* the plain and/or HTML body followed by one `application/octet-stream`
|
|
3124
|
+
* part per attachment.
|
|
3125
|
+
*/
|
|
3126
|
+
function createEmail(options) {
|
|
3127
|
+
const message = mimeBase("multipart", "mixed", []);
|
|
3128
|
+
message.payload = [];
|
|
3129
|
+
message.set("From", options.messageFrom);
|
|
3130
|
+
if (options.messageTo && options.messageTo.length > 0) message.set("To", options.messageTo.join(", "));
|
|
3131
|
+
if (options.messageCc && options.messageCc.length > 0) message.set("Cc", options.messageCc.join(", "));
|
|
3132
|
+
message.set("Date", formatDate());
|
|
3133
|
+
if (options.subject) message.set("Subject", options.subject);
|
|
3134
|
+
for (const [name, value] of Object.entries(options.messageHeaders ?? {})) message.set(name, value);
|
|
3135
|
+
if (options.plainMessage !== null && options.plainMessage !== void 0) message.attach(mimeText(options.plainMessage, "plain"));
|
|
3136
|
+
if (options.htmlMessage !== null && options.htmlMessage !== void 0) message.attach(mimeText(options.htmlMessage, "html"));
|
|
3137
|
+
for (const [filename, payload] of options.attachments ?? []) {
|
|
3138
|
+
const part = mimeApplication(payload, filename);
|
|
3139
|
+
part.set("Content-Disposition", `attachment; filename="${filename}"`);
|
|
3140
|
+
message.attach(part);
|
|
3141
|
+
}
|
|
3142
|
+
return message.asString();
|
|
3143
|
+
}
|
|
3144
|
+
export { AGGREGATE_CSV_FIELDS, BadEscape, BadResponse, BrokenProcessPool, CRITICAL, CalledProcessError, CaptureHandler, ContextFilter, CsvDictWriter, DEBUG, DEFAULT_DNS_MAX_RETRIES, DEFAULT_DNS_TIMEOUT, DEFAULT_JOB_TIMEOUT, DEFAULT_MAX_EMAIL_SIZE, DEFAULT_MAX_EXPANDED_EMAIL_SIZE, DEFAULT_MAX_FAILED_LOOKUPS_PER_REPORT, DEFAULT_MAX_HEADER_LINES, DEFAULT_MAX_LOOKUPS_PER_REPORT, DEFAULT_MAX_LOOKUP_TIME_PER_REPORT, DEFAULT_MAX_MIME_PARTS, DEFAULT_MAX_NORMALIZED_RECORDS, DEFAULT_MAX_SMTP_TLS_REPORT_SIZE, DEFAULT_MAX_WORKER_HEAP_MB, DEFAULT_MAX_XML_DEPTH, DEFAULT_MAX_XML_NODES, DEFAULT_NAMESERVERS, DEFAULT_PARSER_OPTIONS, DNSException, DNSFormError, DNSSocketError, DNSSyntaxError, DNSTimeout, DateParseError, DownloadError, EMAIL_SAMPLE_CONTENT_TYPES, ERROR, EmailParserError, EmailTooLargeError, EmptyLabel, EnrichmentError, ExpatError, ExpiringDict, ExternalClashError, FAILURE_CSV_FIELDS, FileHandler, FolderExistsError, FolderNotFoundError, GelfClient, GelfError, GelfTcpHandler, GelfTlsHandler, GelfUdpHandler, HECClient, Handler, IDNAException, INFO, IPDatabaseError, IP_ADDRESS_CACHE, InvalidAggregateReport, InvalidDMARCReport, InvalidFailureReport, InvalidForensicReport, InvalidIPinfoAPIKey, InvalidNameserverError, InvalidSMTPTLSReport, JobTimeoutError, LOG_USER, LOKI_DEFAULT_LABEL_FIELDS, LOKI_PUSH_PATH, LOKI_TIMESTAMP_MODES, LabelTooLong, LifetimeTimeout, Logger, LokiClient, LokiError, MAX_DECOMPRESSED_REPORT_SIZE, MAX_DECOMPRESSED_SLOW_CODEC_SIZE, MSGCONVERT_TIMEOUT_MS, MailboxConnection, MaildirConnection, MaildirKeyError, NOTSET, NXDOMAIN, NameTooLong, NoAnswer, NoMetaqueries, NoNameservers, NoSuchMailboxError, NormalizedRecordBudget, NotEmptyError, NotImplementedError, NullHandler, PARSEDMARC_COMPAT_VERSION, ParserError, PrivilegeDropError, PyDateTime, PyJsonNumber, RECOMMENDED_DNS_NAMESERVERS, REVERSE_DNS_MAP, RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES, RFC_9990_NAMESPACE, RFC_9990_POLICY_OVERRIDE_TYPES, SEEN_AGGREGATE_REPORT_IDS, SMTP_TLS_CSV_FIELDS, SplunkError, StreamHandler, SysLogHandler, SyslogClient, SyslogError, TEXT_REPORT_REGEX, USER_AGENT, UnknownRdatatype, VERSION, PyValueError as ValueError, WARNING, WebhookClient, WorkerCrashedError, YXDOMAIN, aggregateReportKey, appendCsv, appendJson, appendParsedRecord, basicFormat, bucketIntervalByDay, camelToSnake, captureLogs, checkEmailLimits, configureIpinfoApi, configureLogging, consoleFormat, convertOutlookMsg, createEmail, createParserConfig, csvFormulaEscape, csvSafe, decodeBase64, decompressGzipBounded, emailTooLargeError, extractReport, extractReportFromFilePath, extractReportFromStream, fileFormat, findEnrichmentError, formatAsctime, formatDate, getBaseDomain, getDmarcReportsFromMailbox, getDmarcReportsFromMbox, getFilenameSafeString, getIpAddressCountry, getIpAddressDbRecord, getIpAddressInfo, getLevelName, getLogger, getReportZip, getReverseDns, getServiceFromReverseDnsBaseDomain, humanTimestampToDatetime, humanTimestampToUnixTimestamp, initWorker, ipInfoState, isMbox, isOutlookMsg, loadIpDb, loadPslOverrides, loadReverseDnsMap, logContextData, logger, maskNameserverSecrets, newIpAddressCache, newSeenAggregateReportIds, parallelMap, parseAggregateReportFile, parseAggregateReportFromFilePath, parseAggregateReportXml, parseDate, parseEmail, parseEmailAddress, parseEmailText, parseFailureReport, parseForensicReport, parseReportEmail, parseReportEmailJob, parseReportEmailTask, parseReportFile, parseReportFileJob, parseReportFileTask, parseReportRecord, parseSmtpTlsFailureDetails, parseSmtpTlsReportJson, parseSmtpTlsReportPolicy, parsedAggregateReportsToCsv, parsedAggregateReportsToCsvRows, parsedFailureReportsToCsv, parsedFailureReportsToCsvRows, parsedForensicReportsToCsv, parsedForensicReportsToCsvRows, parsedSmtpTlsReportsToCsv, parsedSmtpTlsReportsToCsvRows, parserConfigFromState, parserConfigToState, pslOverrides, pyFloatRepr, pyJsonDumps, pyRepr, pyStr, queryDns, rawJsonIntegers, redactUrlCredentials, replaceParserConfig, resolveParserConfig, reverseNameFromAddress, rowsToCsv, sanitizeForLog, saveOutput, snakeToCamel, stripComments, timestampToDatetime, timestampToHuman, toCamelCaseKeys, toSnakeCaseKeys, unwrapText, watchInbox };
|
|
3145
|
+
|
|
3146
|
+
//# sourceMappingURL=index.js.map
|