twenty-app-intake 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +293 -8
- package/manifest.json +419 -667
- package/package.json +6 -9
- package/public/cover.generated.png +0 -0
- package/src/front-components/dashboard.mjs +19 -21947
- package/src/front-components/dashboard.mjs.map +4 -4
- package/src/front-components/dashboard.tsx +118 -0
- package/src/front-components/settings-panel.mjs +19 -22103
- package/src/front-components/settings-panel.mjs.map +4 -4
- package/src/front-components/settings-panel.tsx +304 -0
- package/src/front-components/source-panel.mjs +20 -21962
- package/src/front-components/source-panel.mjs.map +4 -4
- package/src/front-components/source-panel.tsx +113 -0
- package/src/logic-functions/check-silence.mjs +263 -0
- package/src/logic-functions/check-silence.mjs.map +7 -0
- package/src/logic-functions/check-silence.ts +52 -0
- package/src/logic-functions/contract.mjs +507 -0
- package/src/logic-functions/contract.mjs.map +7 -0
- package/src/logic-functions/contract.ts +172 -0
- package/src/logic-functions/health.mjs +125 -5
- package/src/logic-functions/health.mjs.map +3 -3
- package/src/logic-functions/health.ts +62 -0
- package/src/logic-functions/quarantine-discard.mjs +380 -0
- package/src/logic-functions/quarantine-discard.mjs.map +7 -0
- package/src/logic-functions/quarantine-discard.ts +65 -0
- package/src/logic-functions/quarantine-list.mjs +211 -0
- package/src/logic-functions/quarantine-list.mjs.map +7 -0
- package/src/logic-functions/quarantine-list.ts +93 -0
- package/src/logic-functions/quarantine-release.mjs +1974 -0
- package/src/logic-functions/quarantine-release.mjs.map +7 -0
- package/src/logic-functions/quarantine-release.ts +94 -0
- package/src/logic-functions/register.mjs +34 -1
- package/src/logic-functions/register.mjs.map +2 -2
- package/src/logic-functions/register.ts +105 -0
- package/src/logic-functions/replay-bulk.mjs +2040 -0
- package/src/logic-functions/replay-bulk.mjs.map +7 -0
- package/src/logic-functions/replay-bulk.ts +83 -0
- package/src/logic-functions/replay-log.mjs +1987 -0
- package/src/logic-functions/replay-log.mjs.map +7 -0
- package/src/logic-functions/replay-log.ts +54 -0
- package/src/logic-functions/retry.mjs +1004 -248
- package/src/logic-functions/retry.mjs.map +4 -4
- package/src/logic-functions/retry.ts +101 -0
- package/src/logic-functions/test-ingest.mjs +616 -28
- package/src/logic-functions/test-ingest.mjs.map +4 -4
- package/src/logic-functions/test-ingest.ts +179 -0
- package/src/logic-functions/webhook.mjs +968 -209
- package/src/logic-functions/webhook.mjs.map +4 -4
- package/src/logic-functions/webhook.ts +58 -0
- package/src/post-install.mjs +34 -1
- package/src/post-install.mjs.map +2 -2
- package/src/post-install.ts +81 -0
|
@@ -21,10 +21,6 @@ var __anyHandler = {
|
|
|
21
21
|
var __anyStub = new Proxy(() => void 0, __anyHandler);
|
|
22
22
|
var defineLogicFunction = __defineFactoryStub;
|
|
23
23
|
|
|
24
|
-
// src/logic-functions/webhook.ts
|
|
25
|
-
import { CoreApiClient } from "twenty-client-sdk/core";
|
|
26
|
-
import { MetadataApiClient } from "twenty-client-sdk/metadata";
|
|
27
|
-
|
|
28
24
|
// src/constants/universal-identifiers.ts
|
|
29
25
|
var IDS = {
|
|
30
26
|
APPLICATION: "96a389dd-384a-43a1-a376-9d3cf01b6ab7",
|
|
@@ -63,6 +59,7 @@ var IDS = {
|
|
|
63
59
|
INTAKE_FIELD_RULE_FIELD_TYPE: "72d1f6d8-d63e-419e-a244-8127d7b4705f",
|
|
64
60
|
INTAKE_FIELD_RULE_PRIORITY: "911df299-940b-44f9-92f7-3fd01fde01a7",
|
|
65
61
|
INTAKE_FIELD_RULE_IS_ACTIVE: "6f9cd96d-aecd-437f-a7b5-d45548782771",
|
|
62
|
+
INTAKE_FIELD_RULE_TARGET_OBJECT: "7b8411a7-5e1d-40f2-a20c-683c07f67926",
|
|
66
63
|
// Logic Functions
|
|
67
64
|
WEBHOOK_LOGIC_FUNCTION: "8323c257-48fe-4bb6-b1dd-166844f575c0",
|
|
68
65
|
HEALTH_LOGIC_FUNCTION: "4ece4e39-286b-4820-81f4-b7beee71d092",
|
|
@@ -88,9 +85,45 @@ var IDS = {
|
|
|
88
85
|
// Views & Navigation
|
|
89
86
|
SOURCES_VIEW: "4435cd43-5c13-472f-b4f9-b686c0f46285",
|
|
90
87
|
LOGS_VIEW: "a2cd0a93-9778-4ba5-99d9-b8b89c4fb39d",
|
|
91
|
-
INTAKE_NAV_ITEM: "e81bf288-5686-4713-86ae-e2947b124cab"
|
|
88
|
+
INTAKE_NAV_ITEM: "e81bf288-5686-4713-86ae-e2947b124cab",
|
|
89
|
+
// ── v0.4.0 ────────────────────────────────────────────────────────────────
|
|
90
|
+
// IntakeLog fields — quarantine, idempotency, replay lineage
|
|
91
|
+
INTAKE_LOG_IDEMPOTENCY_KEY: "bf91e4c0-6477-41f2-a9bd-fb159d3091d4",
|
|
92
|
+
INTAKE_LOG_SPAM_SCORE: "4e250af2-925f-4724-85f6-f353a6bb4cfd",
|
|
93
|
+
INTAKE_LOG_QUARANTINE_REASON: "323b3c3e-a111-45d1-a211-cf70efa6cd9e",
|
|
94
|
+
INTAKE_LOG_REPLAY_OF: "b68876b1-0045-4eaa-a4e7-f0634e110ef1",
|
|
95
|
+
// IntakeSource fields — silence detection, spam, merge policy
|
|
96
|
+
INTAKE_SOURCE_EXPECTED_CADENCE: "a97c3906-5555-4e60-a6d6-84155a4e00a6",
|
|
97
|
+
INTAKE_SOURCE_HEALTH_STATUS: "2ad13b20-5bd7-450a-b52d-bcf208373cf5",
|
|
98
|
+
INTAKE_SOURCE_SILENT_SINCE: "d3395601-f29f-4e57-98eb-561ab011cc72",
|
|
99
|
+
INTAKE_SOURCE_ALERT_WEBHOOK: "5a066d62-0426-41c1-b34c-1a7e881ce854",
|
|
100
|
+
INTAKE_SOURCE_LAST_ALERT_AT: "fc64a598-ccbe-44e3-bd42-716e32d626a2",
|
|
101
|
+
INTAKE_SOURCE_HONEYPOT_FIELD: "5c145686-50de-496c-bb45-a30c8b39034c",
|
|
102
|
+
INTAKE_SOURCE_MERGE_POLICY: "5a946f85-b1c2-493e-92c6-31c50e637a7e",
|
|
103
|
+
// IntakeFieldRule fields
|
|
104
|
+
INTAKE_FIELD_RULE_MERGE_STRATEGY: "40c6fdcb-527a-4691-8518-c40a3d3e34c6",
|
|
105
|
+
// Logic Functions
|
|
106
|
+
CONTRACT_LOGIC_FUNCTION: "2eea4152-1b08-4b8c-903c-f4c7ee7275d0",
|
|
107
|
+
REPLAY_LOG_LOGIC_FUNCTION: "8ff98390-7358-4a91-84f3-435f758182a3",
|
|
108
|
+
REPLAY_BULK_LOGIC_FUNCTION: "b82ed4c4-07bc-4095-8ee5-611200626dc0",
|
|
109
|
+
QUARANTINE_LIST_LOGIC_FUNCTION: "0203643c-2a97-4e7c-b4a5-b76d721cf23a",
|
|
110
|
+
QUARANTINE_RELEASE_LOGIC_FUNCTION: "5a10d4fe-c807-45c6-9e61-c191d4e37b4e",
|
|
111
|
+
QUARANTINE_DISCARD_LOGIC_FUNCTION: "1454a56e-6197-4dfd-9b41-5cc3b1436af7",
|
|
112
|
+
CHECK_SILENCE_LOGIC_FUNCTION: "e88cd5f2-ab96-4f89-ac55-78db66b32d66",
|
|
113
|
+
// Application variables
|
|
114
|
+
VAR_MERGE_POLICY: "a84fe9fd-3d7c-41d9-b0e8-dfdc1a87f348",
|
|
115
|
+
VAR_SPAM_FILTER_ENABLED: "570d3ba8-1904-4bad-a5ce-27e208e6414a",
|
|
116
|
+
VAR_SPAM_SCORE_THRESHOLD: "4395574d-24b4-4e0e-afa3-1e90fcd4f470",
|
|
117
|
+
VAR_HONEYPOT_FIELD: "2cf56c56-f5cb-47c5-aa43-84f301060a95",
|
|
118
|
+
VAR_RAW_PAYLOAD_RETENTION: "f3314ad6-05b9-4127-bb88-cea5f2018dc9",
|
|
119
|
+
VAR_RAW_PAYLOAD_MAX_BYTES: "b002ca76-df3d-47b4-8ac5-d3f562239c48",
|
|
120
|
+
VAR_REPLAY_MAX_BATCH: "cf55956f-177e-4afd-8d9f-7f42c614209c"
|
|
92
121
|
};
|
|
93
122
|
|
|
123
|
+
// src/lib/ingest.ts
|
|
124
|
+
import { CoreApiClient } from "twenty-client-sdk/core";
|
|
125
|
+
import { MetadataApiClient } from "twenty-client-sdk/metadata";
|
|
126
|
+
|
|
94
127
|
// src/lib/pipeline.ts
|
|
95
128
|
var Pipeline = class {
|
|
96
129
|
stages = [];
|
|
@@ -122,6 +155,10 @@ var ErrorCode = {
|
|
|
122
155
|
SOURCE_NOT_FOUND: "SOURCE_NOT_FOUND",
|
|
123
156
|
SOURCE_PAUSED: "SOURCE_PAUSED",
|
|
124
157
|
DUPLICATE_PAYLOAD: "DUPLICATE_PAYLOAD",
|
|
158
|
+
/** Held by the spam filter — recorded, not written, and releasable. */
|
|
159
|
+
QUARANTINED: "QUARANTINED",
|
|
160
|
+
/** Retention is off or the payload was too large, so there is nothing to replay. */
|
|
161
|
+
NO_STORED_PAYLOAD: "NO_STORED_PAYLOAD",
|
|
125
162
|
FIELD_CREATE_FAILED: "FIELD_CREATE_FAILED",
|
|
126
163
|
RECORD_UPSERT_FAILED: "RECORD_UPSERT_FAILED",
|
|
127
164
|
NOTE_CREATE_FAILED: "NOTE_CREATE_FAILED",
|
|
@@ -132,30 +169,29 @@ var ok = (value) => ({ ok: true, value });
|
|
|
132
169
|
var err = (code, message, details) => ({ ok: false, error: { code, message, details } });
|
|
133
170
|
|
|
134
171
|
// src/lib/stages/load-source.stage.ts
|
|
172
|
+
var BASE_FIELDS = {
|
|
173
|
+
id: true,
|
|
174
|
+
name: true,
|
|
175
|
+
slug: true,
|
|
176
|
+
targetObject: true,
|
|
177
|
+
webhookSecret: true,
|
|
178
|
+
status: true,
|
|
179
|
+
createOpportunity: true,
|
|
180
|
+
opportunityNameTemplate: true
|
|
181
|
+
};
|
|
182
|
+
var V4_FIELDS = {
|
|
183
|
+
...BASE_FIELDS,
|
|
184
|
+
honeypotField: true,
|
|
185
|
+
mergePolicy: true
|
|
186
|
+
};
|
|
135
187
|
var LoadSourceStage = class {
|
|
136
188
|
constructor(client) {
|
|
137
189
|
this.client = client;
|
|
138
190
|
}
|
|
191
|
+
client;
|
|
139
192
|
name = "load-source";
|
|
140
193
|
async execute(ctx) {
|
|
141
|
-
const
|
|
142
|
-
intakeSources: {
|
|
143
|
-
__args: { filter: { slug: { eq: ctx.sourceSlug } } },
|
|
144
|
-
edges: {
|
|
145
|
-
node: {
|
|
146
|
-
id: true,
|
|
147
|
-
name: true,
|
|
148
|
-
slug: true,
|
|
149
|
-
targetObject: true,
|
|
150
|
-
webhookSecret: true,
|
|
151
|
-
status: true,
|
|
152
|
-
createOpportunity: true,
|
|
153
|
-
opportunityNameTemplate: true
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
});
|
|
158
|
-
const node = result.intakeSources?.edges?.[0]?.node;
|
|
194
|
+
const node = await this.fetch(ctx.sourceSlug);
|
|
159
195
|
if (!node) {
|
|
160
196
|
return err(ErrorCode.SOURCE_NOT_FOUND, `No Intake source found with slug "${ctx.sourceSlug}"`);
|
|
161
197
|
}
|
|
@@ -172,11 +208,48 @@ var LoadSourceStage = class {
|
|
|
172
208
|
webhookSecret: node.webhookSecret ?? null,
|
|
173
209
|
status: node.status,
|
|
174
210
|
createOpportunity: node.createOpportunity ?? true,
|
|
175
|
-
opportunityNameTemplate: node.opportunityNameTemplate ?? "{{source}} \u2014 {{firstName}} {{lastName}}"
|
|
211
|
+
opportunityNameTemplate: node.opportunityNameTemplate ?? "{{source}} \u2014 {{firstName}} {{lastName}}",
|
|
212
|
+
honeypotField: node.honeypotField ?? null,
|
|
213
|
+
mergePolicy: parseMergePolicy(node.mergePolicy)
|
|
176
214
|
}
|
|
177
215
|
});
|
|
178
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* Ask for the 0.4.0 columns, and fall back to the original set if they are not
|
|
219
|
+
* there yet. A workspace that upgraded the package but has not re-synced its
|
|
220
|
+
* schema would otherwise fail the query and reject every request as an unknown
|
|
221
|
+
* source — an upgrade that takes the webhook offline, which is not acceptable
|
|
222
|
+
* for something sitting in front of someone's contact form.
|
|
223
|
+
*/
|
|
224
|
+
async fetch(slug) {
|
|
225
|
+
const args = { filter: { slug: { eq: slug } } };
|
|
226
|
+
try {
|
|
227
|
+
const result = await this.client.query({
|
|
228
|
+
intakeSources: { __args: args, edges: { node: V4_FIELDS } }
|
|
229
|
+
});
|
|
230
|
+
return result.intakeSources?.edges?.[0]?.node ?? null;
|
|
231
|
+
} catch {
|
|
232
|
+
try {
|
|
233
|
+
const result = await this.client.query({
|
|
234
|
+
intakeSources: { __args: args, edges: { node: BASE_FIELDS } }
|
|
235
|
+
});
|
|
236
|
+
return result.intakeSources?.edges?.[0]?.node ?? null;
|
|
237
|
+
} catch {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
179
242
|
};
|
|
243
|
+
function parseMergePolicy(raw) {
|
|
244
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
245
|
+
case "PRESERVE":
|
|
246
|
+
return "PRESERVE";
|
|
247
|
+
case "NEWEST_WINS":
|
|
248
|
+
return "NEWEST_WINS";
|
|
249
|
+
default:
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
180
253
|
|
|
181
254
|
// src/lib/stages/verify-signature.stage.ts
|
|
182
255
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
@@ -227,42 +300,116 @@ function withinWindow(processedAt, windowMs = 5 * 60 * 1e3) {
|
|
|
227
300
|
return Date.now() - processedAt.getTime() < windowMs;
|
|
228
301
|
}
|
|
229
302
|
|
|
303
|
+
// src/lib/settings.ts
|
|
304
|
+
function str(key, fallback) {
|
|
305
|
+
const v = process.env[key];
|
|
306
|
+
return v === void 0 || v === "" ? fallback : v;
|
|
307
|
+
}
|
|
308
|
+
function int(key, fallback) {
|
|
309
|
+
const parsed = parseInt(process.env[key] ?? "", 10);
|
|
310
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
311
|
+
}
|
|
312
|
+
function bool(key, fallback) {
|
|
313
|
+
const v = process.env[key];
|
|
314
|
+
if (v === void 0 || v === "") return fallback;
|
|
315
|
+
return v.toLowerCase() === "true";
|
|
316
|
+
}
|
|
317
|
+
function appLabel() {
|
|
318
|
+
return str("INTAKE_APP_LABEL", "Intake");
|
|
319
|
+
}
|
|
320
|
+
function defaultOpportunityStage() {
|
|
321
|
+
return str("INTAKE_DEFAULT_OPP_STAGE", "NEW");
|
|
322
|
+
}
|
|
323
|
+
function dedupWindowMinutes() {
|
|
324
|
+
return int("INTAKE_DEDUP_WINDOW_MINUTES", 5);
|
|
325
|
+
}
|
|
326
|
+
function mergePolicy() {
|
|
327
|
+
return str("INTAKE_MERGE_POLICY", "PRESERVE").toUpperCase() === "NEWEST_WINS" ? "NEWEST_WINS" : "PRESERVE";
|
|
328
|
+
}
|
|
329
|
+
function spamFilterEnabled() {
|
|
330
|
+
return bool("INTAKE_SPAM_FILTER_ENABLED", false);
|
|
331
|
+
}
|
|
332
|
+
function spamScoreThreshold() {
|
|
333
|
+
return int("INTAKE_SPAM_SCORE_THRESHOLD", 5);
|
|
334
|
+
}
|
|
335
|
+
function honeypotField() {
|
|
336
|
+
return str("INTAKE_HONEYPOT_FIELD", "");
|
|
337
|
+
}
|
|
338
|
+
function rawPayloadRetention() {
|
|
339
|
+
return str("INTAKE_RAW_PAYLOAD_RETENTION", "FULL").toUpperCase() === "NONE" ? "NONE" : "FULL";
|
|
340
|
+
}
|
|
341
|
+
function rawPayloadMaxBytes() {
|
|
342
|
+
return int("INTAKE_RAW_PAYLOAD_MAX_BYTES", 65e3);
|
|
343
|
+
}
|
|
344
|
+
|
|
230
345
|
// src/lib/stages/idempotency.stage.ts
|
|
231
346
|
var IdempotencyStage = class {
|
|
232
347
|
constructor(client) {
|
|
233
348
|
this.client = client;
|
|
234
349
|
}
|
|
350
|
+
client;
|
|
235
351
|
name = "idempotency";
|
|
236
352
|
async execute(ctx) {
|
|
237
353
|
const hash = computePayloadHash(ctx.rawPayload);
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
id: true,
|
|
249
|
-
status: true,
|
|
250
|
-
recordId: true,
|
|
251
|
-
processedAt: true
|
|
252
|
-
}
|
|
253
|
-
}
|
|
354
|
+
const next = { ...ctx, payloadHash: hash };
|
|
355
|
+
if (ctx.skipDedup) return ok(next);
|
|
356
|
+
if (ctx.idempotencyKey) {
|
|
357
|
+
const prior2 = await this.findByKey(ctx.idempotencyKey);
|
|
358
|
+
if (prior2) {
|
|
359
|
+
return err(
|
|
360
|
+
ErrorCode.DUPLICATE_PAYLOAD,
|
|
361
|
+
`Idempotency key already processed (log ${prior2.id})`,
|
|
362
|
+
{ priorLogId: prior2.id, priorRecordId: prior2.recordId, idempotencyKey: ctx.idempotencyKey }
|
|
363
|
+
);
|
|
254
364
|
}
|
|
255
|
-
}
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
365
|
+
}
|
|
366
|
+
const windowMins = dedupWindowMinutes();
|
|
367
|
+
if (windowMins <= 0) return ok(next);
|
|
368
|
+
const prior = await this.findByHash(hash);
|
|
369
|
+
if (prior?.processedAt && withinWindow(new Date(prior.processedAt), windowMins * 60 * 1e3)) {
|
|
259
370
|
return err(
|
|
260
371
|
ErrorCode.DUPLICATE_PAYLOAD,
|
|
261
|
-
`Duplicate payload \u2014 already processed ${prior.status.toLowerCase()} (log ${prior.id})`,
|
|
372
|
+
`Duplicate payload \u2014 already processed ${String(prior.status).toLowerCase()} (log ${prior.id})`,
|
|
262
373
|
{ priorLogId: prior.id, priorRecordId: prior.recordId }
|
|
263
374
|
);
|
|
264
375
|
}
|
|
265
|
-
return ok(
|
|
376
|
+
return ok(next);
|
|
377
|
+
}
|
|
378
|
+
async findByKey(key) {
|
|
379
|
+
try {
|
|
380
|
+
const res = await this.client.query({
|
|
381
|
+
intakeLogs: {
|
|
382
|
+
__args: {
|
|
383
|
+
filter: { idempotencyKey: { eq: key } },
|
|
384
|
+
orderBy: { processedAt: "DescNullsLast" },
|
|
385
|
+
first: 1
|
|
386
|
+
},
|
|
387
|
+
edges: { node: { id: true, status: true, recordId: true, processedAt: true } }
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
const node = res.intakeLogs?.edges?.[0]?.node ?? null;
|
|
391
|
+
if (node && (node.status === "QUARANTINED" || node.status === "DISCARDED")) return null;
|
|
392
|
+
return node;
|
|
393
|
+
} catch {
|
|
394
|
+
return null;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
async findByHash(hash) {
|
|
398
|
+
try {
|
|
399
|
+
const res = await this.client.query({
|
|
400
|
+
intakeLogs: {
|
|
401
|
+
__args: {
|
|
402
|
+
filter: { payloadHash: { eq: hash } },
|
|
403
|
+
orderBy: { processedAt: "DescNullsLast" },
|
|
404
|
+
first: 1
|
|
405
|
+
},
|
|
406
|
+
edges: { node: { id: true, status: true, recordId: true, processedAt: true } }
|
|
407
|
+
}
|
|
408
|
+
});
|
|
409
|
+
return res.intakeLogs?.edges?.[0]?.node ?? null;
|
|
410
|
+
} catch {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
266
413
|
}
|
|
267
414
|
};
|
|
268
415
|
|
|
@@ -540,7 +687,113 @@ function classify(key, value) {
|
|
|
540
687
|
return "field";
|
|
541
688
|
}
|
|
542
689
|
|
|
690
|
+
// src/lib/merge.ts
|
|
691
|
+
function decideMerge(existing, incoming, policy, strategy = "INHERIT") {
|
|
692
|
+
if (isEmpty(incoming)) return { write: false, status: "preserved" };
|
|
693
|
+
if (isEmpty(existing)) return { write: true, status: "filled-empty" };
|
|
694
|
+
if (sameValue(existing, incoming)) return { write: false, status: "unchanged" };
|
|
695
|
+
const effective = strategy === "INHERIT" ? policy : strategy;
|
|
696
|
+
return effective === "NEWEST_WINS" ? { write: true, status: "set" } : { write: false, status: "preserved" };
|
|
697
|
+
}
|
|
698
|
+
function mergePatch(existing, incoming, policy, strategyFor = () => "INHERIT") {
|
|
699
|
+
const patch = {};
|
|
700
|
+
const decisions = {};
|
|
701
|
+
for (const [key, value] of Object.entries(incoming)) {
|
|
702
|
+
const decision = decideMerge(existing[key], value, policy, strategyFor(key));
|
|
703
|
+
decisions[key] = decision;
|
|
704
|
+
if (decision.write) patch[key] = value;
|
|
705
|
+
}
|
|
706
|
+
return { patch, decisions };
|
|
707
|
+
}
|
|
708
|
+
function parseFieldMergeStrategy(raw) {
|
|
709
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
710
|
+
case "PRESERVE":
|
|
711
|
+
return "PRESERVE";
|
|
712
|
+
case "NEWEST_WINS":
|
|
713
|
+
return "NEWEST_WINS";
|
|
714
|
+
default:
|
|
715
|
+
return "INHERIT";
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
function isEmpty(value) {
|
|
719
|
+
if (value === null || value === void 0) return true;
|
|
720
|
+
if (typeof value === "string") return value.trim() === "";
|
|
721
|
+
if (Array.isArray(value)) return value.length === 0;
|
|
722
|
+
if (typeof value === "object") {
|
|
723
|
+
const entries = Object.entries(value);
|
|
724
|
+
if (entries.length === 0) return true;
|
|
725
|
+
const meaningful = entries.filter(([k]) => !IGNORED_COMPOSITE_KEYS.has(k));
|
|
726
|
+
if (meaningful.length === 0) return true;
|
|
727
|
+
return meaningful.every(([, v]) => isEmpty(v));
|
|
728
|
+
}
|
|
729
|
+
return false;
|
|
730
|
+
}
|
|
731
|
+
var IGNORED_COMPOSITE_KEYS = /* @__PURE__ */ new Set([
|
|
732
|
+
"currencyCode",
|
|
733
|
+
"primaryLinkLabel",
|
|
734
|
+
"primaryPhoneCountryCode",
|
|
735
|
+
"primaryPhoneCallingCode",
|
|
736
|
+
"additionalLinks",
|
|
737
|
+
"additionalEmails",
|
|
738
|
+
"additionalPhones"
|
|
739
|
+
]);
|
|
740
|
+
function sameValue(a, b) {
|
|
741
|
+
if (a === b) return true;
|
|
742
|
+
if (typeof a === "object" && typeof b === "object" && a !== null && b !== null) {
|
|
743
|
+
try {
|
|
744
|
+
return JSON.stringify(sortKeys(a)) === JSON.stringify(sortKeys(b));
|
|
745
|
+
} catch {
|
|
746
|
+
return false;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
return String(a) === String(b);
|
|
750
|
+
}
|
|
751
|
+
function sortKeys(value) {
|
|
752
|
+
if (value === null || typeof value !== "object") return value;
|
|
753
|
+
if (Array.isArray(value)) return value.map(sortKeys);
|
|
754
|
+
const obj = value;
|
|
755
|
+
const out = {};
|
|
756
|
+
for (const key of Object.keys(obj).sort()) out[key] = sortKeys(obj[key]);
|
|
757
|
+
return out;
|
|
758
|
+
}
|
|
759
|
+
|
|
543
760
|
// src/lib/normalizer.ts
|
|
761
|
+
function actionForRule(rule) {
|
|
762
|
+
if (rule.action) return rule.action;
|
|
763
|
+
switch ((rule.fieldType ?? "").toUpperCase()) {
|
|
764
|
+
case "SKIP":
|
|
765
|
+
return "skip";
|
|
766
|
+
case "NOTE":
|
|
767
|
+
return "note";
|
|
768
|
+
default:
|
|
769
|
+
return "field";
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
function typeForRule(rule) {
|
|
773
|
+
switch ((rule.fieldType ?? "").toUpperCase()) {
|
|
774
|
+
case "NOTE":
|
|
775
|
+
return "RICH_TEXT";
|
|
776
|
+
case "SKIP":
|
|
777
|
+
return "TEXT";
|
|
778
|
+
case "":
|
|
779
|
+
return "TEXT";
|
|
780
|
+
default:
|
|
781
|
+
return rule.fieldType;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
var OPPORTUNITY_KEY_PREFIXES = ["opportunity_", "opp_", "deal_"];
|
|
785
|
+
function parseTargetObject(raw) {
|
|
786
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
787
|
+
case "PERSON":
|
|
788
|
+
return "person";
|
|
789
|
+
case "COMPANY":
|
|
790
|
+
return "company";
|
|
791
|
+
case "OPPORTUNITY":
|
|
792
|
+
return "opportunity";
|
|
793
|
+
default:
|
|
794
|
+
return "auto";
|
|
795
|
+
}
|
|
796
|
+
}
|
|
544
797
|
function normalizeFields(flat, rules = []) {
|
|
545
798
|
const sorted = [...rules].sort((a, b) => b.priority - a.priority);
|
|
546
799
|
return Object.entries(flat).map(
|
|
@@ -557,7 +810,8 @@ function normalizeField(key, value, rules) {
|
|
|
557
810
|
twentyType: builtIn.twentyType,
|
|
558
811
|
action: builtIn.action,
|
|
559
812
|
value,
|
|
560
|
-
source: "builtin"
|
|
813
|
+
source: "builtin",
|
|
814
|
+
targetObject: "auto"
|
|
561
815
|
};
|
|
562
816
|
}
|
|
563
817
|
for (const rule of rules) {
|
|
@@ -571,23 +825,29 @@ function normalizeField(key, value, rules) {
|
|
|
571
825
|
return {
|
|
572
826
|
originalKey: key,
|
|
573
827
|
canonicalName: rule.canonicalName,
|
|
574
|
-
twentyType: rule
|
|
575
|
-
action: rule
|
|
828
|
+
twentyType: typeForRule(rule),
|
|
829
|
+
action: actionForRule(rule),
|
|
576
830
|
value,
|
|
577
|
-
source: "rule"
|
|
831
|
+
source: "rule",
|
|
832
|
+
targetObject: parseTargetObject(rule.targetObject),
|
|
833
|
+
mergeStrategy: parseFieldMergeStrategy(rule.mergeStrategy)
|
|
578
834
|
};
|
|
579
835
|
}
|
|
580
836
|
}
|
|
581
837
|
const detectedType = detectType(value);
|
|
582
838
|
const classification = classify(key, value);
|
|
583
|
-
const
|
|
839
|
+
const prefix = OPPORTUNITY_KEY_PREFIXES.find((p) => lookupKey.startsWith(p));
|
|
840
|
+
const targetObject = prefix ? "opportunity" : "auto";
|
|
841
|
+
const bareKey = prefix ? lookupKey.slice(prefix.length) : key;
|
|
842
|
+
const canonicalName = classification === "field" ? toExtFieldName(bareKey) : lookupKey;
|
|
584
843
|
return {
|
|
585
844
|
originalKey: key,
|
|
586
845
|
canonicalName,
|
|
587
846
|
twentyType: detectedType,
|
|
588
847
|
action: classification === "field" ? "field" : classification,
|
|
589
848
|
value,
|
|
590
|
-
source: "passthrough"
|
|
849
|
+
source: "passthrough",
|
|
850
|
+
targetObject
|
|
591
851
|
};
|
|
592
852
|
}
|
|
593
853
|
function toExtFieldName(key) {
|
|
@@ -681,6 +941,7 @@ var NormalizeStage = class {
|
|
|
681
941
|
constructor(client) {
|
|
682
942
|
this.client = client;
|
|
683
943
|
}
|
|
944
|
+
client;
|
|
684
945
|
name = "normalize";
|
|
685
946
|
async execute(ctx) {
|
|
686
947
|
if (!ctx.flat) {
|
|
@@ -697,6 +958,32 @@ var NormalizeStage = class {
|
|
|
697
958
|
});
|
|
698
959
|
}
|
|
699
960
|
async loadRules(sourceId) {
|
|
961
|
+
try {
|
|
962
|
+
const filter = sourceId ? { or: [{ intakeSource: { id: { eq: sourceId } } }, { intakeSource: { id: { is: "NULL" } } }] } : { intakeSource: { id: { is: "NULL" } } };
|
|
963
|
+
const result = await this.client.query({
|
|
964
|
+
intakeFieldRules: {
|
|
965
|
+
__args: {
|
|
966
|
+
filter: { and: [filter, { isActive: { eq: true } }] },
|
|
967
|
+
orderBy: { priority: "DescNullsLast" }
|
|
968
|
+
},
|
|
969
|
+
edges: {
|
|
970
|
+
node: {
|
|
971
|
+
inputPattern: true,
|
|
972
|
+
canonicalName: true,
|
|
973
|
+
fieldType: true,
|
|
974
|
+
priority: true,
|
|
975
|
+
targetObject: true,
|
|
976
|
+
mergeStrategy: true
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
return (result.intakeFieldRules?.edges ?? []).map((e) => e.node);
|
|
982
|
+
} catch {
|
|
983
|
+
return this.loadRulesLegacy(sourceId);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
async loadRulesLegacy(sourceId) {
|
|
700
987
|
try {
|
|
701
988
|
const filter = sourceId ? { or: [{ intakeSource: { id: { eq: sourceId } } }, { intakeSource: { id: { is: "NULL" } } }] } : { intakeSource: { id: { is: "NULL" } } };
|
|
702
989
|
const result = await this.client.query({
|
|
@@ -722,6 +1009,204 @@ var NormalizeStage = class {
|
|
|
722
1009
|
}
|
|
723
1010
|
};
|
|
724
1011
|
|
|
1012
|
+
// src/lib/spam.ts
|
|
1013
|
+
var HONEYPOT_SCORE = 100;
|
|
1014
|
+
var DISPOSABLE_EMAIL_DOMAINS = /* @__PURE__ */ new Set([
|
|
1015
|
+
"mailinator.com",
|
|
1016
|
+
"guerrillamail.com",
|
|
1017
|
+
"guerrillamail.net",
|
|
1018
|
+
"sharklasers.com",
|
|
1019
|
+
"10minutemail.com",
|
|
1020
|
+
"10minutemail.net",
|
|
1021
|
+
"tempmail.com",
|
|
1022
|
+
"temp-mail.org",
|
|
1023
|
+
"throwawaymail.com",
|
|
1024
|
+
"yopmail.com",
|
|
1025
|
+
"yopmail.net",
|
|
1026
|
+
"trashmail.com",
|
|
1027
|
+
"dispostable.com",
|
|
1028
|
+
"getnada.com",
|
|
1029
|
+
"nada.email",
|
|
1030
|
+
"fakeinbox.com",
|
|
1031
|
+
"maildrop.cc",
|
|
1032
|
+
"mintemail.com",
|
|
1033
|
+
"spamgourmet.com",
|
|
1034
|
+
"mailnesia.com",
|
|
1035
|
+
"tempinbox.com",
|
|
1036
|
+
"emailondeck.com",
|
|
1037
|
+
"moakt.com",
|
|
1038
|
+
"mohmal.com",
|
|
1039
|
+
"inboxkitten.com",
|
|
1040
|
+
"mailcatch.com",
|
|
1041
|
+
"mytemp.email",
|
|
1042
|
+
"discard.email"
|
|
1043
|
+
]);
|
|
1044
|
+
var PLACEHOLDER_EMAIL_DOMAINS = /* @__PURE__ */ new Set([
|
|
1045
|
+
"example.com",
|
|
1046
|
+
"example.org",
|
|
1047
|
+
"example.net",
|
|
1048
|
+
"test.com",
|
|
1049
|
+
"localhost",
|
|
1050
|
+
"domain.com",
|
|
1051
|
+
"email.com",
|
|
1052
|
+
"yourdomain.com"
|
|
1053
|
+
]);
|
|
1054
|
+
var URL_RE2 = /https?:\/\/|www\./i;
|
|
1055
|
+
var GLOBAL_URL_RE = /https?:\/\/[^\s<>"']+/gi;
|
|
1056
|
+
var MARKUP_RE = /<\s*a\s|<\s*script|\[url[=\]]|\[link[=\]]|\]\(https?:/i;
|
|
1057
|
+
var EMAIL_RE2 = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/;
|
|
1058
|
+
var NAME_KEYS = /* @__PURE__ */ new Set([
|
|
1059
|
+
"name",
|
|
1060
|
+
"fullName",
|
|
1061
|
+
"firstName",
|
|
1062
|
+
"lastName",
|
|
1063
|
+
"contact_name",
|
|
1064
|
+
"your_name"
|
|
1065
|
+
]);
|
|
1066
|
+
var MESSAGE_KEYS = /* @__PURE__ */ new Set([
|
|
1067
|
+
"message",
|
|
1068
|
+
"description",
|
|
1069
|
+
"comments",
|
|
1070
|
+
"notes",
|
|
1071
|
+
"body",
|
|
1072
|
+
"details",
|
|
1073
|
+
"content",
|
|
1074
|
+
"text",
|
|
1075
|
+
"inquiry",
|
|
1076
|
+
"summary",
|
|
1077
|
+
"feedback"
|
|
1078
|
+
]);
|
|
1079
|
+
function scoreSpam(flat, honeypotKey) {
|
|
1080
|
+
const signals = [];
|
|
1081
|
+
if (honeypotKey) {
|
|
1082
|
+
const hit = Object.entries(flat).find(
|
|
1083
|
+
([k, v]) => k.toLowerCase() === honeypotKey.toLowerCase() && isFilled(v)
|
|
1084
|
+
);
|
|
1085
|
+
if (hit) {
|
|
1086
|
+
return {
|
|
1087
|
+
score: HONEYPOT_SCORE,
|
|
1088
|
+
signals: [{ reason: `Honeypot field "${hit[0]}" was filled`, points: HONEYPOT_SCORE }],
|
|
1089
|
+
honeypot: true
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
const add = (reason, points) => signals.push({ reason, points });
|
|
1094
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1095
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1096
|
+
const bare = key.replace(/^(opportunity_|opp_|deal_)/i, "");
|
|
1097
|
+
if (NAME_KEYS.has(bare) || NAME_KEYS.has(key)) {
|
|
1098
|
+
if (URL_RE2.test(value)) add(`Name field "${key}" contains a URL`, 4);
|
|
1099
|
+
if (MARKUP_RE.test(value)) add(`Name field "${key}" contains markup or a link tag`, 4);
|
|
1100
|
+
}
|
|
1101
|
+
if (MESSAGE_KEYS.has(bare) || MESSAGE_KEYS.has(key)) {
|
|
1102
|
+
const urls = value.match(GLOBAL_URL_RE) ?? [];
|
|
1103
|
+
const distinct = new Set(urls.map((u) => u.toLowerCase()));
|
|
1104
|
+
if (distinct.size >= 5) add(`Message field "${key}" contains ${distinct.size} links`, 3);
|
|
1105
|
+
if (MARKUP_RE.test(value)) add(`Message field "${key}" contains anchor or BBCode markup`, 2);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const email = firstEmail(flat);
|
|
1109
|
+
if (email !== null) {
|
|
1110
|
+
if (!EMAIL_RE2.test(email)) {
|
|
1111
|
+
add(`Email "${truncate(email)}" is not a valid address`, 3);
|
|
1112
|
+
} else {
|
|
1113
|
+
const domain = email.slice(email.lastIndexOf("@") + 1).toLowerCase();
|
|
1114
|
+
if (DISPOSABLE_EMAIL_DOMAINS.has(domain)) {
|
|
1115
|
+
add(`Email domain "${domain}" is a disposable inbox`, 4);
|
|
1116
|
+
} else if (PLACEHOLDER_EMAIL_DOMAINS.has(domain)) {
|
|
1117
|
+
add(`Email domain "${domain}" is a placeholder domain`, 3);
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
const repeated = mostRepeatedValue(flat);
|
|
1122
|
+
if (repeated && repeated.count >= 4) {
|
|
1123
|
+
add(`The same value appears in ${repeated.count} different fields`, 2);
|
|
1124
|
+
}
|
|
1125
|
+
const phone = firstPhone(flat);
|
|
1126
|
+
if (phone && isImplausiblePhone(phone)) {
|
|
1127
|
+
add(`Phone "${truncate(phone)}" is a placeholder number`, 2);
|
|
1128
|
+
}
|
|
1129
|
+
const score = signals.reduce((sum, s) => sum + s.points, 0);
|
|
1130
|
+
return { score, signals, honeypot: false };
|
|
1131
|
+
}
|
|
1132
|
+
function describeVerdict(verdict) {
|
|
1133
|
+
return verdict.signals.map((s) => s.reason).join("; ").slice(0, 500);
|
|
1134
|
+
}
|
|
1135
|
+
function isFilled(value) {
|
|
1136
|
+
if (value === null || value === void 0) return false;
|
|
1137
|
+
if (typeof value === "string") return value.trim() !== "";
|
|
1138
|
+
if (typeof value === "boolean") return value;
|
|
1139
|
+
return true;
|
|
1140
|
+
}
|
|
1141
|
+
function firstEmail(flat) {
|
|
1142
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1143
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1144
|
+
if (/mail/i.test(key)) return value.trim();
|
|
1145
|
+
}
|
|
1146
|
+
return null;
|
|
1147
|
+
}
|
|
1148
|
+
function firstPhone(flat) {
|
|
1149
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1150
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1151
|
+
if (/phone|mobile|tel|cell/i.test(key)) return value.trim();
|
|
1152
|
+
}
|
|
1153
|
+
return null;
|
|
1154
|
+
}
|
|
1155
|
+
function isImplausiblePhone(raw) {
|
|
1156
|
+
const digits = raw.replace(/\D/g, "");
|
|
1157
|
+
if (digits.length < 7) return false;
|
|
1158
|
+
if (/^(\d)\1+$/.test(digits)) return true;
|
|
1159
|
+
const ascending = digits.split("").every((d, i, arr) => i === 0 || Number(d) === (Number(arr[i - 1]) + 1) % 10);
|
|
1160
|
+
return ascending;
|
|
1161
|
+
}
|
|
1162
|
+
function mostRepeatedValue(flat) {
|
|
1163
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1164
|
+
for (const value of Object.values(flat)) {
|
|
1165
|
+
if (typeof value !== "string") continue;
|
|
1166
|
+
const v = value.trim();
|
|
1167
|
+
if (v.length < 8) continue;
|
|
1168
|
+
counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
1169
|
+
}
|
|
1170
|
+
let best = null;
|
|
1171
|
+
for (const [value, count] of counts) {
|
|
1172
|
+
if (!best || count > best.count) best = { value, count };
|
|
1173
|
+
}
|
|
1174
|
+
return best;
|
|
1175
|
+
}
|
|
1176
|
+
function truncate(s) {
|
|
1177
|
+
return s.length > 60 ? `${s.slice(0, 60)}\u2026` : s;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
// src/lib/stages/quarantine.stage.ts
|
|
1181
|
+
var QuarantineStage = class {
|
|
1182
|
+
name = "quarantine";
|
|
1183
|
+
async execute(ctx) {
|
|
1184
|
+
const honeypot = ctx.source?.honeypotField || honeypotField();
|
|
1185
|
+
const filterOn = spamFilterEnabled();
|
|
1186
|
+
if (!honeypot && !filterOn) return ok(ctx);
|
|
1187
|
+
const verdict = scoreSpam(ctx.flat ?? {}, honeypot);
|
|
1188
|
+
const next = { ...ctx, spam: verdict };
|
|
1189
|
+
if (ctx.skipSpamFilter) {
|
|
1190
|
+
if (verdict.score > 0) {
|
|
1191
|
+
next.warnings.push(`Spam filter overruled by release \u2014 score was ${verdict.score}: ${describeVerdict(verdict)}`);
|
|
1192
|
+
}
|
|
1193
|
+
return ok(next);
|
|
1194
|
+
}
|
|
1195
|
+
const quarantine = verdict.honeypot || filterOn && verdict.score >= spamScoreThreshold();
|
|
1196
|
+
if (quarantine) {
|
|
1197
|
+
return err(
|
|
1198
|
+
ErrorCode.QUARANTINED,
|
|
1199
|
+
verdict.honeypot ? "Honeypot field was filled \u2014 payload quarantined" : `Spam score ${verdict.score} met the threshold of ${spamScoreThreshold()} \u2014 payload quarantined`,
|
|
1200
|
+
{ spamScore: verdict.score, reason: describeVerdict(verdict), signals: verdict.signals }
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
if (verdict.score > 0) {
|
|
1204
|
+
next.warnings.push(`Spam score ${verdict.score} (below threshold): ${describeVerdict(verdict)}`);
|
|
1205
|
+
}
|
|
1206
|
+
return ok(next);
|
|
1207
|
+
}
|
|
1208
|
+
};
|
|
1209
|
+
|
|
725
1210
|
// src/lib/rest-client.ts
|
|
726
1211
|
var apiUrl = () => process.env["TWENTY_API_URL"] ?? "";
|
|
727
1212
|
var token = () => process.env["TWENTY_APP_ACCESS_TOKEN"] ?? "";
|
|
@@ -770,66 +1255,122 @@ var STANDARD_FIELDS = /* @__PURE__ */ new Set([
|
|
|
770
1255
|
"city",
|
|
771
1256
|
"employees",
|
|
772
1257
|
"annualRecurringRevenue",
|
|
773
|
-
"idealCustomerProfile"
|
|
1258
|
+
"idealCustomerProfile",
|
|
1259
|
+
...Object.values(FIELD_MAP).map((e) => e.canonicalName)
|
|
774
1260
|
]);
|
|
1261
|
+
var STANDARD_OPPORTUNITY_FIELDS = /* @__PURE__ */ new Set(["name", "stage", "amount", "closeDate"]);
|
|
1262
|
+
var EXT_RE = /^ext[A-Z]/;
|
|
775
1263
|
var SyncSchemaStage = class {
|
|
776
1264
|
name = "sync-schema";
|
|
777
1265
|
objectCache = /* @__PURE__ */ new Map();
|
|
778
1266
|
TTL = 5 * 60 * 1e3;
|
|
779
1267
|
async execute(ctx) {
|
|
780
1268
|
if (!ctx.crmFields || ctx.crmFields.length === 0) return ok(ctx);
|
|
781
|
-
|
|
1269
|
+
const primary = this.primaryTarget(ctx);
|
|
1270
|
+
const creationEnabled = process.env["INTAKE_FIELD_CREATION_ENABLED"] !== "false";
|
|
1271
|
+
if (!creationEnabled) {
|
|
782
1272
|
ctx.warnings.push("Field creation disabled by INTAKE_FIELD_CREATION_ENABLED setting \u2014 ext fields skipped");
|
|
783
|
-
return ok(ctx);
|
|
784
1273
|
}
|
|
785
|
-
const
|
|
1274
|
+
const resolved = ctx.crmFields.map((f) => ({
|
|
1275
|
+
...f,
|
|
1276
|
+
targetObject: this.resolveTarget(f.targetObject, primary, ctx)
|
|
1277
|
+
}));
|
|
786
1278
|
let fieldsCreated = 0;
|
|
787
1279
|
let fieldsMatched = 0;
|
|
788
|
-
|
|
1280
|
+
const demoted = [];
|
|
1281
|
+
for (const objectName of this.targetObjects(resolved)) {
|
|
1282
|
+
const fields = resolved.filter((f) => f.targetObject === objectName);
|
|
1283
|
+
if (fields.length === 0) continue;
|
|
1284
|
+
const standardForObject = objectName === "opportunity" ? STANDARD_OPPORTUNITY_FIELDS : STANDARD_FIELDS;
|
|
789
1285
|
const meta = await this.getObjectMeta(objectName);
|
|
790
|
-
if (!meta)
|
|
1286
|
+
if (!meta) {
|
|
1287
|
+
for (const f of fields) {
|
|
1288
|
+
if (standardForObject.has(f.canonicalName)) {
|
|
1289
|
+
f.writable = true;
|
|
1290
|
+
fieldsMatched++;
|
|
1291
|
+
continue;
|
|
1292
|
+
}
|
|
1293
|
+
ctx.warnings.push(`Could not read schema for ${objectName} \u2014 "${f.canonicalName}" goes to note`);
|
|
1294
|
+
demoted.push(f);
|
|
1295
|
+
}
|
|
1296
|
+
continue;
|
|
1297
|
+
}
|
|
791
1298
|
const maxExt = parseInt(process.env["INTAKE_MAX_EXT_FIELDS"] ?? "50", 10);
|
|
792
|
-
|
|
793
|
-
for (const field of
|
|
794
|
-
|
|
1299
|
+
let extCount = [...meta.fields].filter((f) => EXT_RE.test(f)).length;
|
|
1300
|
+
for (const field of fields) {
|
|
1301
|
+
const name = field.canonicalName;
|
|
1302
|
+
if (standardForObject.has(name)) {
|
|
1303
|
+
field.writable = true;
|
|
795
1304
|
fieldsMatched++;
|
|
796
1305
|
continue;
|
|
797
1306
|
}
|
|
798
|
-
if (
|
|
1307
|
+
if (meta.fields.has(name)) {
|
|
1308
|
+
field.writable = true;
|
|
799
1309
|
fieldsMatched++;
|
|
800
1310
|
continue;
|
|
801
1311
|
}
|
|
802
|
-
if (
|
|
803
|
-
|
|
1312
|
+
if (!EXT_RE.test(name)) {
|
|
1313
|
+
ctx.warnings.push(
|
|
1314
|
+
`Field "${name}" does not exist on ${objectName} and is not ext-prefixed \u2014 value routed to note. Create the field in Twenty first, or rename the rule's canonical name to ext${name.charAt(0).toUpperCase()}${name.slice(1)}.`
|
|
1315
|
+
);
|
|
1316
|
+
demoted.push(field);
|
|
1317
|
+
continue;
|
|
1318
|
+
}
|
|
1319
|
+
if (!creationEnabled) {
|
|
1320
|
+
demoted.push(field);
|
|
804
1321
|
continue;
|
|
805
1322
|
}
|
|
806
|
-
if (
|
|
807
|
-
ctx.warnings.push(`ext field limit (${maxExt}) reached for ${objectName} \u2014 "${
|
|
1323
|
+
if (extCount >= maxExt) {
|
|
1324
|
+
ctx.warnings.push(`ext field limit (${maxExt}) reached for ${objectName} \u2014 "${name}" goes to note`);
|
|
1325
|
+
demoted.push(field);
|
|
808
1326
|
continue;
|
|
809
1327
|
}
|
|
810
|
-
const created = await this.createField(meta.id,
|
|
1328
|
+
const created = await this.createField(meta.id, name, field.twentyType);
|
|
811
1329
|
if (created) {
|
|
812
|
-
meta.fields.add(
|
|
813
|
-
this.objectCache.get(objectName)?.fields.add(
|
|
1330
|
+
meta.fields.add(name);
|
|
1331
|
+
this.objectCache.get(objectName)?.fields.add(name);
|
|
1332
|
+
extCount++;
|
|
814
1333
|
fieldsCreated++;
|
|
1334
|
+
field.writable = true;
|
|
815
1335
|
} else {
|
|
816
|
-
ctx.warnings.push(`Could not create field "${
|
|
1336
|
+
ctx.warnings.push(`Could not create field "${name}" on ${objectName} \u2014 will go to note`);
|
|
1337
|
+
demoted.push(field);
|
|
817
1338
|
}
|
|
818
1339
|
}
|
|
819
1340
|
}
|
|
820
|
-
|
|
1341
|
+
const demotedSet = new Set(demoted);
|
|
1342
|
+
return ok({
|
|
1343
|
+
...ctx,
|
|
1344
|
+
crmFields: resolved.filter((f) => !demotedSet.has(f)),
|
|
1345
|
+
noteFields: [...ctx.noteFields ?? [], ...demoted],
|
|
1346
|
+
fieldsCreated,
|
|
1347
|
+
fieldsMatched
|
|
1348
|
+
});
|
|
821
1349
|
}
|
|
822
|
-
|
|
1350
|
+
/** Where an `auto` field lands: the person, or the company for company-only payloads. */
|
|
1351
|
+
primaryTarget(ctx) {
|
|
823
1352
|
const t = ctx.source?.targetObject ?? "AUTO";
|
|
824
|
-
if (t === "PERSON") return
|
|
825
|
-
if (t === "COMPANY") return
|
|
826
|
-
if (ctx.structuredCompany) return
|
|
1353
|
+
if (t === "PERSON") return "person";
|
|
1354
|
+
if (t === "COMPANY") return "company";
|
|
1355
|
+
if (ctx.structuredCompany) return "company";
|
|
827
1356
|
const hasPersonData = ctx.crmFields?.some((f) => ["firstName", "lastName", "email", "phone", "fullName", "emails", "phones", "name"].includes(f.canonicalName));
|
|
828
1357
|
const hasCompanyData = ctx.crmFields?.some((f) => ["companyName", "domainName"].includes(f.canonicalName));
|
|
829
|
-
|
|
830
|
-
if (
|
|
831
|
-
|
|
832
|
-
|
|
1358
|
+
if (hasPersonData) return "person";
|
|
1359
|
+
if (hasCompanyData) return "company";
|
|
1360
|
+
return "person";
|
|
1361
|
+
}
|
|
1362
|
+
resolveTarget(target, primary, ctx) {
|
|
1363
|
+
if (target === "auto") return primary;
|
|
1364
|
+
if (target === "opportunity" && !ctx.source?.createOpportunity) {
|
|
1365
|
+
ctx.warnings.push("Field routed to opportunity but this source does not create opportunities \u2014 written to the primary record instead");
|
|
1366
|
+
return primary;
|
|
1367
|
+
}
|
|
1368
|
+
return target;
|
|
1369
|
+
}
|
|
1370
|
+
targetObjects(fields) {
|
|
1371
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1372
|
+
for (const f of fields) if (f.targetObject !== "auto") seen.add(f.targetObject);
|
|
1373
|
+
return [...seen];
|
|
833
1374
|
}
|
|
834
1375
|
async getObjectMeta(objectName) {
|
|
835
1376
|
const cached = this.objectCache.get(objectName);
|
|
@@ -902,7 +1443,69 @@ function formatValue(value) {
|
|
|
902
1443
|
return String(value);
|
|
903
1444
|
}
|
|
904
1445
|
|
|
1446
|
+
// src/lib/lookup.ts
|
|
1447
|
+
async function firstMatch(collection, query) {
|
|
1448
|
+
try {
|
|
1449
|
+
const res = await coreApi.get(`/${collection}?${query}&first=1`);
|
|
1450
|
+
return res.data?.[collection]?.edges?.[0]?.node ?? null;
|
|
1451
|
+
} catch {
|
|
1452
|
+
return null;
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
async function findPersonByEmail(email) {
|
|
1456
|
+
if (!email) return null;
|
|
1457
|
+
return firstMatch("people", `filter=emails.primaryEmail[eq]:${encodeURIComponent(email)}`);
|
|
1458
|
+
}
|
|
1459
|
+
async function findCompanyByDomain(domain) {
|
|
1460
|
+
if (!domain) return null;
|
|
1461
|
+
const host = extractHost(domain);
|
|
1462
|
+
if (!host) return null;
|
|
1463
|
+
return firstMatch(
|
|
1464
|
+
"companies",
|
|
1465
|
+
`filter=domainName.primaryLinkUrl[like]:%25${encodeURIComponent(host)}%25`
|
|
1466
|
+
);
|
|
1467
|
+
}
|
|
1468
|
+
function extractHost(url) {
|
|
1469
|
+
try {
|
|
1470
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
1471
|
+
} catch {
|
|
1472
|
+
return url;
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
905
1476
|
// src/lib/stages/upsert-record.stage.ts
|
|
1477
|
+
var STANDARD_BY_OBJECT = {
|
|
1478
|
+
person: /* @__PURE__ */ new Set([
|
|
1479
|
+
"name",
|
|
1480
|
+
"emails",
|
|
1481
|
+
"phones",
|
|
1482
|
+
"firstName",
|
|
1483
|
+
"lastName",
|
|
1484
|
+
"fullName",
|
|
1485
|
+
"email",
|
|
1486
|
+
"phone",
|
|
1487
|
+
"linkedInLink",
|
|
1488
|
+
"xLink",
|
|
1489
|
+
"jobTitle",
|
|
1490
|
+
"city",
|
|
1491
|
+
"companyName",
|
|
1492
|
+
"domainName",
|
|
1493
|
+
"address"
|
|
1494
|
+
]),
|
|
1495
|
+
company: /* @__PURE__ */ new Set([
|
|
1496
|
+
"name",
|
|
1497
|
+
"domainName",
|
|
1498
|
+
"address",
|
|
1499
|
+
"companyName",
|
|
1500
|
+
"linkedInLink",
|
|
1501
|
+
"xLink",
|
|
1502
|
+
"employees",
|
|
1503
|
+
"annualRecurringRevenue",
|
|
1504
|
+
"idealCustomerProfile",
|
|
1505
|
+
"city"
|
|
1506
|
+
]),
|
|
1507
|
+
opportunity: /* @__PURE__ */ new Set(["name", "stage", "amount", "closeDate"])
|
|
1508
|
+
};
|
|
906
1509
|
var UpsertRecordStage = class {
|
|
907
1510
|
name = "upsert-record";
|
|
908
1511
|
async execute(ctx) {
|
|
@@ -913,25 +1516,50 @@ var UpsertRecordStage = class {
|
|
|
913
1516
|
const structuredCompany = ctx.structuredCompany;
|
|
914
1517
|
const structuredPerson = ctx.structuredPerson;
|
|
915
1518
|
const rawCompanyName = structuredCompany?.["name"] ?? assembled["companyName"];
|
|
916
|
-
const
|
|
1519
|
+
const customFor = (object) => {
|
|
1520
|
+
const out = {};
|
|
1521
|
+
for (const f of ctx.crmFields ?? []) {
|
|
1522
|
+
if (f.targetObject !== object || f.writable === false || f.writable === void 0) continue;
|
|
1523
|
+
if (STANDARD_BY_OBJECT[object].has(f.canonicalName)) continue;
|
|
1524
|
+
out[f.canonicalName] = formatExtValue(f.value, f.twentyType);
|
|
1525
|
+
}
|
|
1526
|
+
return out;
|
|
1527
|
+
};
|
|
1528
|
+
const policy = ctx.source?.mergePolicy ?? mergePolicy();
|
|
1529
|
+
const strategyByField = /* @__PURE__ */ new Map();
|
|
917
1530
|
for (const f of ctx.crmFields ?? []) {
|
|
918
|
-
if (
|
|
1531
|
+
if (f.mergeStrategy && f.mergeStrategy !== "INHERIT") {
|
|
1532
|
+
strategyByField.set(f.canonicalName, f.mergeStrategy);
|
|
1533
|
+
}
|
|
919
1534
|
}
|
|
1535
|
+
const strategyFor = (field) => strategyByField.get(field) ?? "INHERIT";
|
|
1536
|
+
const patchExisting = async (collection, object, existing, incoming) => {
|
|
1537
|
+
const { patch, decisions } = mergePatch(existing, incoming, policy, strategyFor);
|
|
1538
|
+
ctx.mergeDecisions = { ...ctx.mergeDecisions ?? {}, [object]: decisions };
|
|
1539
|
+
const preserved = Object.entries(decisions).filter(([, d]) => d.status === "preserved" && d.write === false).map(([field]) => field);
|
|
1540
|
+
if (preserved.length > 0 && policy === "PRESERVE") {
|
|
1541
|
+
ctx.warnings.push(
|
|
1542
|
+
`Kept the existing ${object} values for ${preserved.join(", ")} \u2014 INTAKE_MERGE_POLICY is PRESERVE`
|
|
1543
|
+
);
|
|
1544
|
+
}
|
|
1545
|
+
if (Object.keys(patch).length === 0) return;
|
|
1546
|
+
await coreApi.patch(`/${collection}/${existing.id}`, patch);
|
|
1547
|
+
};
|
|
920
1548
|
try {
|
|
921
1549
|
let companyId;
|
|
922
1550
|
let personId;
|
|
923
1551
|
const needsCompany = rawCompanyName || ctx.source?.targetObject === "COMPANY";
|
|
924
1552
|
if (needsCompany) {
|
|
925
|
-
companyId = await this.upsertCompany(assembled, structuredCompany ?? {}, rawCompanyName,
|
|
1553
|
+
companyId = await this.upsertCompany(assembled, structuredCompany ?? {}, rawCompanyName, customFor("company"), ctx, patchExisting);
|
|
926
1554
|
}
|
|
927
1555
|
const personData = structuredPerson ?? assembled;
|
|
928
1556
|
const hasPersonData = personData["emails"] || personData["name"] || assembled["emails"] || assembled["name"];
|
|
929
1557
|
if (hasPersonData || ctx.source?.targetObject === "PERSON") {
|
|
930
|
-
personId = await this.upsertPerson(assembled, structuredPerson, companyId,
|
|
1558
|
+
personId = await this.upsertPerson(assembled, structuredPerson, companyId, customFor("person"), ctx, patchExisting);
|
|
931
1559
|
}
|
|
932
1560
|
let opportunityId;
|
|
933
1561
|
if (ctx.source?.createOpportunity && (personId || companyId)) {
|
|
934
|
-
opportunityId = await this.createOpportunity(assembled, personId, companyId, ctx);
|
|
1562
|
+
opportunityId = await this.createOpportunity(assembled, personId, companyId, ctx.crmFields ?? [], customFor("opportunity"), ctx);
|
|
935
1563
|
}
|
|
936
1564
|
let noteId;
|
|
937
1565
|
const noteFields = ctx.noteFields ?? [];
|
|
@@ -952,7 +1580,7 @@ var UpsertRecordStage = class {
|
|
|
952
1580
|
return err(ErrorCode.RECORD_UPSERT_FAILED, `Failed to upsert record: ${String(e)}`);
|
|
953
1581
|
}
|
|
954
1582
|
}
|
|
955
|
-
async upsertCompany(assembled, structuredCompany, companyName,
|
|
1583
|
+
async upsertCompany(assembled, structuredCompany, companyName, customData, ctx, patchExisting) {
|
|
956
1584
|
const standard = { ...structuredCompany };
|
|
957
1585
|
if (companyName && !standard["name"]) standard["name"] = companyName;
|
|
958
1586
|
if (!standard["domainName"] && assembled["domainName"]) standard["domainName"] = assembled["domainName"];
|
|
@@ -960,17 +1588,10 @@ var UpsertRecordStage = class {
|
|
|
960
1588
|
const domain = standard["domainName"]?.["primaryLinkUrl"];
|
|
961
1589
|
let companyId;
|
|
962
1590
|
if (domain) {
|
|
963
|
-
const
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
);
|
|
968
|
-
const existing = found.data?.companies?.edges?.[0]?.node;
|
|
969
|
-
if (existing) {
|
|
970
|
-
await coreApi.patch(`/companies/${existing.id}`, standard);
|
|
971
|
-
companyId = existing.id;
|
|
972
|
-
}
|
|
973
|
-
} catch {
|
|
1591
|
+
const existing = await findCompanyByDomain(domain);
|
|
1592
|
+
if (existing) {
|
|
1593
|
+
await patchExisting("companies", "company", existing, { ...standard, ...customData });
|
|
1594
|
+
return existing.id;
|
|
974
1595
|
}
|
|
975
1596
|
}
|
|
976
1597
|
if (!companyId) {
|
|
@@ -991,20 +1612,14 @@ var UpsertRecordStage = class {
|
|
|
991
1612
|
if (!companyId) throw e;
|
|
992
1613
|
}
|
|
993
1614
|
}
|
|
994
|
-
if (Object.keys(
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
}
|
|
999
|
-
if (Object.keys(extData).length > 0) {
|
|
1000
|
-
await coreApi.patch(`/companies/${companyId}`, extData).catch((e) => {
|
|
1001
|
-
ctx.warnings.push(`Could not set ext fields on company: ${e.message.slice(0, 120)}`);
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1615
|
+
if (Object.keys(customData).length > 0) {
|
|
1616
|
+
await coreApi.patch(`/companies/${companyId}`, customData).catch((e) => {
|
|
1617
|
+
ctx.warnings.push(`Could not set custom fields on company: ${e.message.slice(0, 120)}`);
|
|
1618
|
+
});
|
|
1004
1619
|
}
|
|
1005
1620
|
return companyId;
|
|
1006
1621
|
}
|
|
1007
|
-
async upsertPerson(assembled, structuredPerson, companyId,
|
|
1622
|
+
async upsertPerson(assembled, structuredPerson, companyId, customData, ctx, patchExisting) {
|
|
1008
1623
|
const standard = structuredPerson ? { ...structuredPerson } : {};
|
|
1009
1624
|
if (!standard["name"] && assembled["name"]) standard["name"] = assembled["name"];
|
|
1010
1625
|
if (!standard["emails"] && assembled["emails"]) standard["emails"] = assembled["emails"];
|
|
@@ -1014,55 +1629,56 @@ var UpsertRecordStage = class {
|
|
|
1014
1629
|
const email = emails?.["primaryEmail"];
|
|
1015
1630
|
let personId;
|
|
1016
1631
|
if (email) {
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
const existing = found.data?.people?.edges?.[0]?.node;
|
|
1022
|
-
if (existing) {
|
|
1023
|
-
await coreApi.patch(`/people/${existing.id}`, standard);
|
|
1024
|
-
personId = existing.id;
|
|
1025
|
-
}
|
|
1026
|
-
} catch {
|
|
1632
|
+
const existing = await findPersonByEmail(email);
|
|
1633
|
+
if (existing) {
|
|
1634
|
+
await patchExisting("people", "person", existing, { ...standard, ...customData });
|
|
1635
|
+
return existing.id;
|
|
1027
1636
|
}
|
|
1028
1637
|
}
|
|
1029
1638
|
if (!personId) {
|
|
1030
1639
|
const created = await coreApi.post("/people", standard);
|
|
1031
1640
|
personId = created.data.createPerson.id;
|
|
1032
1641
|
}
|
|
1033
|
-
if (
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
}
|
|
1038
|
-
if (Object.keys(extData).length > 0) {
|
|
1039
|
-
await coreApi.patch(`/people/${personId}`, extData).catch((e) => {
|
|
1040
|
-
ctx.warnings.push(`Could not set ext fields on person: ${e.message.slice(0, 120)}`);
|
|
1041
|
-
});
|
|
1042
|
-
}
|
|
1642
|
+
if (Object.keys(customData).length > 0) {
|
|
1643
|
+
await coreApi.patch(`/people/${personId}`, customData).catch((e) => {
|
|
1644
|
+
ctx.warnings.push(`Could not set custom fields on person: ${e.message.slice(0, 120)}`);
|
|
1645
|
+
});
|
|
1043
1646
|
}
|
|
1044
1647
|
return personId;
|
|
1045
1648
|
}
|
|
1046
|
-
async createOpportunity(assembled, personId, companyId, ctx) {
|
|
1649
|
+
async createOpportunity(assembled, personId, companyId, crmFields, customData, ctx) {
|
|
1650
|
+
let opportunityId;
|
|
1047
1651
|
try {
|
|
1048
1652
|
const name = buildOpportunityName(ctx, assembled);
|
|
1049
|
-
const stage =
|
|
1653
|
+
const stage = defaultOpportunityStage();
|
|
1050
1654
|
const data = { name, stage };
|
|
1051
1655
|
if (personId) data["pointOfContactId"] = personId;
|
|
1052
1656
|
if (companyId) data["companyId"] = companyId;
|
|
1657
|
+
for (const f of crmFields) {
|
|
1658
|
+
if (f.targetObject !== "opportunity" || !f.writable) continue;
|
|
1659
|
+
if (f.canonicalName === "amount" || f.canonicalName === "closeDate") {
|
|
1660
|
+
data[f.canonicalName] = formatExtValue(f.value, f.twentyType);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1053
1663
|
const created = await coreApi.post("/opportunities", data);
|
|
1054
|
-
|
|
1664
|
+
opportunityId = created.data.createOpportunity.id;
|
|
1055
1665
|
} catch {
|
|
1056
1666
|
ctx.warnings.push("Opportunity creation failed \u2014 record was still created");
|
|
1057
1667
|
return void 0;
|
|
1058
1668
|
}
|
|
1669
|
+
if (Object.keys(customData).length > 0) {
|
|
1670
|
+
await coreApi.patch(`/opportunities/${opportunityId}`, customData).catch((e) => {
|
|
1671
|
+
ctx.warnings.push(`Could not set custom fields on opportunity: ${e.message.slice(0, 120)}`);
|
|
1672
|
+
});
|
|
1673
|
+
}
|
|
1674
|
+
return opportunityId;
|
|
1059
1675
|
}
|
|
1060
1676
|
async createNote(overflow, personId, companyId, ctx) {
|
|
1061
1677
|
try {
|
|
1062
1678
|
const markdown = formatNote(ctx.source?.name ?? ctx.sourceSlug, overflow);
|
|
1063
|
-
const
|
|
1679
|
+
const label = appLabel();
|
|
1064
1680
|
const created = await coreApi.post("/notes", {
|
|
1065
|
-
title: `${ctx.source?.name ??
|
|
1681
|
+
title: `${ctx.source?.name ?? label} \u2014 Lead`,
|
|
1066
1682
|
bodyV2: { markdown }
|
|
1067
1683
|
});
|
|
1068
1684
|
const noteId = created.data.createNote.id;
|
|
@@ -1080,69 +1696,230 @@ var UpsertRecordStage = class {
|
|
|
1080
1696
|
}
|
|
1081
1697
|
};
|
|
1082
1698
|
function buildOpportunityName(ctx, assembled) {
|
|
1083
|
-
const
|
|
1699
|
+
const label = appLabel();
|
|
1084
1700
|
const defaultTemplate = `{{source}} \u2014 {{firstName}} {{lastName}}`;
|
|
1085
1701
|
const template = ctx.source?.opportunityNameTemplate ?? defaultTemplate;
|
|
1086
1702
|
const name = assembled["name"];
|
|
1087
|
-
return template.replace("{{source}}", ctx.source?.name ??
|
|
1088
|
-
}
|
|
1089
|
-
function extractHost(url) {
|
|
1090
|
-
try {
|
|
1091
|
-
return new URL(url).hostname.replace(/^www\./, "");
|
|
1092
|
-
} catch {
|
|
1093
|
-
return url;
|
|
1094
|
-
}
|
|
1703
|
+
return template.replace("{{source}}", ctx.source?.name ?? label).replace("{{firstName}}", name?.["firstName"] ?? "").replace("{{lastName}}", name?.["lastName"] ?? "").replace("{{email}}", assembled["emails"]?.["primaryEmail"] ?? "").replace("{{company}}", assembled["companyName"] ?? "").trim().replace(/\s+/g, " ").replace(/\s*—\s*$/, "");
|
|
1095
1704
|
}
|
|
1096
1705
|
function formatExtValue(value, type) {
|
|
1097
1706
|
if (type === "LINKS" && typeof value === "string") {
|
|
1098
1707
|
return { primaryLinkUrl: value, primaryLinkLabel: "" };
|
|
1099
1708
|
}
|
|
1709
|
+
if (type === "CURRENCY") {
|
|
1710
|
+
const amount = typeof value === "number" ? value : Number(String(value ?? "").replace(/[^\d.]/g, ""));
|
|
1711
|
+
if (!Number.isFinite(amount) || amount <= 0) return void 0;
|
|
1712
|
+
return { amountMicros: Math.round(amount * 1e6), currencyCode: "USD" };
|
|
1713
|
+
}
|
|
1100
1714
|
return value;
|
|
1101
1715
|
}
|
|
1102
1716
|
|
|
1717
|
+
// src/lib/retention.ts
|
|
1718
|
+
var CREDENTIAL_KEY_PATTERNS = [
|
|
1719
|
+
"password",
|
|
1720
|
+
"passwd",
|
|
1721
|
+
"secret",
|
|
1722
|
+
"token",
|
|
1723
|
+
"apikey",
|
|
1724
|
+
"api_key",
|
|
1725
|
+
"authorization",
|
|
1726
|
+
"auth",
|
|
1727
|
+
"credential",
|
|
1728
|
+
"privatekey",
|
|
1729
|
+
"private_key",
|
|
1730
|
+
"session",
|
|
1731
|
+
"cookie",
|
|
1732
|
+
"ssn",
|
|
1733
|
+
"socialsecurity",
|
|
1734
|
+
"creditcard",
|
|
1735
|
+
"credit_card",
|
|
1736
|
+
"cardnumber",
|
|
1737
|
+
"card_number",
|
|
1738
|
+
"cvv",
|
|
1739
|
+
"cvc",
|
|
1740
|
+
"pin",
|
|
1741
|
+
"accesskey",
|
|
1742
|
+
"access_key"
|
|
1743
|
+
];
|
|
1744
|
+
var REDACTED = "[redacted]";
|
|
1745
|
+
function isCredentialKey(key) {
|
|
1746
|
+
const norm = key.toLowerCase().replace(/[-\s]/g, "_");
|
|
1747
|
+
return CREDENTIAL_KEY_PATTERNS.some((p) => norm.includes(p));
|
|
1748
|
+
}
|
|
1749
|
+
function redactCredentials(value, depth = 0) {
|
|
1750
|
+
if (depth > 12 || value === null || typeof value !== "object") return value;
|
|
1751
|
+
if (Array.isArray(value)) return value.map((v) => redactCredentials(v, depth + 1));
|
|
1752
|
+
const out = {};
|
|
1753
|
+
for (const [key, v] of Object.entries(value)) {
|
|
1754
|
+
out[key] = isCredentialKey(key) ? REDACTED : redactCredentials(v, depth + 1);
|
|
1755
|
+
}
|
|
1756
|
+
return out;
|
|
1757
|
+
}
|
|
1758
|
+
function prepareForStorage(payload) {
|
|
1759
|
+
if (rawPayloadRetention() === "NONE") {
|
|
1760
|
+
return { stored: null, warning: null };
|
|
1761
|
+
}
|
|
1762
|
+
let json;
|
|
1763
|
+
try {
|
|
1764
|
+
json = JSON.stringify(redactCredentials(payload));
|
|
1765
|
+
} catch {
|
|
1766
|
+
return { stored: null, warning: "Payload could not be serialised \u2014 not stored, so replay is unavailable for this log" };
|
|
1767
|
+
}
|
|
1768
|
+
const max = rawPayloadMaxBytes();
|
|
1769
|
+
if (Buffer.byteLength(json, "utf8") > max) {
|
|
1770
|
+
return {
|
|
1771
|
+
stored: null,
|
|
1772
|
+
warning: `Payload exceeds INTAKE_RAW_PAYLOAD_MAX_BYTES (${max}) \u2014 not stored, so replay is unavailable for this log`
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
return { stored: json, warning: null };
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
// src/lib/intake-log-writer.ts
|
|
1779
|
+
async function writeIntakeLog(input) {
|
|
1780
|
+
const { ctx, status } = input;
|
|
1781
|
+
const retained = prepareForStorage(ctx.rawPayload);
|
|
1782
|
+
if (retained.warning) ctx.warnings.push(retained.warning);
|
|
1783
|
+
try {
|
|
1784
|
+
const created = await coreApi.post("/intakeLogs", {
|
|
1785
|
+
intakeSourceId: ctx.source?.id ?? null,
|
|
1786
|
+
status,
|
|
1787
|
+
payloadHash: ctx.payloadHash ?? "",
|
|
1788
|
+
idempotencyKey: ctx.idempotencyKey ?? null,
|
|
1789
|
+
rawPayload: retained.stored,
|
|
1790
|
+
recordId: ctx.personId ?? ctx.companyId ?? null,
|
|
1791
|
+
recordType: ctx.recordType ?? null,
|
|
1792
|
+
fieldsCreated: ctx.fieldsCreated,
|
|
1793
|
+
fieldsMatched: ctx.fieldsMatched,
|
|
1794
|
+
overflowCount: ctx.noteFields?.length ?? 0,
|
|
1795
|
+
processingMs: Date.now() - ctx.startedAt,
|
|
1796
|
+
processedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1797
|
+
error: input.error ?? null,
|
|
1798
|
+
spamScore: input.spamScore ?? ctx.spam?.score ?? null,
|
|
1799
|
+
quarantineReason: input.quarantineReason ?? null,
|
|
1800
|
+
replayOfLogId: ctx.replayOfLogId ?? null
|
|
1801
|
+
});
|
|
1802
|
+
return created.data?.createIntakeLog?.id ?? null;
|
|
1803
|
+
} catch {
|
|
1804
|
+
ctx.warnings.push("Intake log write failed");
|
|
1805
|
+
return null;
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
async function bumpSourceCounters(sourceId, outcome) {
|
|
1809
|
+
try {
|
|
1810
|
+
const current = await coreApi.get(`/intakeSources/${sourceId}`);
|
|
1811
|
+
const node = current.data?.intakeSource;
|
|
1812
|
+
const patch = {};
|
|
1813
|
+
if (outcome === "ingested") {
|
|
1814
|
+
patch["totalIngested"] = (node?.totalIngested ?? 0) + 1;
|
|
1815
|
+
patch["lastIngestedAt"] = (/* @__PURE__ */ new Date()).toISOString();
|
|
1816
|
+
patch["healthStatus"] = "HEALTHY";
|
|
1817
|
+
patch["silentSince"] = null;
|
|
1818
|
+
} else {
|
|
1819
|
+
patch["totalFailed"] = (node?.totalFailed ?? 0) + 1;
|
|
1820
|
+
}
|
|
1821
|
+
await coreApi.patch(`/intakeSources/${sourceId}`, patch);
|
|
1822
|
+
} catch {
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1103
1826
|
// src/lib/stages/write-log.stage.ts
|
|
1104
1827
|
var WriteLogStage = class {
|
|
1105
1828
|
name = "write-log";
|
|
1106
1829
|
async execute(ctx) {
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
intakeSourceId: ctx.source?.id ?? null,
|
|
1111
|
-
status,
|
|
1112
|
-
payloadHash: ctx.payloadHash ?? "",
|
|
1113
|
-
rawPayload: JSON.stringify(ctx.rawPayload).slice(0, 65e3),
|
|
1114
|
-
// cap at 65k chars
|
|
1115
|
-
recordId: ctx.personId ?? ctx.companyId ?? null,
|
|
1116
|
-
recordType: ctx.recordType ?? null,
|
|
1117
|
-
fieldsCreated: ctx.fieldsCreated,
|
|
1118
|
-
fieldsMatched: ctx.fieldsMatched,
|
|
1119
|
-
overflowCount: ctx.noteFields?.length ?? 0,
|
|
1120
|
-
processingMs: Date.now() - ctx.startedAt,
|
|
1121
|
-
processedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1122
|
-
});
|
|
1123
|
-
} catch {
|
|
1124
|
-
ctx.warnings.push("Intake log write failed");
|
|
1125
|
-
}
|
|
1126
|
-
if (ctx.source) {
|
|
1127
|
-
try {
|
|
1128
|
-
const current = await coreApi.get(
|
|
1129
|
-
`/intakeSources/${ctx.source.id}`
|
|
1130
|
-
);
|
|
1131
|
-
const current_count = current.data?.intakeSource?.totalIngested ?? 0;
|
|
1132
|
-
await coreApi.patch(`/intakeSources/${ctx.source.id}`, {
|
|
1133
|
-
totalIngested: current_count + 1,
|
|
1134
|
-
lastIngestedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1135
|
-
});
|
|
1136
|
-
} catch {
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1830
|
+
const status = ctx.personId || ctx.companyId ? "SUCCESS" : "PARTIAL";
|
|
1831
|
+
await writeIntakeLog({ ctx, status });
|
|
1832
|
+
if (ctx.source) await bumpSourceCounters(ctx.source.id, "ingested");
|
|
1139
1833
|
return ok(ctx);
|
|
1140
1834
|
}
|
|
1141
1835
|
};
|
|
1142
1836
|
|
|
1143
1837
|
// src/lib/pipeline-factory.ts
|
|
1144
1838
|
function createPipeline(coreClient, metadataClient) {
|
|
1145
|
-
return new Pipeline().use(new LoadSourceStage(coreClient)).use(new VerifySignatureStage()).use(new IdempotencyStage(coreClient)).use(new FlattenStage()).use(new NormalizeStage(coreClient)).use(new SyncSchemaStage(metadataClient)).use(new UpsertRecordStage(coreClient)).use(new WriteLogStage(coreClient));
|
|
1839
|
+
return new Pipeline().use(new LoadSourceStage(coreClient)).use(new VerifySignatureStage()).use(new IdempotencyStage(coreClient)).use(new FlattenStage()).use(new NormalizeStage(coreClient)).use(new QuarantineStage()).use(new SyncSchemaStage(metadataClient)).use(new UpsertRecordStage(coreClient)).use(new WriteLogStage(coreClient));
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// src/lib/ingest.ts
|
|
1843
|
+
async function runIngestion(options) {
|
|
1844
|
+
const ctx = {
|
|
1845
|
+
sourceSlug: options.sourceSlug,
|
|
1846
|
+
rawBody: options.rawBody,
|
|
1847
|
+
rawPayload: options.rawPayload,
|
|
1848
|
+
headers: options.headers ?? {},
|
|
1849
|
+
startedAt: Date.now(),
|
|
1850
|
+
fieldsCreated: 0,
|
|
1851
|
+
fieldsMatched: 0,
|
|
1852
|
+
warnings: [],
|
|
1853
|
+
events: [],
|
|
1854
|
+
...options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {},
|
|
1855
|
+
...options.skipDedup ? { skipDedup: true } : {},
|
|
1856
|
+
...options.skipSpamFilter ? { skipSpamFilter: true } : {},
|
|
1857
|
+
...options.replayOfLogId ? { replayOfLogId: options.replayOfLogId } : {}
|
|
1858
|
+
};
|
|
1859
|
+
const pipeline = createPipeline(new CoreApiClient(), new MetadataApiClient());
|
|
1860
|
+
const result = await pipeline.run(ctx);
|
|
1861
|
+
if (result.ok) return { ok: true, ctx: result.value };
|
|
1862
|
+
await recordFailure(ctx, result.error);
|
|
1863
|
+
return { ok: false, error: result.error, ctx };
|
|
1864
|
+
}
|
|
1865
|
+
async function recordFailure(ctx, error) {
|
|
1866
|
+
if (error.code === "SOURCE_NOT_FOUND") return;
|
|
1867
|
+
if (error.code === "DUPLICATE_PAYLOAD") return;
|
|
1868
|
+
const details = error.details ?? {};
|
|
1869
|
+
const quarantined = error.code === "QUARANTINED";
|
|
1870
|
+
const status = quarantined ? "QUARANTINED" : "FAILED";
|
|
1871
|
+
await writeIntakeLog({
|
|
1872
|
+
ctx,
|
|
1873
|
+
status,
|
|
1874
|
+
error: `${error.code}: ${error.message}`,
|
|
1875
|
+
spamScore: details.spamScore ?? null,
|
|
1876
|
+
quarantineReason: quarantined ? details.reason ?? error.message : null
|
|
1877
|
+
});
|
|
1878
|
+
if (ctx.source && !quarantined) await bumpSourceCounters(ctx.source.id, "failed");
|
|
1879
|
+
}
|
|
1880
|
+
function statusCodeFor(code) {
|
|
1881
|
+
switch (code) {
|
|
1882
|
+
case "INVALID_SIGNATURE":
|
|
1883
|
+
return 401;
|
|
1884
|
+
case "SOURCE_NOT_FOUND":
|
|
1885
|
+
return 404;
|
|
1886
|
+
case "SOURCE_PAUSED":
|
|
1887
|
+
return 423;
|
|
1888
|
+
case "DUPLICATE_PAYLOAD":
|
|
1889
|
+
return 200;
|
|
1890
|
+
case "QUARANTINED":
|
|
1891
|
+
return 202;
|
|
1892
|
+
case "NO_STORED_PAYLOAD":
|
|
1893
|
+
return 400;
|
|
1894
|
+
default:
|
|
1895
|
+
return 500;
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
function idempotencyKeyFrom(headers, body) {
|
|
1899
|
+
const header = headers?.["idempotency-key"] ?? headers?.["Idempotency-Key"];
|
|
1900
|
+
const fromBody = body["idempotencyKey"];
|
|
1901
|
+
const key = header ?? (typeof fromBody === "string" ? fromBody : void 0);
|
|
1902
|
+
const trimmed = key?.trim();
|
|
1903
|
+
if (!trimmed) return void 0;
|
|
1904
|
+
return trimmed.slice(0, 255);
|
|
1905
|
+
}
|
|
1906
|
+
function successBody(ctx, startedAt) {
|
|
1907
|
+
return {
|
|
1908
|
+
success: true,
|
|
1909
|
+
recordId: ctx.personId ?? ctx.companyId,
|
|
1910
|
+
recordType: ctx.recordType,
|
|
1911
|
+
personId: ctx.personId,
|
|
1912
|
+
companyId: ctx.companyId,
|
|
1913
|
+
opportunityId: ctx.opportunityId,
|
|
1914
|
+
noteId: ctx.noteId,
|
|
1915
|
+
fieldsCreated: ctx.fieldsCreated,
|
|
1916
|
+
fieldsMatched: ctx.fieldsMatched,
|
|
1917
|
+
overflowCount: ctx.noteFields?.length ?? 0,
|
|
1918
|
+
spamScore: ctx.spam?.score ?? null,
|
|
1919
|
+
mergeDecisions: ctx.mergeDecisions ?? null,
|
|
1920
|
+
processingMs: Date.now() - startedAt,
|
|
1921
|
+
warnings: ctx.warnings
|
|
1922
|
+
};
|
|
1146
1923
|
}
|
|
1147
1924
|
|
|
1148
1925
|
// src/logic-functions/webhook.ts
|
|
@@ -1156,43 +1933,25 @@ var handler = async (params) => {
|
|
|
1156
1933
|
if (params.body && typeof params.body === "object" && !Array.isArray(params.body)) {
|
|
1157
1934
|
rawPayload = params.body;
|
|
1158
1935
|
}
|
|
1159
|
-
const
|
|
1160
|
-
const
|
|
1161
|
-
const pipeline = createPipeline(coreClient, metadataClient);
|
|
1162
|
-
const ctx = {
|
|
1936
|
+
const headers = params.headers ?? {};
|
|
1937
|
+
const outcome = await runIngestion({
|
|
1163
1938
|
sourceSlug,
|
|
1164
|
-
rawBody: params.rawBody,
|
|
1165
1939
|
rawPayload,
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
if (!result.ok) {
|
|
1175
|
-
const { code, message } = result.error;
|
|
1176
|
-
const statusCode = code === "INVALID_SIGNATURE" ? 401 : code === "SOURCE_NOT_FOUND" ? 404 : code === "SOURCE_PAUSED" ? 423 : code === "DUPLICATE_PAYLOAD" ? 200 : 500;
|
|
1177
|
-
return { statusCode, body: { error: message, code } };
|
|
1178
|
-
}
|
|
1179
|
-
const { value: finalCtx } = result;
|
|
1180
|
-
return {
|
|
1181
|
-
statusCode: 200,
|
|
1182
|
-
body: {
|
|
1183
|
-
success: true,
|
|
1184
|
-
recordId: finalCtx.personId ?? finalCtx.companyId,
|
|
1185
|
-
recordType: finalCtx.recordType,
|
|
1186
|
-
personId: finalCtx.personId,
|
|
1187
|
-
companyId: finalCtx.companyId,
|
|
1188
|
-
opportunityId: finalCtx.opportunityId,
|
|
1189
|
-
fieldsCreated: finalCtx.fieldsCreated,
|
|
1190
|
-
fieldsMatched: finalCtx.fieldsMatched,
|
|
1191
|
-
overflowCount: finalCtx.noteFields?.length ?? 0,
|
|
1192
|
-
processingMs: Date.now() - startedAt,
|
|
1193
|
-
warnings: finalCtx.warnings
|
|
1940
|
+
rawBody: params.rawBody,
|
|
1941
|
+
headers,
|
|
1942
|
+
idempotencyKey: idempotencyKeyFrom(headers, rawPayload)
|
|
1943
|
+
});
|
|
1944
|
+
if (!outcome.ok) {
|
|
1945
|
+
const { code, message, details } = outcome.error;
|
|
1946
|
+
if (code === "QUARANTINED") {
|
|
1947
|
+
return { statusCode: 202, body: { success: true, status: "held_for_review" } };
|
|
1194
1948
|
}
|
|
1195
|
-
|
|
1949
|
+
return {
|
|
1950
|
+
statusCode: statusCodeFor(code),
|
|
1951
|
+
body: { error: message, code, ...details ? { details } : {} }
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1954
|
+
return { statusCode: 200, body: successBody(outcome.ctx, startedAt) };
|
|
1196
1955
|
};
|
|
1197
1956
|
var webhook_default = defineLogicFunction({
|
|
1198
1957
|
universalIdentifier: IDS.WEBHOOK_LOGIC_FUNCTION,
|
|
@@ -1204,7 +1963,7 @@ var webhook_default = defineLogicFunction({
|
|
|
1204
1963
|
path: "/intake/:slug",
|
|
1205
1964
|
httpMethod: "POST",
|
|
1206
1965
|
isAuthRequired: false,
|
|
1207
|
-
forwardedRequestHeaders: ["x-webhook-signature", "x-intake-source", "content-type"]
|
|
1966
|
+
forwardedRequestHeaders: ["x-webhook-signature", "x-intake-source", "content-type", "idempotency-key"]
|
|
1208
1967
|
}
|
|
1209
1968
|
});
|
|
1210
1969
|
export {
|