twenty-app-intake 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +293 -8
- package/manifest.json +419 -667
- package/package.json +6 -9
- package/public/cover.generated.png +0 -0
- package/src/front-components/dashboard.mjs +19 -21947
- package/src/front-components/dashboard.mjs.map +4 -4
- package/src/front-components/dashboard.tsx +118 -0
- package/src/front-components/settings-panel.mjs +19 -22103
- package/src/front-components/settings-panel.mjs.map +4 -4
- package/src/front-components/settings-panel.tsx +304 -0
- package/src/front-components/source-panel.mjs +20 -21962
- package/src/front-components/source-panel.mjs.map +4 -4
- package/src/front-components/source-panel.tsx +113 -0
- package/src/logic-functions/check-silence.mjs +263 -0
- package/src/logic-functions/check-silence.mjs.map +7 -0
- package/src/logic-functions/check-silence.ts +52 -0
- package/src/logic-functions/contract.mjs +507 -0
- package/src/logic-functions/contract.mjs.map +7 -0
- package/src/logic-functions/contract.ts +172 -0
- package/src/logic-functions/health.mjs +125 -5
- package/src/logic-functions/health.mjs.map +3 -3
- package/src/logic-functions/health.ts +62 -0
- package/src/logic-functions/quarantine-discard.mjs +380 -0
- package/src/logic-functions/quarantine-discard.mjs.map +7 -0
- package/src/logic-functions/quarantine-discard.ts +65 -0
- package/src/logic-functions/quarantine-list.mjs +211 -0
- package/src/logic-functions/quarantine-list.mjs.map +7 -0
- package/src/logic-functions/quarantine-list.ts +93 -0
- package/src/logic-functions/quarantine-release.mjs +1974 -0
- package/src/logic-functions/quarantine-release.mjs.map +7 -0
- package/src/logic-functions/quarantine-release.ts +94 -0
- package/src/logic-functions/register.mjs +34 -1
- package/src/logic-functions/register.mjs.map +2 -2
- package/src/logic-functions/register.ts +105 -0
- package/src/logic-functions/replay-bulk.mjs +2040 -0
- package/src/logic-functions/replay-bulk.mjs.map +7 -0
- package/src/logic-functions/replay-bulk.ts +83 -0
- package/src/logic-functions/replay-log.mjs +1987 -0
- package/src/logic-functions/replay-log.mjs.map +7 -0
- package/src/logic-functions/replay-log.ts +54 -0
- package/src/logic-functions/retry.mjs +1004 -248
- package/src/logic-functions/retry.mjs.map +4 -4
- package/src/logic-functions/retry.ts +101 -0
- package/src/logic-functions/test-ingest.mjs +616 -28
- package/src/logic-functions/test-ingest.mjs.map +4 -4
- package/src/logic-functions/test-ingest.ts +179 -0
- package/src/logic-functions/webhook.mjs +968 -209
- package/src/logic-functions/webhook.mjs.map +4 -4
- package/src/logic-functions/webhook.ts +58 -0
- package/src/post-install.mjs +34 -1
- package/src/post-install.mjs.map +2 -2
- package/src/post-install.ts +81 -0
|
@@ -0,0 +1,1987 @@
|
|
|
1
|
+
import { createRequire as __createRequire } from 'module';
|
|
2
|
+
const require = __createRequire(import.meta.url);
|
|
3
|
+
|
|
4
|
+
// twenty-sdk-define-stub:__twenty-sdk-define-stub__
|
|
5
|
+
var __defineFactoryStub = (config) => ({
|
|
6
|
+
success: true,
|
|
7
|
+
config,
|
|
8
|
+
errors: []
|
|
9
|
+
});
|
|
10
|
+
var __anyHandler = {
|
|
11
|
+
get(_target, prop) {
|
|
12
|
+
if (prop === "__esModule") return true;
|
|
13
|
+
if (prop === Symbol.toPrimitive) return () => "";
|
|
14
|
+
if (typeof prop === "symbol") return void 0;
|
|
15
|
+
return new Proxy(() => void 0, __anyHandler);
|
|
16
|
+
},
|
|
17
|
+
apply() {
|
|
18
|
+
return new Proxy(() => void 0, __anyHandler);
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
var __anyStub = new Proxy(() => void 0, __anyHandler);
|
|
22
|
+
var defineLogicFunction = __defineFactoryStub;
|
|
23
|
+
|
|
24
|
+
// src/constants/universal-identifiers.ts
|
|
25
|
+
var IDS = {
|
|
26
|
+
APPLICATION: "96a389dd-384a-43a1-a376-9d3cf01b6ab7",
|
|
27
|
+
DEFAULT_ROLE: "3dd2cdb0-32c7-44b4-9f7b-0a9c667ec249",
|
|
28
|
+
// Objects
|
|
29
|
+
INTAKE_SOURCE_OBJECT: "8c1c227c-db7d-4834-b328-cacbe8af0bfc",
|
|
30
|
+
INTAKE_LOG_OBJECT: "b77a2439-1503-4ef6-ab8b-8f4293bc19da",
|
|
31
|
+
INTAKE_FIELD_RULE_OBJECT: "e74df817-ff25-4bfb-bdf0-651662ca327e",
|
|
32
|
+
// IntakeSource fields
|
|
33
|
+
INTAKE_SOURCE_NAME: "0deb7e4b-bb90-4c86-b903-5208b42041e2",
|
|
34
|
+
INTAKE_SOURCE_SLUG: "18f7645b-2dce-4145-b1b6-9cc3332c0578",
|
|
35
|
+
INTAKE_SOURCE_TARGET_OBJECT: "65f3d7f7-a601-4375-9c67-684d0044620b",
|
|
36
|
+
INTAKE_SOURCE_SECRET: "4c919402-07f5-4041-ade6-a42402f1881f",
|
|
37
|
+
INTAKE_SOURCE_STATUS: "6f1fa6fe-02b4-48ab-b1ee-97c600bbf483",
|
|
38
|
+
INTAKE_SOURCE_CREATE_OPP: "67ae59aa-2275-4ef2-b594-137f6bfafd99",
|
|
39
|
+
INTAKE_SOURCE_OPP_TEMPLATE: "3907b346-77a6-4c79-a4d7-6e1cc20bb82b",
|
|
40
|
+
INTAKE_SOURCE_TOTAL_INGESTED: "69655a93-bc70-48bd-874e-aeccc63ee1a8",
|
|
41
|
+
INTAKE_SOURCE_TOTAL_FAILED: "27b3a499-183e-4cc3-8e40-938f953a986d",
|
|
42
|
+
INTAKE_SOURCE_LAST_INGESTED: "e167f628-9865-468e-894a-b4b6150cec26",
|
|
43
|
+
// IntakeLog fields
|
|
44
|
+
INTAKE_LOG_SOURCE_RELATION: "b1c623ea-7a9a-47a5-bee4-35fdd668a637",
|
|
45
|
+
INTAKE_LOG_STATUS: "8b6d26d1-f89c-486c-a617-9f9f6a9ca36d",
|
|
46
|
+
INTAKE_LOG_PAYLOAD_HASH: "c43bb1ad-e78a-4c60-bd95-02475ede7f44",
|
|
47
|
+
INTAKE_LOG_RECORD_ID: "77a55116-a64b-4954-8369-341022dd5bdd",
|
|
48
|
+
INTAKE_LOG_RECORD_TYPE: "02154e96-5808-48ed-8838-51083f174550",
|
|
49
|
+
INTAKE_LOG_FIELDS_CREATED: "2295ca2b-b559-4b38-9771-12dd8d3d76d7",
|
|
50
|
+
INTAKE_LOG_FIELDS_MATCHED: "83d07a0f-5bcd-4fb1-9759-f9b04b9381d8",
|
|
51
|
+
INTAKE_LOG_OVERFLOW_COUNT: "4090acb1-f48a-4957-bb91-c642f8f434a7",
|
|
52
|
+
INTAKE_LOG_PROCESSING_MS: "a9191f27-4b40-42e9-8cce-20c75900e12c",
|
|
53
|
+
INTAKE_LOG_ERROR: "8d9979f1-9774-43a8-b958-d1b37bd4ecc7",
|
|
54
|
+
INTAKE_LOG_PROCESSED_AT: "a8d99fbd-722e-4987-8afe-b830ac026f0a",
|
|
55
|
+
// IntakeFieldRule fields
|
|
56
|
+
INTAKE_FIELD_RULE_SOURCE_RELATION: "396dd1f1-3873-4efa-a6bb-7b41eda14d83",
|
|
57
|
+
INTAKE_FIELD_RULE_INPUT_PATTERN: "757201a0-9f64-47b9-bf3a-a793074bf84e",
|
|
58
|
+
INTAKE_FIELD_RULE_CANONICAL_NAME: "fafebc55-a2cc-4c9f-975b-050c2676659d",
|
|
59
|
+
INTAKE_FIELD_RULE_FIELD_TYPE: "72d1f6d8-d63e-419e-a244-8127d7b4705f",
|
|
60
|
+
INTAKE_FIELD_RULE_PRIORITY: "911df299-940b-44f9-92f7-3fd01fde01a7",
|
|
61
|
+
INTAKE_FIELD_RULE_IS_ACTIVE: "6f9cd96d-aecd-437f-a7b5-d45548782771",
|
|
62
|
+
INTAKE_FIELD_RULE_TARGET_OBJECT: "7b8411a7-5e1d-40f2-a20c-683c07f67926",
|
|
63
|
+
// Logic Functions
|
|
64
|
+
WEBHOOK_LOGIC_FUNCTION: "8323c257-48fe-4bb6-b1dd-166844f575c0",
|
|
65
|
+
HEALTH_LOGIC_FUNCTION: "4ece4e39-286b-4820-81f4-b7beee71d092",
|
|
66
|
+
REGISTER_LOGIC_FUNCTION: "56d46e18-6cae-4d99-9893-b7d70740c9db",
|
|
67
|
+
// IntakeSource reverse relation fields
|
|
68
|
+
INTAKE_SOURCE_LOGS_REVERSE: "6ecfa369-22e0-4812-bb75-4338cea8be65",
|
|
69
|
+
INTAKE_SOURCE_RULES_REVERSE: "776fefaf-0eee-4ec0-93e0-d11bd80c1e20",
|
|
70
|
+
// v0.2.0 — rawPayload + retry + post-install + UI
|
|
71
|
+
INTAKE_LOG_RAW_PAYLOAD: "b65c9e1c-f2c1-4c52-a5b3-1e1ac3139692",
|
|
72
|
+
RETRY_LOGIC_FUNCTION: "90f20396-a69a-4ae4-af20-6be65c565bb5",
|
|
73
|
+
POST_INSTALL_LOGIC_FUNCTION: "e836fdd6-fd54-4aa4-ba6d-87d5a02148ef",
|
|
74
|
+
LOG_RETRY_FRONT_COMPONENT: "0cb97f6d-30e2-48c8-90e1-241b82870b4d",
|
|
75
|
+
FIELD_RULES_TABLE_FRONT_COMPONENT: "c35faad9-3468-4746-9c1c-b13d73e5e09b",
|
|
76
|
+
SOURCE_PAGE_LAYOUT: "8c894ba8-5c8c-45ed-b0af-1d22ede93eb9",
|
|
77
|
+
SOURCE_LAYOUT_TAB_RULES: "80f5c49a-083c-4062-8bd5-ae158abb6c33",
|
|
78
|
+
SOURCE_LAYOUT_TAB_RULES_WIDGET: "26a9ebb7-1d59-4b03-a018-56968f6df224",
|
|
79
|
+
LOG_PAGE_LAYOUT: "b0972802-ede5-463c-9f59-cb05f04957db",
|
|
80
|
+
LOG_LAYOUT_TAB_RETRY: "6b109695-ba3a-43de-a31a-23119ccf87ba",
|
|
81
|
+
LOG_LAYOUT_TAB_RETRY_WIDGET: "11727c1a-5145-46de-9742-88d07d64c72e",
|
|
82
|
+
SAMPLE_RULE_PHONE: "8440da8d-8ac4-4f4b-baa2-4d62936cde35",
|
|
83
|
+
SAMPLE_RULE_EMAIL: "d123cde4-05aa-44e1-86f6-454be1d7048b",
|
|
84
|
+
SAMPLE_RULE_FULLNAME: "00e49fe8-1b48-4f5b-aaf4-414d0923c712",
|
|
85
|
+
// Views & Navigation
|
|
86
|
+
SOURCES_VIEW: "4435cd43-5c13-472f-b4f9-b686c0f46285",
|
|
87
|
+
LOGS_VIEW: "a2cd0a93-9778-4ba5-99d9-b8b89c4fb39d",
|
|
88
|
+
INTAKE_NAV_ITEM: "e81bf288-5686-4713-86ae-e2947b124cab",
|
|
89
|
+
// ── v0.4.0 ────────────────────────────────────────────────────────────────
|
|
90
|
+
// IntakeLog fields — quarantine, idempotency, replay lineage
|
|
91
|
+
INTAKE_LOG_IDEMPOTENCY_KEY: "bf91e4c0-6477-41f2-a9bd-fb159d3091d4",
|
|
92
|
+
INTAKE_LOG_SPAM_SCORE: "4e250af2-925f-4724-85f6-f353a6bb4cfd",
|
|
93
|
+
INTAKE_LOG_QUARANTINE_REASON: "323b3c3e-a111-45d1-a211-cf70efa6cd9e",
|
|
94
|
+
INTAKE_LOG_REPLAY_OF: "b68876b1-0045-4eaa-a4e7-f0634e110ef1",
|
|
95
|
+
// IntakeSource fields — silence detection, spam, merge policy
|
|
96
|
+
INTAKE_SOURCE_EXPECTED_CADENCE: "a97c3906-5555-4e60-a6d6-84155a4e00a6",
|
|
97
|
+
INTAKE_SOURCE_HEALTH_STATUS: "2ad13b20-5bd7-450a-b52d-bcf208373cf5",
|
|
98
|
+
INTAKE_SOURCE_SILENT_SINCE: "d3395601-f29f-4e57-98eb-561ab011cc72",
|
|
99
|
+
INTAKE_SOURCE_ALERT_WEBHOOK: "5a066d62-0426-41c1-b34c-1a7e881ce854",
|
|
100
|
+
INTAKE_SOURCE_LAST_ALERT_AT: "fc64a598-ccbe-44e3-bd42-716e32d626a2",
|
|
101
|
+
INTAKE_SOURCE_HONEYPOT_FIELD: "5c145686-50de-496c-bb45-a30c8b39034c",
|
|
102
|
+
INTAKE_SOURCE_MERGE_POLICY: "5a946f85-b1c2-493e-92c6-31c50e637a7e",
|
|
103
|
+
// IntakeFieldRule fields
|
|
104
|
+
INTAKE_FIELD_RULE_MERGE_STRATEGY: "40c6fdcb-527a-4691-8518-c40a3d3e34c6",
|
|
105
|
+
// Logic Functions
|
|
106
|
+
CONTRACT_LOGIC_FUNCTION: "2eea4152-1b08-4b8c-903c-f4c7ee7275d0",
|
|
107
|
+
REPLAY_LOG_LOGIC_FUNCTION: "8ff98390-7358-4a91-84f3-435f758182a3",
|
|
108
|
+
REPLAY_BULK_LOGIC_FUNCTION: "b82ed4c4-07bc-4095-8ee5-611200626dc0",
|
|
109
|
+
QUARANTINE_LIST_LOGIC_FUNCTION: "0203643c-2a97-4e7c-b4a5-b76d721cf23a",
|
|
110
|
+
QUARANTINE_RELEASE_LOGIC_FUNCTION: "5a10d4fe-c807-45c6-9e61-c191d4e37b4e",
|
|
111
|
+
QUARANTINE_DISCARD_LOGIC_FUNCTION: "1454a56e-6197-4dfd-9b41-5cc3b1436af7",
|
|
112
|
+
CHECK_SILENCE_LOGIC_FUNCTION: "e88cd5f2-ab96-4f89-ac55-78db66b32d66",
|
|
113
|
+
// Application variables
|
|
114
|
+
VAR_MERGE_POLICY: "a84fe9fd-3d7c-41d9-b0e8-dfdc1a87f348",
|
|
115
|
+
VAR_SPAM_FILTER_ENABLED: "570d3ba8-1904-4bad-a5ce-27e208e6414a",
|
|
116
|
+
VAR_SPAM_SCORE_THRESHOLD: "4395574d-24b4-4e0e-afa3-1e90fcd4f470",
|
|
117
|
+
VAR_HONEYPOT_FIELD: "2cf56c56-f5cb-47c5-aa43-84f301060a95",
|
|
118
|
+
VAR_RAW_PAYLOAD_RETENTION: "f3314ad6-05b9-4127-bb88-cea5f2018dc9",
|
|
119
|
+
VAR_RAW_PAYLOAD_MAX_BYTES: "b002ca76-df3d-47b4-8ac5-d3f562239c48",
|
|
120
|
+
VAR_REPLAY_MAX_BATCH: "cf55956f-177e-4afd-8d9f-7f42c614209c"
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
// src/lib/rest-client.ts
|
|
124
|
+
var apiUrl = () => process.env["TWENTY_API_URL"] ?? "";
|
|
125
|
+
var token = () => process.env["TWENTY_APP_ACCESS_TOKEN"] ?? "";
|
|
126
|
+
async function request(base, method, path, body) {
|
|
127
|
+
const res = await fetch(`${base}${path}`, {
|
|
128
|
+
method,
|
|
129
|
+
headers: {
|
|
130
|
+
Authorization: `Bearer ${token()}`,
|
|
131
|
+
"Content-Type": "application/json"
|
|
132
|
+
},
|
|
133
|
+
body: body !== void 0 ? JSON.stringify(body) : void 0
|
|
134
|
+
});
|
|
135
|
+
const text = await res.text();
|
|
136
|
+
if (!res.ok) {
|
|
137
|
+
throw new Error(`${method} ${path} \u2192 ${res.status}: ${text}`);
|
|
138
|
+
}
|
|
139
|
+
return JSON.parse(text);
|
|
140
|
+
}
|
|
141
|
+
var coreApi = {
|
|
142
|
+
get: (path) => request(`${apiUrl()}/rest`, "GET", path),
|
|
143
|
+
post: (path, body) => request(`${apiUrl()}/rest`, "POST", path, body),
|
|
144
|
+
patch: (path, body) => request(`${apiUrl()}/rest`, "PATCH", path, body)
|
|
145
|
+
};
|
|
146
|
+
var metaApi = {
|
|
147
|
+
get: (path) => request(`${apiUrl()}/rest/metadata`, "GET", path),
|
|
148
|
+
post: (path, body) => request(`${apiUrl()}/rest/metadata`, "POST", path, body)
|
|
149
|
+
};
|
|
150
|
+
async function gql(query, variables) {
|
|
151
|
+
return request(`${apiUrl()}`, "POST", "/graphql", { query, variables });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// src/lib/ingest.ts
|
|
155
|
+
import { CoreApiClient } from "twenty-client-sdk/core";
|
|
156
|
+
import { MetadataApiClient } from "twenty-client-sdk/metadata";
|
|
157
|
+
|
|
158
|
+
// src/lib/pipeline.ts
|
|
159
|
+
var Pipeline = class {
|
|
160
|
+
stages = [];
|
|
161
|
+
use(stage) {
|
|
162
|
+
this.stages.push(stage);
|
|
163
|
+
return this;
|
|
164
|
+
}
|
|
165
|
+
async run(initial) {
|
|
166
|
+
let ctx = initial;
|
|
167
|
+
for (const stage of this.stages) {
|
|
168
|
+
const result = await stage.execute(ctx);
|
|
169
|
+
if (!result.ok) {
|
|
170
|
+
return {
|
|
171
|
+
ok: false,
|
|
172
|
+
error: { ...result.error, stage: stage.name }
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
ctx = result.value;
|
|
176
|
+
ctx.events.push({ stage: stage.name, message: "completed" });
|
|
177
|
+
}
|
|
178
|
+
return { ok: true, value: ctx };
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// src/lib/result.ts
|
|
183
|
+
var ErrorCode = {
|
|
184
|
+
INVALID_SIGNATURE: "INVALID_SIGNATURE",
|
|
185
|
+
INVALID_PAYLOAD: "INVALID_PAYLOAD",
|
|
186
|
+
SOURCE_NOT_FOUND: "SOURCE_NOT_FOUND",
|
|
187
|
+
SOURCE_PAUSED: "SOURCE_PAUSED",
|
|
188
|
+
DUPLICATE_PAYLOAD: "DUPLICATE_PAYLOAD",
|
|
189
|
+
/** Held by the spam filter — recorded, not written, and releasable. */
|
|
190
|
+
QUARANTINED: "QUARANTINED",
|
|
191
|
+
/** Retention is off or the payload was too large, so there is nothing to replay. */
|
|
192
|
+
NO_STORED_PAYLOAD: "NO_STORED_PAYLOAD",
|
|
193
|
+
FIELD_CREATE_FAILED: "FIELD_CREATE_FAILED",
|
|
194
|
+
RECORD_UPSERT_FAILED: "RECORD_UPSERT_FAILED",
|
|
195
|
+
NOTE_CREATE_FAILED: "NOTE_CREATE_FAILED",
|
|
196
|
+
SCHEMA_FETCH_FAILED: "SCHEMA_FETCH_FAILED",
|
|
197
|
+
INTERNAL_ERROR: "INTERNAL_ERROR"
|
|
198
|
+
};
|
|
199
|
+
var ok = (value) => ({ ok: true, value });
|
|
200
|
+
var err = (code, message, details) => ({ ok: false, error: { code, message, details } });
|
|
201
|
+
|
|
202
|
+
// src/lib/stages/load-source.stage.ts
|
|
203
|
+
var BASE_FIELDS = {
|
|
204
|
+
id: true,
|
|
205
|
+
name: true,
|
|
206
|
+
slug: true,
|
|
207
|
+
targetObject: true,
|
|
208
|
+
webhookSecret: true,
|
|
209
|
+
status: true,
|
|
210
|
+
createOpportunity: true,
|
|
211
|
+
opportunityNameTemplate: true
|
|
212
|
+
};
|
|
213
|
+
var V4_FIELDS = {
|
|
214
|
+
...BASE_FIELDS,
|
|
215
|
+
honeypotField: true,
|
|
216
|
+
mergePolicy: true
|
|
217
|
+
};
|
|
218
|
+
var LoadSourceStage = class {
|
|
219
|
+
constructor(client) {
|
|
220
|
+
this.client = client;
|
|
221
|
+
}
|
|
222
|
+
client;
|
|
223
|
+
name = "load-source";
|
|
224
|
+
async execute(ctx) {
|
|
225
|
+
const node = await this.fetch(ctx.sourceSlug);
|
|
226
|
+
if (!node) {
|
|
227
|
+
return err(ErrorCode.SOURCE_NOT_FOUND, `No Intake source found with slug "${ctx.sourceSlug}"`);
|
|
228
|
+
}
|
|
229
|
+
if (node.status === "PAUSED") {
|
|
230
|
+
return err(ErrorCode.SOURCE_PAUSED, `Source "${node.name}" is paused`);
|
|
231
|
+
}
|
|
232
|
+
return ok({
|
|
233
|
+
...ctx,
|
|
234
|
+
source: {
|
|
235
|
+
id: node.id,
|
|
236
|
+
name: node.name,
|
|
237
|
+
slug: node.slug,
|
|
238
|
+
targetObject: node.targetObject,
|
|
239
|
+
webhookSecret: node.webhookSecret ?? null,
|
|
240
|
+
status: node.status,
|
|
241
|
+
createOpportunity: node.createOpportunity ?? true,
|
|
242
|
+
opportunityNameTemplate: node.opportunityNameTemplate ?? "{{source}} \u2014 {{firstName}} {{lastName}}",
|
|
243
|
+
honeypotField: node.honeypotField ?? null,
|
|
244
|
+
mergePolicy: parseMergePolicy(node.mergePolicy)
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Ask for the 0.4.0 columns, and fall back to the original set if they are not
|
|
250
|
+
* there yet. A workspace that upgraded the package but has not re-synced its
|
|
251
|
+
* schema would otherwise fail the query and reject every request as an unknown
|
|
252
|
+
* source — an upgrade that takes the webhook offline, which is not acceptable
|
|
253
|
+
* for something sitting in front of someone's contact form.
|
|
254
|
+
*/
|
|
255
|
+
async fetch(slug) {
|
|
256
|
+
const args = { filter: { slug: { eq: slug } } };
|
|
257
|
+
try {
|
|
258
|
+
const result = await this.client.query({
|
|
259
|
+
intakeSources: { __args: args, edges: { node: V4_FIELDS } }
|
|
260
|
+
});
|
|
261
|
+
return result.intakeSources?.edges?.[0]?.node ?? null;
|
|
262
|
+
} catch {
|
|
263
|
+
try {
|
|
264
|
+
const result = await this.client.query({
|
|
265
|
+
intakeSources: { __args: args, edges: { node: BASE_FIELDS } }
|
|
266
|
+
});
|
|
267
|
+
return result.intakeSources?.edges?.[0]?.node ?? null;
|
|
268
|
+
} catch {
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
function parseMergePolicy(raw) {
|
|
275
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
276
|
+
case "PRESERVE":
|
|
277
|
+
return "PRESERVE";
|
|
278
|
+
case "NEWEST_WINS":
|
|
279
|
+
return "NEWEST_WINS";
|
|
280
|
+
default:
|
|
281
|
+
return null;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// src/lib/stages/verify-signature.stage.ts
|
|
286
|
+
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
287
|
+
var VerifySignatureStage = class {
|
|
288
|
+
name = "verify-signature";
|
|
289
|
+
async execute(ctx) {
|
|
290
|
+
const secret = ctx.source?.webhookSecret;
|
|
291
|
+
const requireHmac = process.env["INTAKE_REQUIRE_HMAC"] === "true";
|
|
292
|
+
if (!secret) {
|
|
293
|
+
if (requireHmac) return err(ErrorCode.INVALID_SIGNATURE, "This source has no webhook secret \u2014 INTAKE_REQUIRE_HMAC is enabled");
|
|
294
|
+
return ok(ctx);
|
|
295
|
+
}
|
|
296
|
+
const signature = ctx.headers["x-webhook-signature"];
|
|
297
|
+
if (!signature) {
|
|
298
|
+
return err(ErrorCode.INVALID_SIGNATURE, "Missing X-Webhook-Signature header");
|
|
299
|
+
}
|
|
300
|
+
if (!ctx.rawBody) {
|
|
301
|
+
ctx.warnings.push("rawBody unavailable \u2014 skipping HMAC verification");
|
|
302
|
+
return ok(ctx);
|
|
303
|
+
}
|
|
304
|
+
const expected = createHmac("sha256", secret).update(ctx.rawBody).digest("hex");
|
|
305
|
+
const sigBuf = Buffer.from(signature.replace(/^sha256=/, ""));
|
|
306
|
+
const expBuf = Buffer.from(expected);
|
|
307
|
+
if (sigBuf.length !== expBuf.length || !timingSafeEqual(sigBuf, expBuf)) {
|
|
308
|
+
return err(ErrorCode.INVALID_SIGNATURE, "Webhook signature verification failed");
|
|
309
|
+
}
|
|
310
|
+
return ok(ctx);
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
// src/lib/idempotency.ts
|
|
315
|
+
import { createHash } from "node:crypto";
|
|
316
|
+
function computePayloadHash(payload) {
|
|
317
|
+
const stable = stableStringify(payload);
|
|
318
|
+
return createHash("sha256").update(stable).digest("hex");
|
|
319
|
+
}
|
|
320
|
+
function stableStringify(value) {
|
|
321
|
+
if (value === null || value === void 0) return String(value);
|
|
322
|
+
if (typeof value !== "object") return JSON.stringify(value);
|
|
323
|
+
if (Array.isArray(value)) {
|
|
324
|
+
return "[" + value.map(stableStringify).join(",") + "]";
|
|
325
|
+
}
|
|
326
|
+
const obj = value;
|
|
327
|
+
const sorted = Object.keys(obj).sort().map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",");
|
|
328
|
+
return "{" + sorted + "}";
|
|
329
|
+
}
|
|
330
|
+
function withinWindow(processedAt, windowMs = 5 * 60 * 1e3) {
|
|
331
|
+
return Date.now() - processedAt.getTime() < windowMs;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// src/lib/settings.ts
|
|
335
|
+
function str(key, fallback) {
|
|
336
|
+
const v = process.env[key];
|
|
337
|
+
return v === void 0 || v === "" ? fallback : v;
|
|
338
|
+
}
|
|
339
|
+
function int(key, fallback) {
|
|
340
|
+
const parsed = parseInt(process.env[key] ?? "", 10);
|
|
341
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
342
|
+
}
|
|
343
|
+
function bool(key, fallback) {
|
|
344
|
+
const v = process.env[key];
|
|
345
|
+
if (v === void 0 || v === "") return fallback;
|
|
346
|
+
return v.toLowerCase() === "true";
|
|
347
|
+
}
|
|
348
|
+
function appLabel() {
|
|
349
|
+
return str("INTAKE_APP_LABEL", "Intake");
|
|
350
|
+
}
|
|
351
|
+
function defaultOpportunityStage() {
|
|
352
|
+
return str("INTAKE_DEFAULT_OPP_STAGE", "NEW");
|
|
353
|
+
}
|
|
354
|
+
function dedupWindowMinutes() {
|
|
355
|
+
return int("INTAKE_DEDUP_WINDOW_MINUTES", 5);
|
|
356
|
+
}
|
|
357
|
+
function mergePolicy() {
|
|
358
|
+
return str("INTAKE_MERGE_POLICY", "PRESERVE").toUpperCase() === "NEWEST_WINS" ? "NEWEST_WINS" : "PRESERVE";
|
|
359
|
+
}
|
|
360
|
+
function spamFilterEnabled() {
|
|
361
|
+
return bool("INTAKE_SPAM_FILTER_ENABLED", false);
|
|
362
|
+
}
|
|
363
|
+
function spamScoreThreshold() {
|
|
364
|
+
return int("INTAKE_SPAM_SCORE_THRESHOLD", 5);
|
|
365
|
+
}
|
|
366
|
+
function honeypotField() {
|
|
367
|
+
return str("INTAKE_HONEYPOT_FIELD", "");
|
|
368
|
+
}
|
|
369
|
+
function rawPayloadRetention() {
|
|
370
|
+
return str("INTAKE_RAW_PAYLOAD_RETENTION", "FULL").toUpperCase() === "NONE" ? "NONE" : "FULL";
|
|
371
|
+
}
|
|
372
|
+
function rawPayloadMaxBytes() {
|
|
373
|
+
return int("INTAKE_RAW_PAYLOAD_MAX_BYTES", 65e3);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// src/lib/stages/idempotency.stage.ts
|
|
377
|
+
var IdempotencyStage = class {
|
|
378
|
+
constructor(client) {
|
|
379
|
+
this.client = client;
|
|
380
|
+
}
|
|
381
|
+
client;
|
|
382
|
+
name = "idempotency";
|
|
383
|
+
async execute(ctx) {
|
|
384
|
+
const hash = computePayloadHash(ctx.rawPayload);
|
|
385
|
+
const next = { ...ctx, payloadHash: hash };
|
|
386
|
+
if (ctx.skipDedup) return ok(next);
|
|
387
|
+
if (ctx.idempotencyKey) {
|
|
388
|
+
const prior2 = await this.findByKey(ctx.idempotencyKey);
|
|
389
|
+
if (prior2) {
|
|
390
|
+
return err(
|
|
391
|
+
ErrorCode.DUPLICATE_PAYLOAD,
|
|
392
|
+
`Idempotency key already processed (log ${prior2.id})`,
|
|
393
|
+
{ priorLogId: prior2.id, priorRecordId: prior2.recordId, idempotencyKey: ctx.idempotencyKey }
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
const windowMins = dedupWindowMinutes();
|
|
398
|
+
if (windowMins <= 0) return ok(next);
|
|
399
|
+
const prior = await this.findByHash(hash);
|
|
400
|
+
if (prior?.processedAt && withinWindow(new Date(prior.processedAt), windowMins * 60 * 1e3)) {
|
|
401
|
+
return err(
|
|
402
|
+
ErrorCode.DUPLICATE_PAYLOAD,
|
|
403
|
+
`Duplicate payload \u2014 already processed ${String(prior.status).toLowerCase()} (log ${prior.id})`,
|
|
404
|
+
{ priorLogId: prior.id, priorRecordId: prior.recordId }
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
return ok(next);
|
|
408
|
+
}
|
|
409
|
+
async findByKey(key) {
|
|
410
|
+
try {
|
|
411
|
+
const res = await this.client.query({
|
|
412
|
+
intakeLogs: {
|
|
413
|
+
__args: {
|
|
414
|
+
filter: { idempotencyKey: { eq: key } },
|
|
415
|
+
orderBy: { processedAt: "DescNullsLast" },
|
|
416
|
+
first: 1
|
|
417
|
+
},
|
|
418
|
+
edges: { node: { id: true, status: true, recordId: true, processedAt: true } }
|
|
419
|
+
}
|
|
420
|
+
});
|
|
421
|
+
const node = res.intakeLogs?.edges?.[0]?.node ?? null;
|
|
422
|
+
if (node && (node.status === "QUARANTINED" || node.status === "DISCARDED")) return null;
|
|
423
|
+
return node;
|
|
424
|
+
} catch {
|
|
425
|
+
return null;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async findByHash(hash) {
|
|
429
|
+
try {
|
|
430
|
+
const res = await this.client.query({
|
|
431
|
+
intakeLogs: {
|
|
432
|
+
__args: {
|
|
433
|
+
filter: { payloadHash: { eq: hash } },
|
|
434
|
+
orderBy: { processedAt: "DescNullsLast" },
|
|
435
|
+
first: 1
|
|
436
|
+
},
|
|
437
|
+
edges: { node: { id: true, status: true, recordId: true, processedAt: true } }
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
return res.intakeLogs?.edges?.[0]?.node ?? null;
|
|
441
|
+
} catch {
|
|
442
|
+
return null;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
};
|
|
446
|
+
|
|
447
|
+
// src/lib/flattener.ts
|
|
448
|
+
var MAX_DEPTH = 8;
|
|
449
|
+
function flatten(input, prefix = "", depth = 0, seen = /* @__PURE__ */ new WeakSet()) {
|
|
450
|
+
if (depth > MAX_DEPTH) return {};
|
|
451
|
+
if (input === null || input === void 0) return {};
|
|
452
|
+
if (typeof input !== "object") {
|
|
453
|
+
return prefix ? { [prefix]: input } : {};
|
|
454
|
+
}
|
|
455
|
+
if (Array.isArray(input)) {
|
|
456
|
+
return prefix ? { [prefix]: input } : {};
|
|
457
|
+
}
|
|
458
|
+
const obj = input;
|
|
459
|
+
if (seen.has(obj)) return {};
|
|
460
|
+
seen.add(obj);
|
|
461
|
+
const result = {};
|
|
462
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
463
|
+
const newKey = prefix ? `${prefix}${toPascalSegment(key)}` : toCamelSegment(key);
|
|
464
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
465
|
+
const nested = flatten(value, newKey, depth + 1, seen);
|
|
466
|
+
Object.assign(result, nested);
|
|
467
|
+
} else {
|
|
468
|
+
result[newKey] = value;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
return result;
|
|
472
|
+
}
|
|
473
|
+
function toCamelSegment(s) {
|
|
474
|
+
return s.replace(/[-_]([a-zA-Z0-9])/g, (_, c) => c.toUpperCase());
|
|
475
|
+
}
|
|
476
|
+
function toPascalSegment(s) {
|
|
477
|
+
const camel = toCamelSegment(s);
|
|
478
|
+
return camel.charAt(0).toUpperCase() + camel.slice(1);
|
|
479
|
+
}
|
|
480
|
+
function detectStructure(raw) {
|
|
481
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
482
|
+
return { type: "flat", data: {} };
|
|
483
|
+
}
|
|
484
|
+
const obj = raw;
|
|
485
|
+
const hasObjectCompany = "company" in obj && typeof obj["company"] === "object" && obj["company"] !== null && !Array.isArray(obj["company"]);
|
|
486
|
+
const hasObjectPerson = "person" in obj && typeof obj["person"] === "object" && obj["person"] !== null && !Array.isArray(obj["person"]);
|
|
487
|
+
if (!hasObjectCompany && !hasObjectPerson) {
|
|
488
|
+
return { type: "flat", data: flatten(obj) };
|
|
489
|
+
}
|
|
490
|
+
const company = hasObjectCompany ? obj["company"] : null;
|
|
491
|
+
const person = hasObjectPerson ? obj["person"] : null;
|
|
492
|
+
const extra = {};
|
|
493
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
494
|
+
if (k === "company" || k === "person") continue;
|
|
495
|
+
const nested = flatten({ [k]: v });
|
|
496
|
+
Object.assign(extra, nested);
|
|
497
|
+
}
|
|
498
|
+
return { type: "structured", company, person, extra };
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// src/lib/stages/flatten.stage.ts
|
|
502
|
+
var FlattenStage = class {
|
|
503
|
+
name = "flatten";
|
|
504
|
+
async execute(ctx) {
|
|
505
|
+
const structure = detectStructure(ctx.rawPayload);
|
|
506
|
+
if (structure.type === "structured") {
|
|
507
|
+
const extraFlat = flatten(structure.extra ?? {});
|
|
508
|
+
return ok({
|
|
509
|
+
...ctx,
|
|
510
|
+
structuredCompany: structure.company ?? void 0,
|
|
511
|
+
structuredPerson: structure.person ?? void 0,
|
|
512
|
+
flat: extraFlat
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
return ok({ ...ctx, flat: structure.data });
|
|
516
|
+
}
|
|
517
|
+
};
|
|
518
|
+
|
|
519
|
+
// src/constants/field-map.ts
|
|
520
|
+
function normaliseKey(key) {
|
|
521
|
+
return key.replace(/([A-Z])/g, "_$1").toLowerCase().replace(/[-\s.]/g, "_").replace(/_{2,}/g, "_");
|
|
522
|
+
}
|
|
523
|
+
var FIELD_MAP = {
|
|
524
|
+
// ── Phone ─────────────────────────────────────────────────────────────────
|
|
525
|
+
phone: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
526
|
+
phone_number: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
527
|
+
tel: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
528
|
+
telephone: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
529
|
+
mobile: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
530
|
+
mobile_number: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
531
|
+
cell: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
532
|
+
cell_phone: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
533
|
+
contact_phone: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
534
|
+
phone1: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
535
|
+
primary_phone: { canonicalName: "phone", twentyType: "PHONES", action: "field" },
|
|
536
|
+
// ── Email ─────────────────────────────────────────────────────────────────
|
|
537
|
+
email: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
538
|
+
email_address: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
539
|
+
e_mail: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
540
|
+
contact_email: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
541
|
+
email1: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
542
|
+
user_email: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
543
|
+
primary_email: { canonicalName: "email", twentyType: "EMAILS", action: "field" },
|
|
544
|
+
// ── First name ────────────────────────────────────────────────────────────
|
|
545
|
+
first_name: { canonicalName: "firstName", twentyType: "TEXT", action: "field" },
|
|
546
|
+
fname: { canonicalName: "firstName", twentyType: "TEXT", action: "field" },
|
|
547
|
+
given_name: { canonicalName: "firstName", twentyType: "TEXT", action: "field" },
|
|
548
|
+
forename: { canonicalName: "firstName", twentyType: "TEXT", action: "field" },
|
|
549
|
+
// ── Last name ─────────────────────────────────────────────────────────────
|
|
550
|
+
last_name: { canonicalName: "lastName", twentyType: "TEXT", action: "field" },
|
|
551
|
+
lname: { canonicalName: "lastName", twentyType: "TEXT", action: "field" },
|
|
552
|
+
surname: { canonicalName: "lastName", twentyType: "TEXT", action: "field" },
|
|
553
|
+
family_name: { canonicalName: "lastName", twentyType: "TEXT", action: "field" },
|
|
554
|
+
// ── Full name (will be split on ingest) ───────────────────────────────────
|
|
555
|
+
name: { canonicalName: "fullName", twentyType: "FULL_NAME", action: "field" },
|
|
556
|
+
full_name: { canonicalName: "fullName", twentyType: "FULL_NAME", action: "field" },
|
|
557
|
+
contact_name: { canonicalName: "fullName", twentyType: "FULL_NAME", action: "field" },
|
|
558
|
+
your_name: { canonicalName: "fullName", twentyType: "FULL_NAME", action: "field" },
|
|
559
|
+
// ── Company ───────────────────────────────────────────────────────────────
|
|
560
|
+
company: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
561
|
+
company_name: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
562
|
+
business: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
563
|
+
business_name: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
564
|
+
organization: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
565
|
+
organisation: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
566
|
+
employer: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
567
|
+
firm: { canonicalName: "companyName", twentyType: "TEXT", action: "field" },
|
|
568
|
+
// ── Website / Domain ──────────────────────────────────────────────────────
|
|
569
|
+
website: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
570
|
+
website_url: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
571
|
+
url: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
572
|
+
domain: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
573
|
+
homepage: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
574
|
+
web: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
575
|
+
site: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
576
|
+
site_url: { canonicalName: "domainName", twentyType: "LINKS", action: "field" },
|
|
577
|
+
// ── Address ───────────────────────────────────────────────────────────────
|
|
578
|
+
street: { canonicalName: "addressStreet1", twentyType: "TEXT", action: "field" },
|
|
579
|
+
street_address: { canonicalName: "addressStreet1", twentyType: "TEXT", action: "field" },
|
|
580
|
+
address_line_1: { canonicalName: "addressStreet1", twentyType: "TEXT", action: "field" },
|
|
581
|
+
address1: { canonicalName: "addressStreet1", twentyType: "TEXT", action: "field" },
|
|
582
|
+
address_line_2: { canonicalName: "addressStreet2", twentyType: "TEXT", action: "field" },
|
|
583
|
+
address2: { canonicalName: "addressStreet2", twentyType: "TEXT", action: "field" },
|
|
584
|
+
city: { canonicalName: "addressCity", twentyType: "TEXT", action: "field" },
|
|
585
|
+
address_city: { canonicalName: "addressCity", twentyType: "TEXT", action: "field" },
|
|
586
|
+
town: { canonicalName: "addressCity", twentyType: "TEXT", action: "field" },
|
|
587
|
+
state: { canonicalName: "addressState", twentyType: "TEXT", action: "field" },
|
|
588
|
+
province: { canonicalName: "addressState", twentyType: "TEXT", action: "field" },
|
|
589
|
+
region: { canonicalName: "addressState", twentyType: "TEXT", action: "field" },
|
|
590
|
+
address_state: { canonicalName: "addressState", twentyType: "TEXT", action: "field" },
|
|
591
|
+
zip: { canonicalName: "addressPostcode", twentyType: "TEXT", action: "field" },
|
|
592
|
+
zip_code: { canonicalName: "addressPostcode", twentyType: "TEXT", action: "field" },
|
|
593
|
+
postal_code: { canonicalName: "addressPostcode", twentyType: "TEXT", action: "field" },
|
|
594
|
+
postcode: { canonicalName: "addressPostcode", twentyType: "TEXT", action: "field" },
|
|
595
|
+
country: { canonicalName: "addressCountry", twentyType: "TEXT", action: "field" },
|
|
596
|
+
address_country: { canonicalName: "addressCountry", twentyType: "TEXT", action: "field" },
|
|
597
|
+
// ── Social ────────────────────────────────────────────────────────────────
|
|
598
|
+
linkedin: { canonicalName: "linkedInLink", twentyType: "LINKS", action: "field" },
|
|
599
|
+
linkedin_url: { canonicalName: "linkedInLink", twentyType: "LINKS", action: "field" },
|
|
600
|
+
linkedin_profile: { canonicalName: "linkedInLink", twentyType: "LINKS", action: "field" },
|
|
601
|
+
twitter: { canonicalName: "xLink", twentyType: "LINKS", action: "field" },
|
|
602
|
+
twitter_url: { canonicalName: "xLink", twentyType: "LINKS", action: "field" },
|
|
603
|
+
x_profile: { canonicalName: "xLink", twentyType: "LINKS", action: "field" },
|
|
604
|
+
// ── Pipeline-specific (Google / business intel) ───────────────────────────
|
|
605
|
+
google_places_url: { canonicalName: "extGooglePlacesUrl", twentyType: "LINKS", action: "field" },
|
|
606
|
+
places_url: { canonicalName: "extGooglePlacesUrl", twentyType: "LINKS", action: "field" },
|
|
607
|
+
google_maps_url: { canonicalName: "extGooglePlacesUrl", twentyType: "LINKS", action: "field" },
|
|
608
|
+
maps_url: { canonicalName: "extGooglePlacesUrl", twentyType: "LINKS", action: "field" },
|
|
609
|
+
rating: { canonicalName: "extGoogleRating", twentyType: "NUMBER", action: "field" },
|
|
610
|
+
google_rating: { canonicalName: "extGoogleRating", twentyType: "NUMBER", action: "field" },
|
|
611
|
+
stars: { canonicalName: "extGoogleRating", twentyType: "NUMBER", action: "field" },
|
|
612
|
+
review_count: { canonicalName: "extReviewCount", twentyType: "NUMBER", action: "field" },
|
|
613
|
+
reviews_count: { canonicalName: "extReviewCount", twentyType: "NUMBER", action: "field" },
|
|
614
|
+
total_reviews: { canonicalName: "extReviewCount", twentyType: "NUMBER", action: "field" },
|
|
615
|
+
num_reviews: { canonicalName: "extReviewCount", twentyType: "NUMBER", action: "field" },
|
|
616
|
+
marketing_score: { canonicalName: "extMarketingScore", twentyType: "NUMBER", action: "field" },
|
|
617
|
+
lead_score: { canonicalName: "extLeadScore", twentyType: "NUMBER", action: "field" },
|
|
618
|
+
analyzed_url: { canonicalName: "extAnalyzedUrl", twentyType: "LINKS", action: "field" },
|
|
619
|
+
// ── UTM — always captured in note ────────────────────────────────────────
|
|
620
|
+
utm_source: { canonicalName: "utm_source", twentyType: "TEXT", action: "note" },
|
|
621
|
+
utm_medium: { canonicalName: "utm_medium", twentyType: "TEXT", action: "note" },
|
|
622
|
+
utm_campaign: { canonicalName: "utm_campaign", twentyType: "TEXT", action: "note" },
|
|
623
|
+
utm_content: { canonicalName: "utm_content", twentyType: "TEXT", action: "note" },
|
|
624
|
+
utm_term: { canonicalName: "utm_term", twentyType: "TEXT", action: "note" },
|
|
625
|
+
ref: { canonicalName: "ref", twentyType: "TEXT", action: "note" },
|
|
626
|
+
referrer: { canonicalName: "referrer", twentyType: "TEXT", action: "note" },
|
|
627
|
+
source_page: { canonicalName: "source_page", twentyType: "TEXT", action: "note" },
|
|
628
|
+
landing_page: { canonicalName: "landing_page", twentyType: "TEXT", action: "note" },
|
|
629
|
+
// ── Known note fields — always prose ─────────────────────────────────────
|
|
630
|
+
message: { canonicalName: "message", twentyType: "RICH_TEXT", action: "note" },
|
|
631
|
+
description: { canonicalName: "description", twentyType: "RICH_TEXT", action: "note" },
|
|
632
|
+
analysis: { canonicalName: "analysis", twentyType: "RICH_TEXT", action: "note" },
|
|
633
|
+
notes: { canonicalName: "notes", twentyType: "RICH_TEXT", action: "note" },
|
|
634
|
+
comments: { canonicalName: "comments", twentyType: "RICH_TEXT", action: "note" },
|
|
635
|
+
summary: { canonicalName: "summary", twentyType: "RICH_TEXT", action: "note" },
|
|
636
|
+
inquiry: { canonicalName: "inquiry", twentyType: "RICH_TEXT", action: "note" },
|
|
637
|
+
body: { canonicalName: "body", twentyType: "RICH_TEXT", action: "note" },
|
|
638
|
+
details: { canonicalName: "details", twentyType: "RICH_TEXT", action: "note" },
|
|
639
|
+
content: { canonicalName: "content", twentyType: "RICH_TEXT", action: "note" },
|
|
640
|
+
text: { canonicalName: "text", twentyType: "RICH_TEXT", action: "note" },
|
|
641
|
+
feedback: { canonicalName: "feedback", twentyType: "RICH_TEXT", action: "note" },
|
|
642
|
+
about: { canonicalName: "about", twentyType: "RICH_TEXT", action: "note" },
|
|
643
|
+
additional_info: { canonicalName: "additional_info", twentyType: "RICH_TEXT", action: "note" },
|
|
644
|
+
// ── System / internal — skip entirely ────────────────────────────────────
|
|
645
|
+
_id: { canonicalName: "_id", twentyType: "TEXT", action: "skip" },
|
|
646
|
+
__v: { canonicalName: "__v", twentyType: "TEXT", action: "skip" },
|
|
647
|
+
created_at: { canonicalName: "createdAt", twentyType: "DATE_TIME", action: "skip" },
|
|
648
|
+
updated_at: { canonicalName: "updatedAt", twentyType: "DATE_TIME", action: "skip" },
|
|
649
|
+
form_id: { canonicalName: "form_id", twentyType: "TEXT", action: "skip" },
|
|
650
|
+
submission_id: { canonicalName: "submission_id", twentyType: "TEXT", action: "skip" },
|
|
651
|
+
token: { canonicalName: "token", twentyType: "TEXT", action: "skip" },
|
|
652
|
+
password: { canonicalName: "password", twentyType: "TEXT", action: "skip" },
|
|
653
|
+
secret: { canonicalName: "secret", twentyType: "TEXT", action: "skip" }
|
|
654
|
+
};
|
|
655
|
+
|
|
656
|
+
// src/lib/type-detector.ts
|
|
657
|
+
var URL_RE = /^https?:\/\/[^\s/$.?#].[^\s]*$/i;
|
|
658
|
+
var EMAIL_RE = /^[^@\s]+@[^@\s]+\.[^@\s]{2,}$/;
|
|
659
|
+
var PHONE_RE = /^[+]?[(]?[0-9]{1,4}[)]?[-\s.]?[(]?[0-9]{1,4}[)]?[-\s.]?[0-9]{3,9}([-\s.][0-9]{1,9})?$/;
|
|
660
|
+
var DATE_RE = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?)?$/;
|
|
661
|
+
function detectType(value) {
|
|
662
|
+
if (value === null || value === void 0) return "TEXT";
|
|
663
|
+
if (typeof value === "boolean") return "BOOLEAN";
|
|
664
|
+
if (typeof value === "number") return "NUMBER";
|
|
665
|
+
if (typeof value === "string") {
|
|
666
|
+
const s = value.trim();
|
|
667
|
+
if (!s) return "TEXT";
|
|
668
|
+
if (URL_RE.test(s)) return "LINKS";
|
|
669
|
+
if (EMAIL_RE.test(s)) return "EMAILS";
|
|
670
|
+
if (PHONE_RE.test(s)) return "PHONES";
|
|
671
|
+
if (DATE_RE.test(s)) return "DATE_TIME";
|
|
672
|
+
if (!isNaN(Number(s)) && s !== "") return "NUMBER";
|
|
673
|
+
return "TEXT";
|
|
674
|
+
}
|
|
675
|
+
if (Array.isArray(value)) return "RAW_JSON";
|
|
676
|
+
if (typeof value === "object") return "RAW_JSON";
|
|
677
|
+
return "TEXT";
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// src/lib/classifier.ts
|
|
681
|
+
var PROSE_KEYS = /* @__PURE__ */ new Set([
|
|
682
|
+
"message",
|
|
683
|
+
"description",
|
|
684
|
+
"analysis",
|
|
685
|
+
"notes",
|
|
686
|
+
"comments",
|
|
687
|
+
"summary",
|
|
688
|
+
"inquiry",
|
|
689
|
+
"body",
|
|
690
|
+
"details",
|
|
691
|
+
"content",
|
|
692
|
+
"text",
|
|
693
|
+
"feedback",
|
|
694
|
+
"about",
|
|
695
|
+
"additional_info",
|
|
696
|
+
"info",
|
|
697
|
+
"context",
|
|
698
|
+
"requirements",
|
|
699
|
+
"goals",
|
|
700
|
+
"challenges",
|
|
701
|
+
"question",
|
|
702
|
+
"request"
|
|
703
|
+
]);
|
|
704
|
+
var NOTE_KEY_PREFIXES = ["utm_", "ga_", "fb_", "gclid", "msclkid"];
|
|
705
|
+
var SENTENCE_RE = /[.!?]\s+[A-Z]/g;
|
|
706
|
+
var LINE_BREAK_RE = /\n|\r/;
|
|
707
|
+
function classify(key, value) {
|
|
708
|
+
const normKey = key.toLowerCase().replace(/[-\s]/g, "_");
|
|
709
|
+
if (Array.isArray(value)) return "note";
|
|
710
|
+
if (value !== null && typeof value === "object") return "note";
|
|
711
|
+
if (PROSE_KEYS.has(normKey)) return "note";
|
|
712
|
+
if (NOTE_KEY_PREFIXES.some((p) => normKey.startsWith(p))) return "note";
|
|
713
|
+
if (typeof value === "string") {
|
|
714
|
+
if (LINE_BREAK_RE.test(value)) return "note";
|
|
715
|
+
const matches = value.match(SENTENCE_RE);
|
|
716
|
+
if (matches && matches.length >= 2) return "note";
|
|
717
|
+
}
|
|
718
|
+
return "field";
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// src/lib/merge.ts
|
|
722
|
+
function decideMerge(existing, incoming, policy, strategy = "INHERIT") {
|
|
723
|
+
if (isEmpty(incoming)) return { write: false, status: "preserved" };
|
|
724
|
+
if (isEmpty(existing)) return { write: true, status: "filled-empty" };
|
|
725
|
+
if (sameValue(existing, incoming)) return { write: false, status: "unchanged" };
|
|
726
|
+
const effective = strategy === "INHERIT" ? policy : strategy;
|
|
727
|
+
return effective === "NEWEST_WINS" ? { write: true, status: "set" } : { write: false, status: "preserved" };
|
|
728
|
+
}
|
|
729
|
+
function mergePatch(existing, incoming, policy, strategyFor = () => "INHERIT") {
|
|
730
|
+
const patch = {};
|
|
731
|
+
const decisions = {};
|
|
732
|
+
for (const [key, value] of Object.entries(incoming)) {
|
|
733
|
+
const decision = decideMerge(existing[key], value, policy, strategyFor(key));
|
|
734
|
+
decisions[key] = decision;
|
|
735
|
+
if (decision.write) patch[key] = value;
|
|
736
|
+
}
|
|
737
|
+
return { patch, decisions };
|
|
738
|
+
}
|
|
739
|
+
function parseFieldMergeStrategy(raw) {
|
|
740
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
741
|
+
case "PRESERVE":
|
|
742
|
+
return "PRESERVE";
|
|
743
|
+
case "NEWEST_WINS":
|
|
744
|
+
return "NEWEST_WINS";
|
|
745
|
+
default:
|
|
746
|
+
return "INHERIT";
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
function isEmpty(value) {
|
|
750
|
+
if (value === null || value === void 0) return true;
|
|
751
|
+
if (typeof value === "string") return value.trim() === "";
|
|
752
|
+
if (Array.isArray(value)) return value.length === 0;
|
|
753
|
+
if (typeof value === "object") {
|
|
754
|
+
const entries = Object.entries(value);
|
|
755
|
+
if (entries.length === 0) return true;
|
|
756
|
+
const meaningful = entries.filter(([k]) => !IGNORED_COMPOSITE_KEYS.has(k));
|
|
757
|
+
if (meaningful.length === 0) return true;
|
|
758
|
+
return meaningful.every(([, v]) => isEmpty(v));
|
|
759
|
+
}
|
|
760
|
+
return false;
|
|
761
|
+
}
|
|
762
|
+
var IGNORED_COMPOSITE_KEYS = /* @__PURE__ */ new Set([
|
|
763
|
+
"currencyCode",
|
|
764
|
+
"primaryLinkLabel",
|
|
765
|
+
"primaryPhoneCountryCode",
|
|
766
|
+
"primaryPhoneCallingCode",
|
|
767
|
+
"additionalLinks",
|
|
768
|
+
"additionalEmails",
|
|
769
|
+
"additionalPhones"
|
|
770
|
+
]);
|
|
771
|
+
function sameValue(a, b) {
|
|
772
|
+
if (a === b) return true;
|
|
773
|
+
if (typeof a === "object" && typeof b === "object" && a !== null && b !== null) {
|
|
774
|
+
try {
|
|
775
|
+
return JSON.stringify(sortKeys(a)) === JSON.stringify(sortKeys(b));
|
|
776
|
+
} catch {
|
|
777
|
+
return false;
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
return String(a) === String(b);
|
|
781
|
+
}
|
|
782
|
+
function sortKeys(value) {
|
|
783
|
+
if (value === null || typeof value !== "object") return value;
|
|
784
|
+
if (Array.isArray(value)) return value.map(sortKeys);
|
|
785
|
+
const obj = value;
|
|
786
|
+
const out = {};
|
|
787
|
+
for (const key of Object.keys(obj).sort()) out[key] = sortKeys(obj[key]);
|
|
788
|
+
return out;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// src/lib/normalizer.ts
|
|
792
|
+
function actionForRule(rule) {
|
|
793
|
+
if (rule.action) return rule.action;
|
|
794
|
+
switch ((rule.fieldType ?? "").toUpperCase()) {
|
|
795
|
+
case "SKIP":
|
|
796
|
+
return "skip";
|
|
797
|
+
case "NOTE":
|
|
798
|
+
return "note";
|
|
799
|
+
default:
|
|
800
|
+
return "field";
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
function typeForRule(rule) {
|
|
804
|
+
switch ((rule.fieldType ?? "").toUpperCase()) {
|
|
805
|
+
case "NOTE":
|
|
806
|
+
return "RICH_TEXT";
|
|
807
|
+
case "SKIP":
|
|
808
|
+
return "TEXT";
|
|
809
|
+
case "":
|
|
810
|
+
return "TEXT";
|
|
811
|
+
default:
|
|
812
|
+
return rule.fieldType;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
var OPPORTUNITY_KEY_PREFIXES = ["opportunity_", "opp_", "deal_"];
|
|
816
|
+
function parseTargetObject(raw) {
|
|
817
|
+
switch ((raw ?? "").toUpperCase()) {
|
|
818
|
+
case "PERSON":
|
|
819
|
+
return "person";
|
|
820
|
+
case "COMPANY":
|
|
821
|
+
return "company";
|
|
822
|
+
case "OPPORTUNITY":
|
|
823
|
+
return "opportunity";
|
|
824
|
+
default:
|
|
825
|
+
return "auto";
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
function normalizeFields(flat, rules = []) {
|
|
829
|
+
const sorted = [...rules].sort((a, b) => b.priority - a.priority);
|
|
830
|
+
return Object.entries(flat).map(
|
|
831
|
+
([key, value]) => normalizeField(key, value, sorted)
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
function normalizeField(key, value, rules) {
|
|
835
|
+
const lookupKey = normaliseKey(key);
|
|
836
|
+
const builtIn = FIELD_MAP[lookupKey];
|
|
837
|
+
if (builtIn) {
|
|
838
|
+
return {
|
|
839
|
+
originalKey: key,
|
|
840
|
+
canonicalName: builtIn.canonicalName,
|
|
841
|
+
twentyType: builtIn.twentyType,
|
|
842
|
+
action: builtIn.action,
|
|
843
|
+
value,
|
|
844
|
+
source: "builtin",
|
|
845
|
+
targetObject: "auto"
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
for (const rule of rules) {
|
|
849
|
+
let matches = false;
|
|
850
|
+
try {
|
|
851
|
+
matches = rule.inputPattern === key || rule.inputPattern === lookupKey || new RegExp(rule.inputPattern, "i").test(key);
|
|
852
|
+
} catch {
|
|
853
|
+
matches = rule.inputPattern === key;
|
|
854
|
+
}
|
|
855
|
+
if (matches) {
|
|
856
|
+
return {
|
|
857
|
+
originalKey: key,
|
|
858
|
+
canonicalName: rule.canonicalName,
|
|
859
|
+
twentyType: typeForRule(rule),
|
|
860
|
+
action: actionForRule(rule),
|
|
861
|
+
value,
|
|
862
|
+
source: "rule",
|
|
863
|
+
targetObject: parseTargetObject(rule.targetObject),
|
|
864
|
+
mergeStrategy: parseFieldMergeStrategy(rule.mergeStrategy)
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
const detectedType = detectType(value);
|
|
869
|
+
const classification = classify(key, value);
|
|
870
|
+
const prefix = OPPORTUNITY_KEY_PREFIXES.find((p) => lookupKey.startsWith(p));
|
|
871
|
+
const targetObject = prefix ? "opportunity" : "auto";
|
|
872
|
+
const bareKey = prefix ? lookupKey.slice(prefix.length) : key;
|
|
873
|
+
const canonicalName = classification === "field" ? toExtFieldName(bareKey) : lookupKey;
|
|
874
|
+
return {
|
|
875
|
+
originalKey: key,
|
|
876
|
+
canonicalName,
|
|
877
|
+
twentyType: detectedType,
|
|
878
|
+
action: classification === "field" ? "field" : classification,
|
|
879
|
+
value,
|
|
880
|
+
source: "passthrough",
|
|
881
|
+
targetObject
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
function toExtFieldName(key) {
|
|
885
|
+
const cleaned = key.replace(/[-_\s]+([a-zA-Z0-9])/g, (_, c) => c.toUpperCase()).replace(/[^a-zA-Z0-9]/g, "");
|
|
886
|
+
return "ext" + cleaned.charAt(0).toUpperCase() + cleaned.slice(1);
|
|
887
|
+
}
|
|
888
|
+
function partition(fields) {
|
|
889
|
+
const crmFields = [];
|
|
890
|
+
const noteFields = [];
|
|
891
|
+
const skipped = [];
|
|
892
|
+
for (const f of fields) {
|
|
893
|
+
if (f.action === "skip") skipped.push(f);
|
|
894
|
+
else if (f.action === "note") noteFields.push(f);
|
|
895
|
+
else crmFields.push(f);
|
|
896
|
+
}
|
|
897
|
+
return { crmFields, noteFields, skipped };
|
|
898
|
+
}
|
|
899
|
+
function assembleComposites(crmFields) {
|
|
900
|
+
const assembled = {};
|
|
901
|
+
const consumed = /* @__PURE__ */ new Set();
|
|
902
|
+
const firstName = crmFields.find((f) => f.canonicalName === "firstName");
|
|
903
|
+
const lastName = crmFields.find((f) => f.canonicalName === "lastName");
|
|
904
|
+
const fullName = crmFields.find((f) => f.canonicalName === "fullName");
|
|
905
|
+
if (fullName && !firstName && !lastName) {
|
|
906
|
+
const parts = splitFullName(String(fullName.value ?? ""));
|
|
907
|
+
assembled["name"] = { firstName: parts[0], lastName: parts[1] };
|
|
908
|
+
consumed.add("fullName");
|
|
909
|
+
} else if (firstName || lastName) {
|
|
910
|
+
assembled["name"] = {
|
|
911
|
+
firstName: String(firstName?.value ?? ""),
|
|
912
|
+
lastName: String(lastName?.value ?? "")
|
|
913
|
+
};
|
|
914
|
+
if (firstName) consumed.add("firstName");
|
|
915
|
+
if (lastName) consumed.add("lastName");
|
|
916
|
+
}
|
|
917
|
+
const email = crmFields.find((f) => f.canonicalName === "email");
|
|
918
|
+
if (email) {
|
|
919
|
+
assembled["emails"] = { primaryEmail: String(email.value ?? "") };
|
|
920
|
+
consumed.add("email");
|
|
921
|
+
}
|
|
922
|
+
const phone = crmFields.find((f) => f.canonicalName === "phone");
|
|
923
|
+
if (phone) {
|
|
924
|
+
const raw = String(phone.value ?? "");
|
|
925
|
+
assembled["phones"] = {
|
|
926
|
+
primaryPhoneNumber: raw.replace(/\D/g, "").slice(-10),
|
|
927
|
+
primaryPhoneCountryCode: "US",
|
|
928
|
+
primaryPhoneCallingCode: "+1"
|
|
929
|
+
};
|
|
930
|
+
consumed.add("phone");
|
|
931
|
+
}
|
|
932
|
+
const domain = crmFields.find((f) => f.canonicalName === "domainName");
|
|
933
|
+
if (domain) {
|
|
934
|
+
assembled["domainName"] = {
|
|
935
|
+
primaryLinkUrl: String(domain.value ?? ""),
|
|
936
|
+
primaryLinkLabel: ""
|
|
937
|
+
};
|
|
938
|
+
consumed.add("domainName");
|
|
939
|
+
}
|
|
940
|
+
const addressFields = [
|
|
941
|
+
["addressStreet1", "addressStreet1"],
|
|
942
|
+
["addressStreet2", "addressStreet2"],
|
|
943
|
+
["addressCity", "addressCity"],
|
|
944
|
+
["addressState", "addressState"],
|
|
945
|
+
["addressPostcode", "addressPostcode"],
|
|
946
|
+
["addressCountry", "addressCountry"]
|
|
947
|
+
];
|
|
948
|
+
const addressParts = {};
|
|
949
|
+
for (const [canonical, outKey] of addressFields) {
|
|
950
|
+
const f = crmFields.find((x) => x.canonicalName === canonical);
|
|
951
|
+
if (f) {
|
|
952
|
+
addressParts[outKey] = f.value;
|
|
953
|
+
consumed.add(canonical);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
if (Object.keys(addressParts).length > 0) {
|
|
957
|
+
assembled["address"] = addressParts;
|
|
958
|
+
}
|
|
959
|
+
const remainder = crmFields.filter((f) => !consumed.has(f.canonicalName));
|
|
960
|
+
return { assembled, remainder };
|
|
961
|
+
}
|
|
962
|
+
function splitFullName(fullName) {
|
|
963
|
+
const parts = fullName.trim().split(/\s+/);
|
|
964
|
+
if (parts.length === 0) return ["", ""];
|
|
965
|
+
if (parts.length === 1) return [parts[0] ?? "", ""];
|
|
966
|
+
const lastName = parts.pop() ?? "";
|
|
967
|
+
return [parts.join(" "), lastName];
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
// src/lib/stages/normalize.stage.ts
|
|
971
|
+
var NormalizeStage = class {
|
|
972
|
+
constructor(client) {
|
|
973
|
+
this.client = client;
|
|
974
|
+
}
|
|
975
|
+
client;
|
|
976
|
+
name = "normalize";
|
|
977
|
+
async execute(ctx) {
|
|
978
|
+
if (!ctx.flat) {
|
|
979
|
+
return err(ErrorCode.INTERNAL_ERROR, "FlattenStage must run before NormalizeStage");
|
|
980
|
+
}
|
|
981
|
+
const rules = await this.loadRules(ctx.source?.id);
|
|
982
|
+
const normalized = normalizeFields(ctx.flat, rules);
|
|
983
|
+
const { crmFields, noteFields } = partition(normalized);
|
|
984
|
+
return ok({
|
|
985
|
+
...ctx,
|
|
986
|
+
normalizedFields: normalized,
|
|
987
|
+
crmFields,
|
|
988
|
+
noteFields
|
|
989
|
+
});
|
|
990
|
+
}
|
|
991
|
+
async loadRules(sourceId) {
|
|
992
|
+
try {
|
|
993
|
+
const filter = sourceId ? { or: [{ intakeSource: { id: { eq: sourceId } } }, { intakeSource: { id: { is: "NULL" } } }] } : { intakeSource: { id: { is: "NULL" } } };
|
|
994
|
+
const result = await this.client.query({
|
|
995
|
+
intakeFieldRules: {
|
|
996
|
+
__args: {
|
|
997
|
+
filter: { and: [filter, { isActive: { eq: true } }] },
|
|
998
|
+
orderBy: { priority: "DescNullsLast" }
|
|
999
|
+
},
|
|
1000
|
+
edges: {
|
|
1001
|
+
node: {
|
|
1002
|
+
inputPattern: true,
|
|
1003
|
+
canonicalName: true,
|
|
1004
|
+
fieldType: true,
|
|
1005
|
+
priority: true,
|
|
1006
|
+
targetObject: true,
|
|
1007
|
+
mergeStrategy: true
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
});
|
|
1012
|
+
return (result.intakeFieldRules?.edges ?? []).map((e) => e.node);
|
|
1013
|
+
} catch {
|
|
1014
|
+
return this.loadRulesLegacy(sourceId);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
async loadRulesLegacy(sourceId) {
|
|
1018
|
+
try {
|
|
1019
|
+
const filter = sourceId ? { or: [{ intakeSource: { id: { eq: sourceId } } }, { intakeSource: { id: { is: "NULL" } } }] } : { intakeSource: { id: { is: "NULL" } } };
|
|
1020
|
+
const result = await this.client.query({
|
|
1021
|
+
intakeFieldRules: {
|
|
1022
|
+
__args: {
|
|
1023
|
+
filter: { and: [filter, { isActive: { eq: true } }] },
|
|
1024
|
+
orderBy: { priority: "DescNullsLast" }
|
|
1025
|
+
},
|
|
1026
|
+
edges: {
|
|
1027
|
+
node: {
|
|
1028
|
+
inputPattern: true,
|
|
1029
|
+
canonicalName: true,
|
|
1030
|
+
fieldType: true,
|
|
1031
|
+
priority: true
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
});
|
|
1036
|
+
return (result.intakeFieldRules?.edges ?? []).map((e) => e.node);
|
|
1037
|
+
} catch {
|
|
1038
|
+
return [];
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
};
|
|
1042
|
+
|
|
1043
|
+
// src/lib/spam.ts
|
|
1044
|
+
var HONEYPOT_SCORE = 100;
|
|
1045
|
+
var DISPOSABLE_EMAIL_DOMAINS = /* @__PURE__ */ new Set([
|
|
1046
|
+
"mailinator.com",
|
|
1047
|
+
"guerrillamail.com",
|
|
1048
|
+
"guerrillamail.net",
|
|
1049
|
+
"sharklasers.com",
|
|
1050
|
+
"10minutemail.com",
|
|
1051
|
+
"10minutemail.net",
|
|
1052
|
+
"tempmail.com",
|
|
1053
|
+
"temp-mail.org",
|
|
1054
|
+
"throwawaymail.com",
|
|
1055
|
+
"yopmail.com",
|
|
1056
|
+
"yopmail.net",
|
|
1057
|
+
"trashmail.com",
|
|
1058
|
+
"dispostable.com",
|
|
1059
|
+
"getnada.com",
|
|
1060
|
+
"nada.email",
|
|
1061
|
+
"fakeinbox.com",
|
|
1062
|
+
"maildrop.cc",
|
|
1063
|
+
"mintemail.com",
|
|
1064
|
+
"spamgourmet.com",
|
|
1065
|
+
"mailnesia.com",
|
|
1066
|
+
"tempinbox.com",
|
|
1067
|
+
"emailondeck.com",
|
|
1068
|
+
"moakt.com",
|
|
1069
|
+
"mohmal.com",
|
|
1070
|
+
"inboxkitten.com",
|
|
1071
|
+
"mailcatch.com",
|
|
1072
|
+
"mytemp.email",
|
|
1073
|
+
"discard.email"
|
|
1074
|
+
]);
|
|
1075
|
+
var PLACEHOLDER_EMAIL_DOMAINS = /* @__PURE__ */ new Set([
|
|
1076
|
+
"example.com",
|
|
1077
|
+
"example.org",
|
|
1078
|
+
"example.net",
|
|
1079
|
+
"test.com",
|
|
1080
|
+
"localhost",
|
|
1081
|
+
"domain.com",
|
|
1082
|
+
"email.com",
|
|
1083
|
+
"yourdomain.com"
|
|
1084
|
+
]);
|
|
1085
|
+
var URL_RE2 = /https?:\/\/|www\./i;
|
|
1086
|
+
var GLOBAL_URL_RE = /https?:\/\/[^\s<>"']+/gi;
|
|
1087
|
+
var MARKUP_RE = /<\s*a\s|<\s*script|\[url[=\]]|\[link[=\]]|\]\(https?:/i;
|
|
1088
|
+
var EMAIL_RE2 = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/;
|
|
1089
|
+
var NAME_KEYS = /* @__PURE__ */ new Set([
|
|
1090
|
+
"name",
|
|
1091
|
+
"fullName",
|
|
1092
|
+
"firstName",
|
|
1093
|
+
"lastName",
|
|
1094
|
+
"contact_name",
|
|
1095
|
+
"your_name"
|
|
1096
|
+
]);
|
|
1097
|
+
var MESSAGE_KEYS = /* @__PURE__ */ new Set([
|
|
1098
|
+
"message",
|
|
1099
|
+
"description",
|
|
1100
|
+
"comments",
|
|
1101
|
+
"notes",
|
|
1102
|
+
"body",
|
|
1103
|
+
"details",
|
|
1104
|
+
"content",
|
|
1105
|
+
"text",
|
|
1106
|
+
"inquiry",
|
|
1107
|
+
"summary",
|
|
1108
|
+
"feedback"
|
|
1109
|
+
]);
|
|
1110
|
+
function scoreSpam(flat, honeypotKey) {
|
|
1111
|
+
const signals = [];
|
|
1112
|
+
if (honeypotKey) {
|
|
1113
|
+
const hit = Object.entries(flat).find(
|
|
1114
|
+
([k, v]) => k.toLowerCase() === honeypotKey.toLowerCase() && isFilled(v)
|
|
1115
|
+
);
|
|
1116
|
+
if (hit) {
|
|
1117
|
+
return {
|
|
1118
|
+
score: HONEYPOT_SCORE,
|
|
1119
|
+
signals: [{ reason: `Honeypot field "${hit[0]}" was filled`, points: HONEYPOT_SCORE }],
|
|
1120
|
+
honeypot: true
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
const add = (reason, points) => signals.push({ reason, points });
|
|
1125
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1126
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1127
|
+
const bare = key.replace(/^(opportunity_|opp_|deal_)/i, "");
|
|
1128
|
+
if (NAME_KEYS.has(bare) || NAME_KEYS.has(key)) {
|
|
1129
|
+
if (URL_RE2.test(value)) add(`Name field "${key}" contains a URL`, 4);
|
|
1130
|
+
if (MARKUP_RE.test(value)) add(`Name field "${key}" contains markup or a link tag`, 4);
|
|
1131
|
+
}
|
|
1132
|
+
if (MESSAGE_KEYS.has(bare) || MESSAGE_KEYS.has(key)) {
|
|
1133
|
+
const urls = value.match(GLOBAL_URL_RE) ?? [];
|
|
1134
|
+
const distinct = new Set(urls.map((u) => u.toLowerCase()));
|
|
1135
|
+
if (distinct.size >= 5) add(`Message field "${key}" contains ${distinct.size} links`, 3);
|
|
1136
|
+
if (MARKUP_RE.test(value)) add(`Message field "${key}" contains anchor or BBCode markup`, 2);
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
const email = firstEmail(flat);
|
|
1140
|
+
if (email !== null) {
|
|
1141
|
+
if (!EMAIL_RE2.test(email)) {
|
|
1142
|
+
add(`Email "${truncate(email)}" is not a valid address`, 3);
|
|
1143
|
+
} else {
|
|
1144
|
+
const domain = email.slice(email.lastIndexOf("@") + 1).toLowerCase();
|
|
1145
|
+
if (DISPOSABLE_EMAIL_DOMAINS.has(domain)) {
|
|
1146
|
+
add(`Email domain "${domain}" is a disposable inbox`, 4);
|
|
1147
|
+
} else if (PLACEHOLDER_EMAIL_DOMAINS.has(domain)) {
|
|
1148
|
+
add(`Email domain "${domain}" is a placeholder domain`, 3);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
const repeated = mostRepeatedValue(flat);
|
|
1153
|
+
if (repeated && repeated.count >= 4) {
|
|
1154
|
+
add(`The same value appears in ${repeated.count} different fields`, 2);
|
|
1155
|
+
}
|
|
1156
|
+
const phone = firstPhone(flat);
|
|
1157
|
+
if (phone && isImplausiblePhone(phone)) {
|
|
1158
|
+
add(`Phone "${truncate(phone)}" is a placeholder number`, 2);
|
|
1159
|
+
}
|
|
1160
|
+
const score = signals.reduce((sum, s) => sum + s.points, 0);
|
|
1161
|
+
return { score, signals, honeypot: false };
|
|
1162
|
+
}
|
|
1163
|
+
function describeVerdict(verdict) {
|
|
1164
|
+
return verdict.signals.map((s) => s.reason).join("; ").slice(0, 500);
|
|
1165
|
+
}
|
|
1166
|
+
function isFilled(value) {
|
|
1167
|
+
if (value === null || value === void 0) return false;
|
|
1168
|
+
if (typeof value === "string") return value.trim() !== "";
|
|
1169
|
+
if (typeof value === "boolean") return value;
|
|
1170
|
+
return true;
|
|
1171
|
+
}
|
|
1172
|
+
function firstEmail(flat) {
|
|
1173
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1174
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1175
|
+
if (/mail/i.test(key)) return value.trim();
|
|
1176
|
+
}
|
|
1177
|
+
return null;
|
|
1178
|
+
}
|
|
1179
|
+
function firstPhone(flat) {
|
|
1180
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
1181
|
+
if (typeof value !== "string" || value === "") continue;
|
|
1182
|
+
if (/phone|mobile|tel|cell/i.test(key)) return value.trim();
|
|
1183
|
+
}
|
|
1184
|
+
return null;
|
|
1185
|
+
}
|
|
1186
|
+
function isImplausiblePhone(raw) {
|
|
1187
|
+
const digits = raw.replace(/\D/g, "");
|
|
1188
|
+
if (digits.length < 7) return false;
|
|
1189
|
+
if (/^(\d)\1+$/.test(digits)) return true;
|
|
1190
|
+
const ascending = digits.split("").every((d, i, arr) => i === 0 || Number(d) === (Number(arr[i - 1]) + 1) % 10);
|
|
1191
|
+
return ascending;
|
|
1192
|
+
}
|
|
1193
|
+
function mostRepeatedValue(flat) {
|
|
1194
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1195
|
+
for (const value of Object.values(flat)) {
|
|
1196
|
+
if (typeof value !== "string") continue;
|
|
1197
|
+
const v = value.trim();
|
|
1198
|
+
if (v.length < 8) continue;
|
|
1199
|
+
counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
1200
|
+
}
|
|
1201
|
+
let best = null;
|
|
1202
|
+
for (const [value, count] of counts) {
|
|
1203
|
+
if (!best || count > best.count) best = { value, count };
|
|
1204
|
+
}
|
|
1205
|
+
return best;
|
|
1206
|
+
}
|
|
1207
|
+
function truncate(s) {
|
|
1208
|
+
return s.length > 60 ? `${s.slice(0, 60)}\u2026` : s;
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
// src/lib/stages/quarantine.stage.ts
|
|
1212
|
+
var QuarantineStage = class {
|
|
1213
|
+
name = "quarantine";
|
|
1214
|
+
async execute(ctx) {
|
|
1215
|
+
const honeypot = ctx.source?.honeypotField || honeypotField();
|
|
1216
|
+
const filterOn = spamFilterEnabled();
|
|
1217
|
+
if (!honeypot && !filterOn) return ok(ctx);
|
|
1218
|
+
const verdict = scoreSpam(ctx.flat ?? {}, honeypot);
|
|
1219
|
+
const next = { ...ctx, spam: verdict };
|
|
1220
|
+
if (ctx.skipSpamFilter) {
|
|
1221
|
+
if (verdict.score > 0) {
|
|
1222
|
+
next.warnings.push(`Spam filter overruled by release \u2014 score was ${verdict.score}: ${describeVerdict(verdict)}`);
|
|
1223
|
+
}
|
|
1224
|
+
return ok(next);
|
|
1225
|
+
}
|
|
1226
|
+
const quarantine = verdict.honeypot || filterOn && verdict.score >= spamScoreThreshold();
|
|
1227
|
+
if (quarantine) {
|
|
1228
|
+
return err(
|
|
1229
|
+
ErrorCode.QUARANTINED,
|
|
1230
|
+
verdict.honeypot ? "Honeypot field was filled \u2014 payload quarantined" : `Spam score ${verdict.score} met the threshold of ${spamScoreThreshold()} \u2014 payload quarantined`,
|
|
1231
|
+
{ spamScore: verdict.score, reason: describeVerdict(verdict), signals: verdict.signals }
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
if (verdict.score > 0) {
|
|
1235
|
+
next.warnings.push(`Spam score ${verdict.score} (below threshold): ${describeVerdict(verdict)}`);
|
|
1236
|
+
}
|
|
1237
|
+
return ok(next);
|
|
1238
|
+
}
|
|
1239
|
+
};
|
|
1240
|
+
|
|
1241
|
+
// src/lib/stages/sync-schema.stage.ts
|
|
1242
|
+
var STANDARD_FIELDS = /* @__PURE__ */ new Set([
|
|
1243
|
+
"name",
|
|
1244
|
+
"emails",
|
|
1245
|
+
"phones",
|
|
1246
|
+
"domainName",
|
|
1247
|
+
"address",
|
|
1248
|
+
"linkedInLink",
|
|
1249
|
+
"xLink",
|
|
1250
|
+
"companyName",
|
|
1251
|
+
"firstName",
|
|
1252
|
+
"lastName",
|
|
1253
|
+
"fullName",
|
|
1254
|
+
"jobTitle",
|
|
1255
|
+
"city",
|
|
1256
|
+
"employees",
|
|
1257
|
+
"annualRecurringRevenue",
|
|
1258
|
+
"idealCustomerProfile",
|
|
1259
|
+
...Object.values(FIELD_MAP).map((e) => e.canonicalName)
|
|
1260
|
+
]);
|
|
1261
|
+
var STANDARD_OPPORTUNITY_FIELDS = /* @__PURE__ */ new Set(["name", "stage", "amount", "closeDate"]);
|
|
1262
|
+
var EXT_RE = /^ext[A-Z]/;
|
|
1263
|
+
var SyncSchemaStage = class {
|
|
1264
|
+
name = "sync-schema";
|
|
1265
|
+
objectCache = /* @__PURE__ */ new Map();
|
|
1266
|
+
TTL = 5 * 60 * 1e3;
|
|
1267
|
+
async execute(ctx) {
|
|
1268
|
+
if (!ctx.crmFields || ctx.crmFields.length === 0) return ok(ctx);
|
|
1269
|
+
const primary = this.primaryTarget(ctx);
|
|
1270
|
+
const creationEnabled = process.env["INTAKE_FIELD_CREATION_ENABLED"] !== "false";
|
|
1271
|
+
if (!creationEnabled) {
|
|
1272
|
+
ctx.warnings.push("Field creation disabled by INTAKE_FIELD_CREATION_ENABLED setting \u2014 ext fields skipped");
|
|
1273
|
+
}
|
|
1274
|
+
const resolved = ctx.crmFields.map((f) => ({
|
|
1275
|
+
...f,
|
|
1276
|
+
targetObject: this.resolveTarget(f.targetObject, primary, ctx)
|
|
1277
|
+
}));
|
|
1278
|
+
let fieldsCreated = 0;
|
|
1279
|
+
let fieldsMatched = 0;
|
|
1280
|
+
const demoted = [];
|
|
1281
|
+
for (const objectName of this.targetObjects(resolved)) {
|
|
1282
|
+
const fields = resolved.filter((f) => f.targetObject === objectName);
|
|
1283
|
+
if (fields.length === 0) continue;
|
|
1284
|
+
const standardForObject = objectName === "opportunity" ? STANDARD_OPPORTUNITY_FIELDS : STANDARD_FIELDS;
|
|
1285
|
+
const meta = await this.getObjectMeta(objectName);
|
|
1286
|
+
if (!meta) {
|
|
1287
|
+
for (const f of fields) {
|
|
1288
|
+
if (standardForObject.has(f.canonicalName)) {
|
|
1289
|
+
f.writable = true;
|
|
1290
|
+
fieldsMatched++;
|
|
1291
|
+
continue;
|
|
1292
|
+
}
|
|
1293
|
+
ctx.warnings.push(`Could not read schema for ${objectName} \u2014 "${f.canonicalName}" goes to note`);
|
|
1294
|
+
demoted.push(f);
|
|
1295
|
+
}
|
|
1296
|
+
continue;
|
|
1297
|
+
}
|
|
1298
|
+
const maxExt = parseInt(process.env["INTAKE_MAX_EXT_FIELDS"] ?? "50", 10);
|
|
1299
|
+
let extCount = [...meta.fields].filter((f) => EXT_RE.test(f)).length;
|
|
1300
|
+
for (const field of fields) {
|
|
1301
|
+
const name = field.canonicalName;
|
|
1302
|
+
if (standardForObject.has(name)) {
|
|
1303
|
+
field.writable = true;
|
|
1304
|
+
fieldsMatched++;
|
|
1305
|
+
continue;
|
|
1306
|
+
}
|
|
1307
|
+
if (meta.fields.has(name)) {
|
|
1308
|
+
field.writable = true;
|
|
1309
|
+
fieldsMatched++;
|
|
1310
|
+
continue;
|
|
1311
|
+
}
|
|
1312
|
+
if (!EXT_RE.test(name)) {
|
|
1313
|
+
ctx.warnings.push(
|
|
1314
|
+
`Field "${name}" does not exist on ${objectName} and is not ext-prefixed \u2014 value routed to note. Create the field in Twenty first, or rename the rule's canonical name to ext${name.charAt(0).toUpperCase()}${name.slice(1)}.`
|
|
1315
|
+
);
|
|
1316
|
+
demoted.push(field);
|
|
1317
|
+
continue;
|
|
1318
|
+
}
|
|
1319
|
+
if (!creationEnabled) {
|
|
1320
|
+
demoted.push(field);
|
|
1321
|
+
continue;
|
|
1322
|
+
}
|
|
1323
|
+
if (extCount >= maxExt) {
|
|
1324
|
+
ctx.warnings.push(`ext field limit (${maxExt}) reached for ${objectName} \u2014 "${name}" goes to note`);
|
|
1325
|
+
demoted.push(field);
|
|
1326
|
+
continue;
|
|
1327
|
+
}
|
|
1328
|
+
const created = await this.createField(meta.id, name, field.twentyType);
|
|
1329
|
+
if (created) {
|
|
1330
|
+
meta.fields.add(name);
|
|
1331
|
+
this.objectCache.get(objectName)?.fields.add(name);
|
|
1332
|
+
extCount++;
|
|
1333
|
+
fieldsCreated++;
|
|
1334
|
+
field.writable = true;
|
|
1335
|
+
} else {
|
|
1336
|
+
ctx.warnings.push(`Could not create field "${name}" on ${objectName} \u2014 will go to note`);
|
|
1337
|
+
demoted.push(field);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
const demotedSet = new Set(demoted);
|
|
1342
|
+
return ok({
|
|
1343
|
+
...ctx,
|
|
1344
|
+
crmFields: resolved.filter((f) => !demotedSet.has(f)),
|
|
1345
|
+
noteFields: [...ctx.noteFields ?? [], ...demoted],
|
|
1346
|
+
fieldsCreated,
|
|
1347
|
+
fieldsMatched
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
/** Where an `auto` field lands: the person, or the company for company-only payloads. */
|
|
1351
|
+
primaryTarget(ctx) {
|
|
1352
|
+
const t = ctx.source?.targetObject ?? "AUTO";
|
|
1353
|
+
if (t === "PERSON") return "person";
|
|
1354
|
+
if (t === "COMPANY") return "company";
|
|
1355
|
+
if (ctx.structuredCompany) return "company";
|
|
1356
|
+
const hasPersonData = ctx.crmFields?.some((f) => ["firstName", "lastName", "email", "phone", "fullName", "emails", "phones", "name"].includes(f.canonicalName));
|
|
1357
|
+
const hasCompanyData = ctx.crmFields?.some((f) => ["companyName", "domainName"].includes(f.canonicalName));
|
|
1358
|
+
if (hasPersonData) return "person";
|
|
1359
|
+
if (hasCompanyData) return "company";
|
|
1360
|
+
return "person";
|
|
1361
|
+
}
|
|
1362
|
+
resolveTarget(target, primary, ctx) {
|
|
1363
|
+
if (target === "auto") return primary;
|
|
1364
|
+
if (target === "opportunity" && !ctx.source?.createOpportunity) {
|
|
1365
|
+
ctx.warnings.push("Field routed to opportunity but this source does not create opportunities \u2014 written to the primary record instead");
|
|
1366
|
+
return primary;
|
|
1367
|
+
}
|
|
1368
|
+
return target;
|
|
1369
|
+
}
|
|
1370
|
+
targetObjects(fields) {
|
|
1371
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1372
|
+
for (const f of fields) if (f.targetObject !== "auto") seen.add(f.targetObject);
|
|
1373
|
+
return [...seen];
|
|
1374
|
+
}
|
|
1375
|
+
async getObjectMeta(objectName) {
|
|
1376
|
+
const cached = this.objectCache.get(objectName);
|
|
1377
|
+
if (cached && Date.now() - cached.at < this.TTL) return cached;
|
|
1378
|
+
try {
|
|
1379
|
+
const res = await metaApi.get("/objects");
|
|
1380
|
+
const objects = Array.isArray(res) ? res : res.data ?? [];
|
|
1381
|
+
const node = objects.find((o) => o.nameSingular === objectName);
|
|
1382
|
+
if (!node) return null;
|
|
1383
|
+
const fields = new Set((node.fields ?? []).map((f) => f.name));
|
|
1384
|
+
const entry = { id: node.id, fields, at: Date.now() };
|
|
1385
|
+
this.objectCache.set(objectName, entry);
|
|
1386
|
+
return entry;
|
|
1387
|
+
} catch {
|
|
1388
|
+
return null;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
async createField(objectMetadataId, name, type) {
|
|
1392
|
+
try {
|
|
1393
|
+
await metaApi.post("/fields", {
|
|
1394
|
+
objectMetadataId,
|
|
1395
|
+
name,
|
|
1396
|
+
label: toLabel(name),
|
|
1397
|
+
type,
|
|
1398
|
+
description: "Auto-created by Intake",
|
|
1399
|
+
icon: "IconPlug"
|
|
1400
|
+
});
|
|
1401
|
+
return true;
|
|
1402
|
+
} catch {
|
|
1403
|
+
return false;
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
};
|
|
1407
|
+
function toLabel(name) {
|
|
1408
|
+
return name.replace(/^ext_/, "").replace(/_/g, " ").replace(/([A-Z])/g, " $1").split(" ").map((w) => w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()).join(" ").trim();
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
// src/lib/note-formatter.ts
|
|
1412
|
+
function formatNote(sourceName, fields) {
|
|
1413
|
+
const lines = [];
|
|
1414
|
+
lines.push(`**Source:** ${sourceName}`);
|
|
1415
|
+
const entries = buildEntries(fields);
|
|
1416
|
+
for (const { label, value } of entries) {
|
|
1417
|
+
const formatted = formatValue(value);
|
|
1418
|
+
if (formatted !== null) {
|
|
1419
|
+
lines.push(`**${label}:** ${formatted}`);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
return lines.join("\n");
|
|
1423
|
+
}
|
|
1424
|
+
function buildEntries(fields) {
|
|
1425
|
+
return Object.entries(fields).map(([key, value]) => ({
|
|
1426
|
+
label: toLabel2(key),
|
|
1427
|
+
value
|
|
1428
|
+
}));
|
|
1429
|
+
}
|
|
1430
|
+
function toLabel2(key) {
|
|
1431
|
+
return key.replace(/^ext_/, "").replace(/_/g, " ").replace(/([A-Z])/g, " $1").split(" ").map((w) => w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()).join(" ").trim();
|
|
1432
|
+
}
|
|
1433
|
+
function formatValue(value) {
|
|
1434
|
+
if (value === null || value === void 0 || value === "") return null;
|
|
1435
|
+
if (Array.isArray(value)) return value.join(", ");
|
|
1436
|
+
if (typeof value === "object") {
|
|
1437
|
+
try {
|
|
1438
|
+
return JSON.stringify(value, null, 2);
|
|
1439
|
+
} catch {
|
|
1440
|
+
return String(value);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
return String(value);
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// src/lib/lookup.ts
|
|
1447
|
+
async function firstMatch(collection, query) {
|
|
1448
|
+
try {
|
|
1449
|
+
const res = await coreApi.get(`/${collection}?${query}&first=1`);
|
|
1450
|
+
return res.data?.[collection]?.edges?.[0]?.node ?? null;
|
|
1451
|
+
} catch {
|
|
1452
|
+
return null;
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
async function findPersonByEmail(email) {
|
|
1456
|
+
if (!email) return null;
|
|
1457
|
+
return firstMatch("people", `filter=emails.primaryEmail[eq]:${encodeURIComponent(email)}`);
|
|
1458
|
+
}
|
|
1459
|
+
async function findCompanyByDomain(domain) {
|
|
1460
|
+
if (!domain) return null;
|
|
1461
|
+
const host = extractHost(domain);
|
|
1462
|
+
if (!host) return null;
|
|
1463
|
+
return firstMatch(
|
|
1464
|
+
"companies",
|
|
1465
|
+
`filter=domainName.primaryLinkUrl[like]:%25${encodeURIComponent(host)}%25`
|
|
1466
|
+
);
|
|
1467
|
+
}
|
|
1468
|
+
function extractHost(url) {
|
|
1469
|
+
try {
|
|
1470
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
1471
|
+
} catch {
|
|
1472
|
+
return url;
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
// src/lib/stages/upsert-record.stage.ts
|
|
1477
|
+
var STANDARD_BY_OBJECT = {
|
|
1478
|
+
person: /* @__PURE__ */ new Set([
|
|
1479
|
+
"name",
|
|
1480
|
+
"emails",
|
|
1481
|
+
"phones",
|
|
1482
|
+
"firstName",
|
|
1483
|
+
"lastName",
|
|
1484
|
+
"fullName",
|
|
1485
|
+
"email",
|
|
1486
|
+
"phone",
|
|
1487
|
+
"linkedInLink",
|
|
1488
|
+
"xLink",
|
|
1489
|
+
"jobTitle",
|
|
1490
|
+
"city",
|
|
1491
|
+
"companyName",
|
|
1492
|
+
"domainName",
|
|
1493
|
+
"address"
|
|
1494
|
+
]),
|
|
1495
|
+
company: /* @__PURE__ */ new Set([
|
|
1496
|
+
"name",
|
|
1497
|
+
"domainName",
|
|
1498
|
+
"address",
|
|
1499
|
+
"companyName",
|
|
1500
|
+
"linkedInLink",
|
|
1501
|
+
"xLink",
|
|
1502
|
+
"employees",
|
|
1503
|
+
"annualRecurringRevenue",
|
|
1504
|
+
"idealCustomerProfile",
|
|
1505
|
+
"city"
|
|
1506
|
+
]),
|
|
1507
|
+
opportunity: /* @__PURE__ */ new Set(["name", "stage", "amount", "closeDate"])
|
|
1508
|
+
};
|
|
1509
|
+
var UpsertRecordStage = class {
|
|
1510
|
+
name = "upsert-record";
|
|
1511
|
+
async execute(ctx) {
|
|
1512
|
+
if (!ctx.crmFields) return err(ErrorCode.INTERNAL_ERROR, "No CRM fields to upsert");
|
|
1513
|
+
const cleanFields = ctx.crmFields.filter((f) => !f.originalKey.startsWith("__"));
|
|
1514
|
+
const { assembled, remainder } = assembleComposites(cleanFields);
|
|
1515
|
+
for (const f of remainder) assembled[f.canonicalName] = f.value;
|
|
1516
|
+
const structuredCompany = ctx.structuredCompany;
|
|
1517
|
+
const structuredPerson = ctx.structuredPerson;
|
|
1518
|
+
const rawCompanyName = structuredCompany?.["name"] ?? assembled["companyName"];
|
|
1519
|
+
const customFor = (object) => {
|
|
1520
|
+
const out = {};
|
|
1521
|
+
for (const f of ctx.crmFields ?? []) {
|
|
1522
|
+
if (f.targetObject !== object || f.writable === false || f.writable === void 0) continue;
|
|
1523
|
+
if (STANDARD_BY_OBJECT[object].has(f.canonicalName)) continue;
|
|
1524
|
+
out[f.canonicalName] = formatExtValue(f.value, f.twentyType);
|
|
1525
|
+
}
|
|
1526
|
+
return out;
|
|
1527
|
+
};
|
|
1528
|
+
const policy = ctx.source?.mergePolicy ?? mergePolicy();
|
|
1529
|
+
const strategyByField = /* @__PURE__ */ new Map();
|
|
1530
|
+
for (const f of ctx.crmFields ?? []) {
|
|
1531
|
+
if (f.mergeStrategy && f.mergeStrategy !== "INHERIT") {
|
|
1532
|
+
strategyByField.set(f.canonicalName, f.mergeStrategy);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
const strategyFor = (field) => strategyByField.get(field) ?? "INHERIT";
|
|
1536
|
+
const patchExisting = async (collection, object, existing, incoming) => {
|
|
1537
|
+
const { patch, decisions } = mergePatch(existing, incoming, policy, strategyFor);
|
|
1538
|
+
ctx.mergeDecisions = { ...ctx.mergeDecisions ?? {}, [object]: decisions };
|
|
1539
|
+
const preserved = Object.entries(decisions).filter(([, d]) => d.status === "preserved" && d.write === false).map(([field]) => field);
|
|
1540
|
+
if (preserved.length > 0 && policy === "PRESERVE") {
|
|
1541
|
+
ctx.warnings.push(
|
|
1542
|
+
`Kept the existing ${object} values for ${preserved.join(", ")} \u2014 INTAKE_MERGE_POLICY is PRESERVE`
|
|
1543
|
+
);
|
|
1544
|
+
}
|
|
1545
|
+
if (Object.keys(patch).length === 0) return;
|
|
1546
|
+
await coreApi.patch(`/${collection}/${existing.id}`, patch);
|
|
1547
|
+
};
|
|
1548
|
+
try {
|
|
1549
|
+
let companyId;
|
|
1550
|
+
let personId;
|
|
1551
|
+
const needsCompany = rawCompanyName || ctx.source?.targetObject === "COMPANY";
|
|
1552
|
+
if (needsCompany) {
|
|
1553
|
+
companyId = await this.upsertCompany(assembled, structuredCompany ?? {}, rawCompanyName, customFor("company"), ctx, patchExisting);
|
|
1554
|
+
}
|
|
1555
|
+
const personData = structuredPerson ?? assembled;
|
|
1556
|
+
const hasPersonData = personData["emails"] || personData["name"] || assembled["emails"] || assembled["name"];
|
|
1557
|
+
if (hasPersonData || ctx.source?.targetObject === "PERSON") {
|
|
1558
|
+
personId = await this.upsertPerson(assembled, structuredPerson, companyId, customFor("person"), ctx, patchExisting);
|
|
1559
|
+
}
|
|
1560
|
+
let opportunityId;
|
|
1561
|
+
if (ctx.source?.createOpportunity && (personId || companyId)) {
|
|
1562
|
+
opportunityId = await this.createOpportunity(assembled, personId, companyId, ctx.crmFields ?? [], customFor("opportunity"), ctx);
|
|
1563
|
+
}
|
|
1564
|
+
let noteId;
|
|
1565
|
+
const noteFields = ctx.noteFields ?? [];
|
|
1566
|
+
if (noteFields.length > 0) {
|
|
1567
|
+
const overflow = {};
|
|
1568
|
+
for (const f of noteFields) overflow[f.canonicalName] = f.value;
|
|
1569
|
+
noteId = await this.createNote(overflow, personId, companyId, ctx);
|
|
1570
|
+
}
|
|
1571
|
+
return ok({
|
|
1572
|
+
...ctx,
|
|
1573
|
+
personId,
|
|
1574
|
+
companyId,
|
|
1575
|
+
opportunityId,
|
|
1576
|
+
noteId,
|
|
1577
|
+
recordType: personId ? "PERSON" : "COMPANY"
|
|
1578
|
+
});
|
|
1579
|
+
} catch (e) {
|
|
1580
|
+
return err(ErrorCode.RECORD_UPSERT_FAILED, `Failed to upsert record: ${String(e)}`);
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
async upsertCompany(assembled, structuredCompany, companyName, customData, ctx, patchExisting) {
|
|
1584
|
+
const standard = { ...structuredCompany };
|
|
1585
|
+
if (companyName && !standard["name"]) standard["name"] = companyName;
|
|
1586
|
+
if (!standard["domainName"] && assembled["domainName"]) standard["domainName"] = assembled["domainName"];
|
|
1587
|
+
if (!standard["address"] && assembled["address"]) standard["address"] = assembled["address"];
|
|
1588
|
+
const domain = standard["domainName"]?.["primaryLinkUrl"];
|
|
1589
|
+
let companyId;
|
|
1590
|
+
if (domain) {
|
|
1591
|
+
const existing = await findCompanyByDomain(domain);
|
|
1592
|
+
if (existing) {
|
|
1593
|
+
await patchExisting("companies", "company", existing, { ...standard, ...customData });
|
|
1594
|
+
return existing.id;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
if (!companyId) {
|
|
1598
|
+
try {
|
|
1599
|
+
const created = await coreApi.post("/companies", standard);
|
|
1600
|
+
companyId = created.data.createCompany.id;
|
|
1601
|
+
} catch (e) {
|
|
1602
|
+
if (String(e).toLowerCase().includes("duplicate") && standard["name"]) {
|
|
1603
|
+
try {
|
|
1604
|
+
const found = await gql(
|
|
1605
|
+
`query FindCompany($name: String!) { companies(filter: { name: { eq: $name } }) { edges { node { id } } } }`,
|
|
1606
|
+
{ name: String(standard["name"]) }
|
|
1607
|
+
);
|
|
1608
|
+
companyId = found.data?.companies?.edges?.[0]?.node?.id;
|
|
1609
|
+
} catch {
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
if (!companyId) throw e;
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
if (Object.keys(customData).length > 0) {
|
|
1616
|
+
await coreApi.patch(`/companies/${companyId}`, customData).catch((e) => {
|
|
1617
|
+
ctx.warnings.push(`Could not set custom fields on company: ${e.message.slice(0, 120)}`);
|
|
1618
|
+
});
|
|
1619
|
+
}
|
|
1620
|
+
return companyId;
|
|
1621
|
+
}
|
|
1622
|
+
async upsertPerson(assembled, structuredPerson, companyId, customData, ctx, patchExisting) {
|
|
1623
|
+
const standard = structuredPerson ? { ...structuredPerson } : {};
|
|
1624
|
+
if (!standard["name"] && assembled["name"]) standard["name"] = assembled["name"];
|
|
1625
|
+
if (!standard["emails"] && assembled["emails"]) standard["emails"] = assembled["emails"];
|
|
1626
|
+
if (!standard["phones"] && assembled["phones"]) standard["phones"] = assembled["phones"];
|
|
1627
|
+
if (companyId) standard["companyId"] = companyId;
|
|
1628
|
+
const emails = standard["emails"] ?? assembled["emails"];
|
|
1629
|
+
const email = emails?.["primaryEmail"];
|
|
1630
|
+
let personId;
|
|
1631
|
+
if (email) {
|
|
1632
|
+
const existing = await findPersonByEmail(email);
|
|
1633
|
+
if (existing) {
|
|
1634
|
+
await patchExisting("people", "person", existing, { ...standard, ...customData });
|
|
1635
|
+
return existing.id;
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
if (!personId) {
|
|
1639
|
+
const created = await coreApi.post("/people", standard);
|
|
1640
|
+
personId = created.data.createPerson.id;
|
|
1641
|
+
}
|
|
1642
|
+
if (Object.keys(customData).length > 0) {
|
|
1643
|
+
await coreApi.patch(`/people/${personId}`, customData).catch((e) => {
|
|
1644
|
+
ctx.warnings.push(`Could not set custom fields on person: ${e.message.slice(0, 120)}`);
|
|
1645
|
+
});
|
|
1646
|
+
}
|
|
1647
|
+
return personId;
|
|
1648
|
+
}
|
|
1649
|
+
async createOpportunity(assembled, personId, companyId, crmFields, customData, ctx) {
|
|
1650
|
+
let opportunityId;
|
|
1651
|
+
try {
|
|
1652
|
+
const name = buildOpportunityName(ctx, assembled);
|
|
1653
|
+
const stage = defaultOpportunityStage();
|
|
1654
|
+
const data = { name, stage };
|
|
1655
|
+
if (personId) data["pointOfContactId"] = personId;
|
|
1656
|
+
if (companyId) data["companyId"] = companyId;
|
|
1657
|
+
for (const f of crmFields) {
|
|
1658
|
+
if (f.targetObject !== "opportunity" || !f.writable) continue;
|
|
1659
|
+
if (f.canonicalName === "amount" || f.canonicalName === "closeDate") {
|
|
1660
|
+
data[f.canonicalName] = formatExtValue(f.value, f.twentyType);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
const created = await coreApi.post("/opportunities", data);
|
|
1664
|
+
opportunityId = created.data.createOpportunity.id;
|
|
1665
|
+
} catch {
|
|
1666
|
+
ctx.warnings.push("Opportunity creation failed \u2014 record was still created");
|
|
1667
|
+
return void 0;
|
|
1668
|
+
}
|
|
1669
|
+
if (Object.keys(customData).length > 0) {
|
|
1670
|
+
await coreApi.patch(`/opportunities/${opportunityId}`, customData).catch((e) => {
|
|
1671
|
+
ctx.warnings.push(`Could not set custom fields on opportunity: ${e.message.slice(0, 120)}`);
|
|
1672
|
+
});
|
|
1673
|
+
}
|
|
1674
|
+
return opportunityId;
|
|
1675
|
+
}
|
|
1676
|
+
async createNote(overflow, personId, companyId, ctx) {
|
|
1677
|
+
try {
|
|
1678
|
+
const markdown = formatNote(ctx.source?.name ?? ctx.sourceSlug, overflow);
|
|
1679
|
+
const label = appLabel();
|
|
1680
|
+
const created = await coreApi.post("/notes", {
|
|
1681
|
+
title: `${ctx.source?.name ?? label} \u2014 Lead`,
|
|
1682
|
+
bodyV2: { markdown }
|
|
1683
|
+
});
|
|
1684
|
+
const noteId = created.data.createNote.id;
|
|
1685
|
+
if (personId) {
|
|
1686
|
+
await coreApi.post("/noteTargets", { noteId, personId }).catch(() => null);
|
|
1687
|
+
}
|
|
1688
|
+
if (companyId) {
|
|
1689
|
+
await coreApi.post("/noteTargets", { noteId, companyId }).catch(() => null);
|
|
1690
|
+
}
|
|
1691
|
+
return noteId;
|
|
1692
|
+
} catch {
|
|
1693
|
+
ctx.warnings.push("Note creation failed \u2014 record was still created");
|
|
1694
|
+
return void 0;
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
};
|
|
1698
|
+
function buildOpportunityName(ctx, assembled) {
|
|
1699
|
+
const label = appLabel();
|
|
1700
|
+
const defaultTemplate = `{{source}} \u2014 {{firstName}} {{lastName}}`;
|
|
1701
|
+
const template = ctx.source?.opportunityNameTemplate ?? defaultTemplate;
|
|
1702
|
+
const name = assembled["name"];
|
|
1703
|
+
return template.replace("{{source}}", ctx.source?.name ?? label).replace("{{firstName}}", name?.["firstName"] ?? "").replace("{{lastName}}", name?.["lastName"] ?? "").replace("{{email}}", assembled["emails"]?.["primaryEmail"] ?? "").replace("{{company}}", assembled["companyName"] ?? "").trim().replace(/\s+/g, " ").replace(/\s*—\s*$/, "");
|
|
1704
|
+
}
|
|
1705
|
+
function formatExtValue(value, type) {
|
|
1706
|
+
if (type === "LINKS" && typeof value === "string") {
|
|
1707
|
+
return { primaryLinkUrl: value, primaryLinkLabel: "" };
|
|
1708
|
+
}
|
|
1709
|
+
if (type === "CURRENCY") {
|
|
1710
|
+
const amount = typeof value === "number" ? value : Number(String(value ?? "").replace(/[^\d.]/g, ""));
|
|
1711
|
+
if (!Number.isFinite(amount) || amount <= 0) return void 0;
|
|
1712
|
+
return { amountMicros: Math.round(amount * 1e6), currencyCode: "USD" };
|
|
1713
|
+
}
|
|
1714
|
+
return value;
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
// src/lib/retention.ts
|
|
1718
|
+
var CREDENTIAL_KEY_PATTERNS = [
|
|
1719
|
+
"password",
|
|
1720
|
+
"passwd",
|
|
1721
|
+
"secret",
|
|
1722
|
+
"token",
|
|
1723
|
+
"apikey",
|
|
1724
|
+
"api_key",
|
|
1725
|
+
"authorization",
|
|
1726
|
+
"auth",
|
|
1727
|
+
"credential",
|
|
1728
|
+
"privatekey",
|
|
1729
|
+
"private_key",
|
|
1730
|
+
"session",
|
|
1731
|
+
"cookie",
|
|
1732
|
+
"ssn",
|
|
1733
|
+
"socialsecurity",
|
|
1734
|
+
"creditcard",
|
|
1735
|
+
"credit_card",
|
|
1736
|
+
"cardnumber",
|
|
1737
|
+
"card_number",
|
|
1738
|
+
"cvv",
|
|
1739
|
+
"cvc",
|
|
1740
|
+
"pin",
|
|
1741
|
+
"accesskey",
|
|
1742
|
+
"access_key"
|
|
1743
|
+
];
|
|
1744
|
+
var REDACTED = "[redacted]";
|
|
1745
|
+
function isCredentialKey(key) {
|
|
1746
|
+
const norm = key.toLowerCase().replace(/[-\s]/g, "_");
|
|
1747
|
+
return CREDENTIAL_KEY_PATTERNS.some((p) => norm.includes(p));
|
|
1748
|
+
}
|
|
1749
|
+
function redactCredentials(value, depth = 0) {
|
|
1750
|
+
if (depth > 12 || value === null || typeof value !== "object") return value;
|
|
1751
|
+
if (Array.isArray(value)) return value.map((v) => redactCredentials(v, depth + 1));
|
|
1752
|
+
const out = {};
|
|
1753
|
+
for (const [key, v] of Object.entries(value)) {
|
|
1754
|
+
out[key] = isCredentialKey(key) ? REDACTED : redactCredentials(v, depth + 1);
|
|
1755
|
+
}
|
|
1756
|
+
return out;
|
|
1757
|
+
}
|
|
1758
|
+
function prepareForStorage(payload) {
|
|
1759
|
+
if (rawPayloadRetention() === "NONE") {
|
|
1760
|
+
return { stored: null, warning: null };
|
|
1761
|
+
}
|
|
1762
|
+
let json;
|
|
1763
|
+
try {
|
|
1764
|
+
json = JSON.stringify(redactCredentials(payload));
|
|
1765
|
+
} catch {
|
|
1766
|
+
return { stored: null, warning: "Payload could not be serialised \u2014 not stored, so replay is unavailable for this log" };
|
|
1767
|
+
}
|
|
1768
|
+
const max = rawPayloadMaxBytes();
|
|
1769
|
+
if (Buffer.byteLength(json, "utf8") > max) {
|
|
1770
|
+
return {
|
|
1771
|
+
stored: null,
|
|
1772
|
+
warning: `Payload exceeds INTAKE_RAW_PAYLOAD_MAX_BYTES (${max}) \u2014 not stored, so replay is unavailable for this log`
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
return { stored: json, warning: null };
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
// src/lib/intake-log-writer.ts
|
|
1779
|
+
async function writeIntakeLog(input) {
|
|
1780
|
+
const { ctx, status } = input;
|
|
1781
|
+
const retained = prepareForStorage(ctx.rawPayload);
|
|
1782
|
+
if (retained.warning) ctx.warnings.push(retained.warning);
|
|
1783
|
+
try {
|
|
1784
|
+
const created = await coreApi.post("/intakeLogs", {
|
|
1785
|
+
intakeSourceId: ctx.source?.id ?? null,
|
|
1786
|
+
status,
|
|
1787
|
+
payloadHash: ctx.payloadHash ?? "",
|
|
1788
|
+
idempotencyKey: ctx.idempotencyKey ?? null,
|
|
1789
|
+
rawPayload: retained.stored,
|
|
1790
|
+
recordId: ctx.personId ?? ctx.companyId ?? null,
|
|
1791
|
+
recordType: ctx.recordType ?? null,
|
|
1792
|
+
fieldsCreated: ctx.fieldsCreated,
|
|
1793
|
+
fieldsMatched: ctx.fieldsMatched,
|
|
1794
|
+
overflowCount: ctx.noteFields?.length ?? 0,
|
|
1795
|
+
processingMs: Date.now() - ctx.startedAt,
|
|
1796
|
+
processedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1797
|
+
error: input.error ?? null,
|
|
1798
|
+
spamScore: input.spamScore ?? ctx.spam?.score ?? null,
|
|
1799
|
+
quarantineReason: input.quarantineReason ?? null,
|
|
1800
|
+
replayOfLogId: ctx.replayOfLogId ?? null
|
|
1801
|
+
});
|
|
1802
|
+
return created.data?.createIntakeLog?.id ?? null;
|
|
1803
|
+
} catch {
|
|
1804
|
+
ctx.warnings.push("Intake log write failed");
|
|
1805
|
+
return null;
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
async function bumpSourceCounters(sourceId, outcome) {
|
|
1809
|
+
try {
|
|
1810
|
+
const current = await coreApi.get(`/intakeSources/${sourceId}`);
|
|
1811
|
+
const node = current.data?.intakeSource;
|
|
1812
|
+
const patch = {};
|
|
1813
|
+
if (outcome === "ingested") {
|
|
1814
|
+
patch["totalIngested"] = (node?.totalIngested ?? 0) + 1;
|
|
1815
|
+
patch["lastIngestedAt"] = (/* @__PURE__ */ new Date()).toISOString();
|
|
1816
|
+
patch["healthStatus"] = "HEALTHY";
|
|
1817
|
+
patch["silentSince"] = null;
|
|
1818
|
+
} else {
|
|
1819
|
+
patch["totalFailed"] = (node?.totalFailed ?? 0) + 1;
|
|
1820
|
+
}
|
|
1821
|
+
await coreApi.patch(`/intakeSources/${sourceId}`, patch);
|
|
1822
|
+
} catch {
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
// src/lib/stages/write-log.stage.ts
|
|
1827
|
+
var WriteLogStage = class {
|
|
1828
|
+
name = "write-log";
|
|
1829
|
+
async execute(ctx) {
|
|
1830
|
+
const status = ctx.personId || ctx.companyId ? "SUCCESS" : "PARTIAL";
|
|
1831
|
+
await writeIntakeLog({ ctx, status });
|
|
1832
|
+
if (ctx.source) await bumpSourceCounters(ctx.source.id, "ingested");
|
|
1833
|
+
return ok(ctx);
|
|
1834
|
+
}
|
|
1835
|
+
};
|
|
1836
|
+
|
|
1837
|
+
// src/lib/pipeline-factory.ts
|
|
1838
|
+
function createPipeline(coreClient, metadataClient) {
|
|
1839
|
+
return new Pipeline().use(new LoadSourceStage(coreClient)).use(new VerifySignatureStage()).use(new IdempotencyStage(coreClient)).use(new FlattenStage()).use(new NormalizeStage(coreClient)).use(new QuarantineStage()).use(new SyncSchemaStage(metadataClient)).use(new UpsertRecordStage(coreClient)).use(new WriteLogStage(coreClient));
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// src/lib/ingest.ts
|
|
1843
|
+
async function runIngestion(options) {
|
|
1844
|
+
const ctx = {
|
|
1845
|
+
sourceSlug: options.sourceSlug,
|
|
1846
|
+
rawBody: options.rawBody,
|
|
1847
|
+
rawPayload: options.rawPayload,
|
|
1848
|
+
headers: options.headers ?? {},
|
|
1849
|
+
startedAt: Date.now(),
|
|
1850
|
+
fieldsCreated: 0,
|
|
1851
|
+
fieldsMatched: 0,
|
|
1852
|
+
warnings: [],
|
|
1853
|
+
events: [],
|
|
1854
|
+
...options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {},
|
|
1855
|
+
...options.skipDedup ? { skipDedup: true } : {},
|
|
1856
|
+
...options.skipSpamFilter ? { skipSpamFilter: true } : {},
|
|
1857
|
+
...options.replayOfLogId ? { replayOfLogId: options.replayOfLogId } : {}
|
|
1858
|
+
};
|
|
1859
|
+
const pipeline = createPipeline(new CoreApiClient(), new MetadataApiClient());
|
|
1860
|
+
const result = await pipeline.run(ctx);
|
|
1861
|
+
if (result.ok) return { ok: true, ctx: result.value };
|
|
1862
|
+
await recordFailure(ctx, result.error);
|
|
1863
|
+
return { ok: false, error: result.error, ctx };
|
|
1864
|
+
}
|
|
1865
|
+
async function recordFailure(ctx, error) {
|
|
1866
|
+
if (error.code === "SOURCE_NOT_FOUND") return;
|
|
1867
|
+
if (error.code === "DUPLICATE_PAYLOAD") return;
|
|
1868
|
+
const details = error.details ?? {};
|
|
1869
|
+
const quarantined = error.code === "QUARANTINED";
|
|
1870
|
+
const status = quarantined ? "QUARANTINED" : "FAILED";
|
|
1871
|
+
await writeIntakeLog({
|
|
1872
|
+
ctx,
|
|
1873
|
+
status,
|
|
1874
|
+
error: `${error.code}: ${error.message}`,
|
|
1875
|
+
spamScore: details.spamScore ?? null,
|
|
1876
|
+
quarantineReason: quarantined ? details.reason ?? error.message : null
|
|
1877
|
+
});
|
|
1878
|
+
if (ctx.source && !quarantined) await bumpSourceCounters(ctx.source.id, "failed");
|
|
1879
|
+
}
|
|
1880
|
+
|
|
1881
|
+
// src/lib/replay.ts
|
|
1882
|
+
async function loadLog(logId) {
|
|
1883
|
+
try {
|
|
1884
|
+
const res = await coreApi.get(`/intakeLogs/${logId}`);
|
|
1885
|
+
return res.data?.intakeLog ?? null;
|
|
1886
|
+
} catch {
|
|
1887
|
+
return null;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
async function slugForSource(sourceId) {
|
|
1891
|
+
if (!sourceId) return null;
|
|
1892
|
+
try {
|
|
1893
|
+
const res = await coreApi.get(`/intakeSources/${sourceId}`);
|
|
1894
|
+
return res.data?.intakeSource?.slug ?? null;
|
|
1895
|
+
} catch {
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
async function replayLog(log) {
|
|
1900
|
+
if (!log.rawPayload) {
|
|
1901
|
+
return {
|
|
1902
|
+
logId: log.id,
|
|
1903
|
+
ok: false,
|
|
1904
|
+
reason: "No stored payload \u2014 retention was off, or the payload exceeded the size cap when it arrived"
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
let payload;
|
|
1908
|
+
try {
|
|
1909
|
+
const parsed = JSON.parse(log.rawPayload);
|
|
1910
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1911
|
+
return { logId: log.id, ok: false, reason: "Stored payload is not a JSON object" };
|
|
1912
|
+
}
|
|
1913
|
+
payload = parsed;
|
|
1914
|
+
} catch {
|
|
1915
|
+
return { logId: log.id, ok: false, reason: "Stored payload is not valid JSON" };
|
|
1916
|
+
}
|
|
1917
|
+
const slug = await slugForSource(log.intakeSourceId);
|
|
1918
|
+
if (!slug) {
|
|
1919
|
+
return { logId: log.id, ok: false, reason: "The source for this log no longer exists" };
|
|
1920
|
+
}
|
|
1921
|
+
const outcome = await runIngestion({
|
|
1922
|
+
sourceSlug: slug,
|
|
1923
|
+
rawPayload: payload,
|
|
1924
|
+
// The signature was checked when the payload first arrived; there is no body
|
|
1925
|
+
// to re-verify against and the request is now coming from an authenticated
|
|
1926
|
+
// operator rather than the network.
|
|
1927
|
+
skipDedup: true,
|
|
1928
|
+
replayOfLogId: log.id
|
|
1929
|
+
});
|
|
1930
|
+
if (!outcome.ok) {
|
|
1931
|
+
return { logId: log.id, ok: false, reason: `${outcome.error.code}: ${outcome.error.message}` };
|
|
1932
|
+
}
|
|
1933
|
+
return {
|
|
1934
|
+
logId: log.id,
|
|
1935
|
+
ok: true,
|
|
1936
|
+
recordId: outcome.ctx.personId ?? outcome.ctx.companyId,
|
|
1937
|
+
personId: outcome.ctx.personId,
|
|
1938
|
+
companyId: outcome.ctx.companyId,
|
|
1939
|
+
opportunityId: outcome.ctx.opportunityId,
|
|
1940
|
+
fieldsCreated: outcome.ctx.fieldsCreated,
|
|
1941
|
+
warnings: outcome.ctx.warnings
|
|
1942
|
+
};
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
// src/logic-functions/replay-log.ts
|
|
1946
|
+
var handler = async (params) => {
|
|
1947
|
+
const logId = params.pathParameters?.["logId"];
|
|
1948
|
+
if (!logId) return { statusCode: 400, body: { error: "Missing logId in URL" } };
|
|
1949
|
+
const log = await loadLog(logId);
|
|
1950
|
+
if (!log) return { statusCode: 404, body: { error: `IntakeLog ${logId} not found` } };
|
|
1951
|
+
const result = await replayLog(log);
|
|
1952
|
+
if (!result.ok) {
|
|
1953
|
+
return {
|
|
1954
|
+
statusCode: result.reason?.startsWith("No stored payload") ? 400 : 500,
|
|
1955
|
+
body: { error: result.reason, logId, code: "REPLAY_FAILED" }
|
|
1956
|
+
};
|
|
1957
|
+
}
|
|
1958
|
+
return {
|
|
1959
|
+
statusCode: 200,
|
|
1960
|
+
body: {
|
|
1961
|
+
success: true,
|
|
1962
|
+
replayedLogId: logId,
|
|
1963
|
+
recordId: result.recordId,
|
|
1964
|
+
personId: result.personId,
|
|
1965
|
+
companyId: result.companyId,
|
|
1966
|
+
opportunityId: result.opportunityId,
|
|
1967
|
+
fieldsCreated: result.fieldsCreated,
|
|
1968
|
+
warnings: result.warnings
|
|
1969
|
+
}
|
|
1970
|
+
};
|
|
1971
|
+
};
|
|
1972
|
+
var replay_log_default = defineLogicFunction({
|
|
1973
|
+
universalIdentifier: IDS.REPLAY_LOG_LOGIC_FUNCTION,
|
|
1974
|
+
name: "intake-replay-log",
|
|
1975
|
+
description: "Re-run a single stored payload through the current field rules.",
|
|
1976
|
+
timeoutSeconds: 30,
|
|
1977
|
+
handler,
|
|
1978
|
+
httpRouteTriggerSettings: {
|
|
1979
|
+
path: "/intake/logs/:logId/replay",
|
|
1980
|
+
httpMethod: "POST",
|
|
1981
|
+
isAuthRequired: true
|
|
1982
|
+
}
|
|
1983
|
+
});
|
|
1984
|
+
export {
|
|
1985
|
+
replay_log_default as default
|
|
1986
|
+
};
|
|
1987
|
+
//# sourceMappingURL=replay-log.mjs.map
|