thinkwork-cli 0.13.1 → 1.0.0

package/dist/plugins/twenty/scripts/lib/load-records.ts

@@ -24,15 +24,24 @@ import {
 
 const QUERY_PAGE = 200;
 
-/** True when Twenty rejects a filter because the sourceId custom field does
- * not exist yet (virgin workspace before schema-ensure applies — the dry-run
- * plans the field but cannot create it). No field ⇒ no record carries it. */
+/**
+ * True when Twenty rejects a query because schema this run would create does
+ * not exist yet: a missing `sourceId` custom field, or a whole missing custom
+ * object (`opportunityProduct`, `organization`). A dry-run plans that schema
+ * but cannot create it, so nothing can carry a sourceId and the honest answer
+ * to "what already exists?" is "nothing".
+ *
+ * Apply runs create objects and fields before any record pass, so this can
+ * only fire in dry-run; a genuine apply-time schema gap aborts in the metadata
+ * phase, before record loading.
+ */
 export function isMissingSourceIdFieldError(error: unknown): boolean {
-  return (
-    error instanceof TwentyGraphqlError &&
-    error.errors.some((entry) =>
-      /doesn't have any .?sourceId.? field/i.test(entry.message),
-    )
+  if (!(error instanceof TwentyGraphqlError)) return false;
+  return error.errors.some(
+    (entry) =>
+      /doesn't have any .?sourceId.? field/i.test(entry.message) ||
+      /Unknown type "\w+FilterInput"/i.test(entry.message) ||
+      /Cannot query field "\w+" on type "Query"/i.test(entry.message),
   );
 }
 
@@ -71,6 +80,18 @@ export const OPPORTUNITY_PRODUCT: EntityShape = {
   capSingular: "OpportunityProduct",
   capPlural: "OpportunityProducts",
 };
+export const ORGANIZATION: EntityShape = {
+  singular: "organization",
+  plural: "organizations",
+  capSingular: "Organization",
+  capPlural: "Organizations",
+};
+export const PRODUCT: EntityShape = {
+  singular: "product",
+  plural: "products",
+  capSingular: "Product",
+  capPlural: "Products",
+};
 export const NOTE: EntityShape = {
   singular: "note",
   plural: "notes",
@@ -471,11 +492,19 @@ export async function upsertNotes(options: {
     return false;
   });
 
+  // createdAt is replayed from LastMile so the activity feed reads in the order
+  // reps actually wrote the notes. Twenty accepts it on create AND update, so
+  // notes seeded before this behaviour existed heal on the next run (their
+  // content hash changes when createdAt joins it).
   const mapped: MappedRecord[] = resolvable.map((note) => ({
     sourceId: note.sourceId,
     input: {
       title: note.title,
       bodyV2: { markdown: note.bodyMarkdown },
+      ...(note.createdAt
+        ? { createdAt: note.createdAt, updatedAt: note.createdAt }
+        : {}),
+      ...(note.authorName ? { author: note.authorName } : {}),
       sourceId: note.sourceId,
     },
     warnings: [],
@@ -499,41 +528,106 @@ export async function upsertNotes(options: {
 
   // NoteTarget is ensured UNCONDITIONALLY — including for hash-skipped notes —
   // so a crash between the Note write and the NoteTarget write heals on
-  // re-run (plan U6).
-  for (const note of resolvable) {
-    if (dryRun) continue; // note creates already planned; target ensure follows them
-    const noteId = noteIdBySourceId.get(note.sourceId);
-    const targetId = targetIdBySourceId.get(note.targetSourceId);
-    if (!noteId || !targetId) continue;
-    try {
-      const existing = await client.requestWithRetry<{
-        noteTargets: { edges: Array<{ node: { id: string } }> };
-      }>(
-        "/graphql",
-        `query MigrationNoteTargets($filter: NoteTargetFilterInput) {
-          noteTargets(filter: $filter, first: 1) { edges { node { id } } }
-        }`,
-        { filter: { noteId: { eq: noteId } } },
+  // re-run (plan U6). The existence check is BATCHED: one paginated
+  // `noteTargets` query per QUERY_PAGE notes rather than one query per note.
+  // The per-note form dominated a steady-state run — every already-paired note
+  // (thousands of them) cost a round-trip at the API rate limit, ~20 min of a
+  // 26 min run. Batched, the same phase is a handful of queries.
+  if (!dryRun) {
+    const pairs = resolvable
+      .map((note) => ({
+        note,
+        noteId: noteIdBySourceId.get(note.sourceId),
+        targetId: targetIdBySourceId.get(note.targetSourceId),
+      }))
+      .filter(
+        (p): p is { note: MappedNote; noteId: string; targetId: string } =>
+          Boolean(p.noteId) && Boolean(p.targetId),
       );
-      if (existing.noteTargets.edges.length > 0) continue;
-      const targetField =
-        note.targetKind === "company"
+
+    // Which noteIds already have a target? Paginate within each chunk so a
+    // multi-target note can never be truncated into a false "unpaired".
+    type NoteTargetPage = {
+      noteTargets: {
+        edges: Array<{ node: { noteId: string } }>;
+        pageInfo: { hasNextPage: boolean; endCursor: string | null };
+      };
+    };
+    const alreadyPaired = new Set<string>();
+    const noteIds = [...new Set(pairs.map((p) => p.noteId))];
+    for (const page of chunk(noteIds, QUERY_PAGE)) {
+      let after: string | null = null;
+      try {
+        do {
+          const data: NoteTargetPage =
+            await client.requestWithRetry<NoteTargetPage>(
+              "/graphql",
+              `query MigrationNoteTargets($filter: NoteTargetFilterInput, $after: String) {
+              noteTargets(filter: $filter, first: ${QUERY_PAGE}, after: $after) {
+                edges { node { noteId } }
+                pageInfo { hasNextPage endCursor }
+              }
+            }`,
+              { filter: { noteId: { in: [...page] } }, after },
+            );
+          for (const { node } of data.noteTargets.edges)
+            alreadyPaired.add(node.noteId);
+          after = data.noteTargets.pageInfo.hasNextPage
+            ? data.noteTargets.pageInfo.endCursor
+            : null;
+        } while (after);
+      } catch (error) {
+        counters.failed += page.length;
+        counters.gaps.push(
+          `noteTarget lookup batch failed: ${
+            error instanceof Error ? error.message.slice(0, 300) : String(error)
+          }`,
+        );
+      }
+    }
+
+    // Create the missing targets, grouped by FK field, in BATCH_LIMIT chunks
+    // (createNoteTargets takes an array).
+    const toCreate = new Map<
+      string,
+      Array<{ noteId: string; targetId: string }>
+    >();
+    for (const p of pairs) {
+      if (alreadyPaired.has(p.noteId)) continue;
+      const field =
+        p.note.targetKind === "company"
           ? "targetCompanyId"
           : "targetOpportunityId";
-      await client.requestOnce(
-        "/graphql",
-        `mutation MigrationCreateNoteTargets($data: [NoteTargetCreateInput!]!) {
-          createNoteTargets(data: $data) { id }
-        }`,
-        { data: [{ noteId, [targetField]: targetId }] },
-      );
-    } catch (error) {
-      counters.failed += 1;
-      counters.gaps.push(
-        `noteTarget for ${note.sourceId} failed: ${
-          error instanceof Error ? error.message.slice(0, 300) : String(error)
-        }`,
-      );
+      const list = toCreate.get(field) ?? [];
+      list.push({ noteId: p.noteId, targetId: p.targetId });
+      toCreate.set(field, list);
+    }
+    for (const [field, list] of toCreate) {
+      for (const batch of chunk(list, BATCH_LIMIT)) {
+        try {
+          await client.requestOnce(
+            "/graphql",
+            `mutation MigrationCreateNoteTargets($data: [NoteTargetCreateInput!]!) {
+              createNoteTargets(data: $data) { id }
+            }`,
+            {
+              data: batch.map((p) => ({
+                noteId: p.noteId,
+                [field]: p.targetId,
+              })),
+            },
+          );
+        } catch (error) {
+          counters.failed += batch.length;
+          counters.gaps.push(
+            `noteTarget create batch (${field}) failed: ${
+              error instanceof Error
+                ? error.message.slice(0, 300)
+                : String(error)
+            }`,
+          );
+        }
+      }
     }
   }
 
@@ -592,11 +686,19 @@ export async function upsertOpportunityProducts(options: {
   products: MappedOpportunityProduct[];
   /** opportunity sourceId -> Twenty opportunity id. */
   opportunityIdBySourceId: ReadonlyMap<string, string>;
+  /** catalog product sourceId -> Twenty product id. */
+  productIdBySourceId: ReadonlyMap<string, string>;
   dryRun: boolean;
   counters: EntityCounters;
 }): Promise<void> {
-  const { client, products, opportunityIdBySourceId, dryRun, counters } =
-    options;
+  const {
+    client,
+    products,
+    opportunityIdBySourceId,
+    productIdBySourceId,
+    dryRun,
+    counters,
+  } = options;
 
   const resolvable: MappedRecord[] = [];
   for (const product of products) {
@@ -611,9 +713,24 @@ export async function upsertOpportunityProducts(options: {
       );
       continue;
     }
+    // A line whose brand did not map to a catalog product still migrates with
+    // its quantity and amount; it simply has no product. The warning already
+    // rode along from the mapper.
+    const productId = product.productSourceId
+      ? (productIdBySourceId.get(product.productSourceId) ?? null)
+      : null;
+    if (product.productSourceId && !productId) {
+      counters.gaps.push(
+        `line ${product.sourceId}: catalog product ${product.productSourceId} missing`,
+      );
+    }
     resolvable.push({
       sourceId: product.sourceId,
-      input: { ...product.input, opportunityId },
+      input: {
+        ...product.input,
+        opportunityId,
+        ...(productId ? { productId } : {}),
+      },
       warnings: product.warnings,
     });
   }