thinkwork-cli 0.12.15 → 0.12.17

package/dist/plugins/twenty/scripts/lib/lastmile-reader.ts

@@ -0,0 +1,282 @@
+/**
+ * Read-only access to LastMile's dispatch Postgres for the Twenty migration.
+ *
+ * Per plan KTD8: plain `pg` connection from LASTMILE_DATABASE_URL with the
+ * session forced read-only and a statement timeout, following
+ * docs/solutions/security/analyst-external-postgres-role-provisioning-runbook-2026-07.md.
+ * Read queries never mutate LastMile.
+ *
+ * Entity shapes were read from the live `dispatch` database on 2026-07-09 (see
+ * the plan's Outstanding Questions resolutions): leads and opportunities are
+ * separate tables; CRM notes are `task_comment` rows on lead/opportunity tasks;
+ * the `note` table targets dispatch `customer` rows, which only reach Twenty
+ * through an exact-unique-name match to a CRM `account`.
+ */
+
+import pg from "pg";
+
+export interface LastmileRep {
+  id: string;
+  firstName: string | null;
+  lastName: string | null;
+  /** LastMile short username, e.g. "jbake" — owner refs sometimes use it. */
+  alias: string | null;
+  email: string | null;
+  archived: boolean;
+  userIsActive: boolean | null;
+}
+
+export interface LastmileAccount {
+  id: string;
+  name: string | null;
+  ownerRepId: string | null;
+}
+
+export interface LastmileContact {
+  id: string;
+  accountId: string | null;
+  firstName: string | null;
+  lastName: string | null;
+  email: string | null;
+  phone: string | null;
+  phoneCellular: string | null;
+  title: string | null;
+}
+
+export interface LastmileLead {
+  id: string;
+  status: string | null;
+  companyName: string | null;
+  firstName: string | null;
+  lastName: string | null;
+  email: string | null;
+  phone: string | null;
+  source: string | null;
+  description: string | null;
+  ownerRepId: string | null;
+  dateCreated: Date | null;
+}
+
+export interface LastmileOpportunity {
+  id: string;
+  name: string | null;
+  stage: string | null;
+  amount: string | null;
+  quantity: string | null;
+  productType: string | null;
+  brand: string | null;
+  closed: string | null;
+  won: string | null;
+  accountId: string | null;
+  ownerRepId: string | null;
+  expectedCloseDate: string | null;
+  description: string | null;
+  dateCreated: Date | null;
+}
+
+/**
+ * A product line on an opportunity. LastMile keeps them as a JSON array at
+ * `task.entity_data->'items'` on the opportunity's task row — 816 of 2,971
+ * opportunities have one, and 82 carry 2-5 lines. `brand` is the product
+ * ("MOBIL", "GOLDEN WEST", "DEF"); the opportunity row's own `product_type`
+ * is the coarser category ("Fuel", "Equipment").
+ */
+export interface LastmileOpportunityItem {
+  /** Owning opportunity id (lead/opportunity entity_id on the task row). */
+  opportunityId: string;
+  /** Position within the items array — part of the line's stable identity. */
+  index: number;
+  brand: string | null;
+  quantity: string | null;
+  amount: string | null;
+}
+
+export interface LastmileCrmComment {
+  id: string;
+  entityType: "lead" | "opportunity";
+  entityId: string;
+  content: string | null;
+  isDeleted: boolean;
+  createdAt: Date | null;
+}
+
+export interface LastmileCrmAttachment {
+  id: string;
+  entityType: "lead" | "opportunity";
+  entityId: string;
+  filename: string | null;
+  filePath: string | null;
+  fileType: string | null;
+  bucketName: string | null;
+}
+
+export interface LastmileCustomerNote {
+  id: string;
+  noteText: string | null;
+  customerId: string;
+  customerName: string | null;
+  /** Set only when exactly one account matches the customer name. */
+  matchedAccountId: string | null;
+  dateCreated: Date | null;
+}
+
+export interface LastmileReader {
+  readReps(): Promise<LastmileRep[]>;
+  readAccounts(): Promise<LastmileAccount[]>;
+  readContacts(): Promise<LastmileContact[]>;
+  readLeads(): Promise<LastmileLead[]>;
+  readOpportunities(): Promise<LastmileOpportunity[]>;
+  readOpportunityItems(): Promise<LastmileOpportunityItem[]>;
+  readCrmComments(): Promise<LastmileCrmComment[]>;
+  readCrmAttachments(): Promise<LastmileCrmAttachment[]>;
+  readCustomerNotes(): Promise<LastmileCustomerNote[]>;
+  close(): Promise<void>;
+}
+
+export function createLastmileReader(databaseUrl: string): LastmileReader {
+  const pool = new pg.Pool({
+    connectionString: databaseUrl,
+    max: 2,
+    options: "-c default_transaction_read_only=on -c statement_timeout=60000",
+  });
+
+  async function rows<T>(sql: string): Promise<T[]> {
+    const result = await pool.query(sql);
+    return result.rows as T[];
+  }
+
+  return {
+    readReps: () =>
+      rows<LastmileRep>(`
+        select sr.id,
+               sr.first_name  as "firstName",
+               sr.last_name   as "lastName",
+               nullif(trim(sr.alias), '') as "alias",
+               lower(trim(coalesce(nullif(sr.email_address, ''), u.email))) as "email",
+               coalesce(sr.archived, false) as "archived",
+               u.is_active    as "userIsActive"
+        from sales_rep sr
+        left join users u on u.id = sr.user_id
+        order by sr.id
+      `),
+    readAccounts: () =>
+      rows<LastmileAccount>(`
+        select id, nullif(trim(name), '') as "name", nullif(owner, '') as "ownerRepId"
+        from account
+        order by id
+      `),
+    readContacts: () =>
+      rows<LastmileContact>(`
+        select id,
+               nullif(account_id, '')      as "accountId",
+               nullif(trim(first_name), '') as "firstName",
+               nullif(trim(last_name), '')  as "lastName",
+               nullif(trim(email_address), '') as "email",
+               nullif(trim(phone), '')     as "phone",
+               nullif(trim(phone_cellular), '') as "phoneCellular",
+               nullif(trim(title), '')     as "title"
+        from contact
+        order by id
+      `),
+    readLeads: () =>
+      rows<LastmileLead>(`
+        select id,
+               nullif(trim(status), '')     as "status",
+               nullif(trim(company), '')    as "companyName",
+               nullif(trim(first_name), '') as "firstName",
+               nullif(trim(last_name), '')  as "lastName",
+               nullif(trim(email), '')      as "email",
+               nullif(trim(phone), '')      as "phone",
+               nullif(trim(source), '')     as "source",
+               nullif(trim(description), '') as "description",
+               coalesce(nullif(sales_rep_id, ''), nullif(owner, '')) as "ownerRepId",
+               date_created                 as "dateCreated"
+        from lead
+        order by id
+      `),
+    readOpportunities: () =>
+      rows<LastmileOpportunity>(`
+        select id,
+               coalesce(nullif(trim(opp_name), ''), nullif(trim(description), '')) as "name",
+               nullif(trim(stage), '')       as "stage",
+               amount::text                  as "amount",
+               nullif(trim(quantity), '')    as "quantity",
+               nullif(trim(product_type), '') as "productType",
+               nullif(trim(brand), '')       as "brand",
+               nullif(trim(closed), '')      as "closed",
+               nullif(trim(won), '')         as "won",
+               nullif(account_id, '')        as "accountId",
+               coalesce(nullif(sales_rep_id, ''), null) as "ownerRepId",
+               expected_close_date::text     as "expectedCloseDate",
+               nullif(trim(description), '') as "description",
+               date_created                  as "dateCreated"
+        from opportunity
+        order by id
+      `),
+    readOpportunityItems: () =>
+      rows<LastmileOpportunityItem>(`
+        select t.entity_id                       as "opportunityId",
+               (it.ordinality - 1)::int          as "index",
+               nullif(trim(it.value ->> 'brand'), '')    as "brand",
+               nullif(trim(it.value ->> 'quantity'), '') as "quantity",
+               nullif(trim(it.value ->> 'amount'), '')   as "amount"
+        from task t
+        cross join lateral jsonb_array_elements(t.entity_data -> 'items')
+          with ordinality as it(value, ordinality)
+        where t.entity_type = 'opportunity'
+          and t.entity_id is not null
+          and t.entity_data ? 'items'
+          and jsonb_typeof(t.entity_data -> 'items') = 'array'
+        order by t.entity_id, it.ordinality
+      `),
+    readCrmComments: () =>
+      rows<LastmileCrmComment>(`
+        select tc.id,
+               t.entity_type as "entityType",
+               t.entity_id   as "entityId",
+               nullif(trim(tc.content), '') as "content",
+               coalesce(tc.is_deleted, false) as "isDeleted",
+               tc.created_at as "createdAt"
+        from task_comment tc
+        join task t on t.id = tc.task_id
+        where t.entity_type in ('lead', 'opportunity')
+          and t.entity_id is not null
+        order by tc.id
+      `),
+    readCrmAttachments: () =>
+      rows<LastmileCrmAttachment>(`
+        select ta.id,
+               t.entity_type as "entityType",
+               t.entity_id   as "entityId",
+               nullif(trim(ta.filename), '')  as "filename",
+               nullif(trim(ta.file_path), '') as "filePath",
+               nullif(trim(ta.file_type), '') as "fileType",
+               nullif(trim(ta.bucket_name), '') as "bucketName"
+        from task_attachment ta
+        join task t on t.id = ta.task_id
+        where t.entity_type in ('lead', 'opportunity')
+          and t.entity_id is not null
+        order by ta.id
+      `),
+    readCustomerNotes: () =>
+      rows<LastmileCustomerNote>(`
+        with account_names as (
+          select lower(trim(name)) as norm_name, min(id) as account_id, count(*) as n
+          from account
+          where nullif(trim(name), '') is not null
+          group by lower(trim(name))
+        )
+        select n.id,
+               nullif(trim(n.note_text), '') as "noteText",
+               c.id   as "customerId",
+               nullif(trim(c.name), '') as "customerName",
+               case when an.n = 1 then an.account_id else null end as "matchedAccountId",
+               n.date_created as "dateCreated"
+        from note n
+        join customer c on c.id = n.owner_id
+        left join account_names an on an.norm_name = lower(trim(c.name))
+        order by n.id
+      `),
+    close: () => pool.end(),
+  };
+}

package/dist/plugins/twenty/scripts/lib/load-attachments.ts

@@ -0,0 +1,220 @@
+/**
+ * CRM attachment migration (plan U6, KTD6): query the Attachment record by
+ * sourceId BEFORE uploading (never re-upload orphaned binaries on re-runs);
+ * on a miss, fetch the binary from LastMile's S3 bucket, upload via Twenty's
+ * multipart `uploadFile` mutation, then create the Attachment record with the
+ * file reference and the direct target*Id FK. Missing binaries migrate
+ * nothing for that file and are listed in the report.
+ *
+ * Only a handful of LastMile attachments target CRM records (the 2026-07-09
+ * schema read found the other ~12.5k attach to dispatch bills-of-lading and
+ * loadsheets, which do not migrate).
+ */
+
+import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+import type { LastmileCrmAttachment } from "./lastmile-reader";
+import { fetchExistingBySourceIds, type EntityCounters } from "./load-records";
+import { sourceId as makeSourceId } from "./mappers";
+import { TwentyClient, TwentyGraphqlError } from "./twenty-client";
+
+const ATTACHMENT_ENTITY = {
+  singular: "attachment",
+  plural: "attachments",
+  capSingular: "Attachment",
+  capPlural: "Attachments",
+};
+
+export interface AttachmentUploadDeps {
+  /** Fetches the binary; null when the object is missing. Defaults to S3. */
+  fetchBinary?: (
+    attachment: LastmileCrmAttachment,
+  ) => Promise<Uint8Array | null>;
+  fetchImpl?: typeof fetch;
+}
+
+export async function loadCrmAttachments(options: {
+  client: TwentyClient;
+  authToken: string;
+  baseUrl: string;
+  attachments: LastmileCrmAttachment[];
+  /** target sourceId ("lead:…" or "opportunity:…") → Twenty opportunity id. */
+  targetIdBySourceId: ReadonlyMap<string, string>;
+  dryRun: boolean;
+  counters: EntityCounters;
+  deps?: AttachmentUploadDeps;
+}): Promise<void> {
+  const { client, attachments, targetIdBySourceId, dryRun, counters } = options;
+  counters.sourceTotal += attachments.length;
+  const fetchBinary = options.deps?.fetchBinary ?? makeS3BinaryFetcher();
+  const fetchImpl = options.deps?.fetchImpl ?? fetch;
+
+  const bySourceId = new Map(
+    attachments.map((attachment) => [
+      makeSourceId("task_attachment", attachment.id),
+      attachment,
+    ]),
+  );
+  const existing = await fetchExistingBySourceIds(client, ATTACHMENT_ENTITY, [
+    ...bySourceId.keys(),
+  ]);
+
+  for (const [attachmentSourceId, attachment] of bySourceId) {
+    if (existing.has(attachmentSourceId)) {
+      counters.skipped += 1; // Crash-healing: record exists → no re-upload.
+      continue;
+    }
+    const targetSourceId = makeSourceId(
+      attachment.entityType,
+      attachment.entityId,
+    );
+    const targetId = targetIdBySourceId.get(targetSourceId);
+    if (!targetId) {
+      counters.skipped += 1;
+      counters.gaps.push(
+        `attachment ${attachmentSourceId}: target ${targetSourceId} not migrated`,
+      );
+      continue;
+    }
+    if (dryRun) {
+      counters.created += 1;
+      counters.plannedMutations.push(
+        `upload+create attachment ${attachmentSourceId} → ${targetSourceId}`,
+      );
+      continue;
+    }
+
+    try {
+      const binary = await fetchBinary(attachment);
+      if (!binary) {
+        counters.gaps.push(
+          `attachment ${attachmentSourceId}: binary missing (${attachment.bucketName ?? "?"}/${attachment.filePath ?? "?"})`,
+        );
+        continue;
+      }
+      const fileId = await uploadBinary({
+        baseUrl: options.baseUrl,
+        authToken: options.authToken,
+        filename: attachment.filename ?? "attachment",
+        contentType: attachment.fileType ?? "application/octet-stream",
+        binary,
+        fetchImpl,
+        retries: 1, // Upload failure retries once, then reports and continues (U6).
+      });
+      await client.requestOnce(
+        "/graphql",
+        `mutation MigrationCreateAttachments($data: [AttachmentCreateInput!]!) {
+          createAttachments(data: $data) { id }
+        }`,
+        {
+          data: [
+            {
+              name: attachment.filename ?? "attachment",
+              file: { fileId, label: attachment.filename ?? "attachment" },
+              targetOpportunityId: targetId,
+              sourceId: attachmentSourceId,
+            },
+          ],
+        },
+      );
+      counters.created += 1;
+    } catch (error) {
+      counters.failed += 1;
+      counters.gaps.push(
+        `attachment ${attachmentSourceId} failed: ${
+          error instanceof Error ? error.message.slice(0, 300) : String(error)
+        }`,
+      );
+    }
+  }
+}
+
+function makeS3BinaryFetcher(): (
+  attachment: LastmileCrmAttachment,
+) => Promise<Uint8Array | null> {
+  const s3 = new S3Client({});
+  return async (attachment) => {
+    if (!attachment.bucketName || !attachment.filePath) return null;
+    try {
+      const result = await s3.send(
+        new GetObjectCommand({
+          Bucket: attachment.bucketName,
+          Key: attachment.filePath,
+        }),
+      );
+      const bytes = await result.Body?.transformToByteArray();
+      return bytes ?? null;
+    } catch {
+      return null;
+    }
+  };
+}
+
+/**
+ * GraphQL multipart upload (graphql-multipart-request-spec) of a single file
+ * via Twenty's `uploadFile` mutation. Returns the file id for FileItemInput.
+ */
+async function uploadBinary(options: {
+  baseUrl: string;
+  authToken: string;
+  filename: string;
+  contentType: string;
+  binary: Uint8Array;
+  fetchImpl: typeof fetch;
+  retries: number;
+}): Promise<string> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= options.retries; attempt += 1) {
+    try {
+      const form = new FormData();
+      form.append(
+        "operations",
+        JSON.stringify({
+          query: `mutation MigrationUpload($file: Upload!) {
+            uploadFile(file: $file, fileFolder: Attachment) { path token }
+          }`,
+          variables: { file: null },
+        }),
+      );
+      form.append("map", JSON.stringify({ "0": ["variables.file"] }));
+      form.append(
+        "0",
+        new Blob([options.binary as BlobPart], { type: options.contentType }),
+        options.filename,
+      );
+      const response = await options.fetchImpl(`${options.baseUrl}/graphql`, {
+        method: "POST",
+        headers: {
+          authorization: `Bearer ${options.authToken}`,
+          "apollo-require-preflight": "true",
+        },
+        body: form,
+      });
+      const body = (await response.json()) as {
+        data?: { uploadFile?: { path?: string; token?: string } };
+        errors?: Array<{ message: string }>;
+      };
+      if (body.errors?.length) {
+        throw new TwentyGraphqlError(
+          `uploadFile failed: ${JSON.stringify(body.errors)}`,
+          {
+            errors: body.errors,
+          },
+        );
+      }
+      const path = body.data?.uploadFile?.path;
+      if (!path) throw new Error("uploadFile returned no path");
+      // Twenty file paths embed the file id as the first path segment under
+      // the folder: e.g. "attachment/<uuid>/original/<name>". Extract the uuid.
+      const match = path.match(
+        /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i,
+      );
+      if (!match)
+        throw new Error(`could not extract file id from upload path: ${path}`);
+      return match[1];
+    } catch (error) {
+      lastError = error;
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}