strapi-content-sync-pro 1.0.0

package/server/src/utils/applier.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const { generateSignature } = require('./hmac');
+const { markAsRemoteUpdate } = require('./sync-guard');
+
+/**
+ * Apply a record received from a remote instance to the local database.
+ */
+async function applyLocal(strapi, uid, record, fields) {
+  const data = filterFields(record, fields);
+  const syncId = record.syncId;
+
+  // Mark so the afterCreate/afterUpdate hook skips re-pushing
+  markAsRemoteUpdate(`${uid}:${syncId}`);
+
+  const existing = await strapi.documents(uid).findMany({
+    filters: { syncId },
+    limit: 1,
+  });
+
+  if (existing && existing.length > 0) {
+    return strapi.documents(uid).update({
+      documentId: existing[0].documentId,
+      data,
+    });
+  }
+
+  data.syncId = syncId;
+  return strapi.documents(uid).create({ data });
+}
+
+/**
+ * Push a local record to the remote instance via the plugin's /receive endpoint.
+ */
+async function applyRemote(remoteConfig, uid, record, fields) {
+  const { baseUrl, apiToken, sharedSecret } = remoteConfig;
+  const url = new URL('/strapi-content-sync-pro/receive', baseUrl);
+
+  const body = {
+    uid,
+    data: filterFields(record, fields),
+    syncId: record.syncId,
+  };
+
+  const timestamp = Date.now().toString();
+  const signature = generateSignature(body, sharedSecret, timestamp);
+
+  const response = await fetch(url.toString(), {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiToken}`,
+      'Content-Type': 'application/json',
+      'x-sync-signature': signature,
+      'x-sync-timestamp': timestamp,
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Remote apply failed for ${uid}: ${response.status} – ${text}`);
+  }
+
+  return response.json();
+}
+
+/**
+ * Return only the requested fields from a record, stripping Strapi internals.
+ */
+function filterFields(record, fields) {
+  if (!fields || fields.length === 0) {
+    const {
+      id, documentId, createdAt, updatedAt, publishedAt,
+      createdBy, updatedBy, locale, localizations,
+      ...data
+    } = record;
+    return data;
+  }
+
+  const data = {};
+  for (const field of fields) {
+    if (record[field] !== undefined) {
+      data[field] = record[field];
+    }
+  }
+  return data;
+}
+
+module.exports = { applyLocal, applyRemote, filterFields };

package/server/src/utils/comparator.js

@@ -0,0 +1,83 @@
+'use strict';
+
+/**
+ * Compare local and remote record sets and return the required sync actions.
+ *
+ * @param {Array}  localRecords
+ * @param {Array}  remoteRecords
+ * @param {Object} options
+ * @param {string} options.direction        – "push" | "pull" | "both"
+ * @param {string} options.conflictStrategy – "latest" | "local_wins" | "remote_wins"
+ * @returns {{ toPush, toPull, toCreateRemote, toCreateLocal }}
+ */
+function compareRecords(localRecords, remoteRecords, options = {}) {
+  const { direction = 'both', conflictStrategy = 'latest' } = options;
+
+  const result = {
+    toPush: [],
+    toPull: [],
+    toCreateRemote: [],
+    toCreateLocal: [],
+  };
+
+  const localBySyncId = new Map();
+  const remoteBySyncId = new Map();
+
+  for (const r of localRecords) {
+    if (r.syncId) localBySyncId.set(r.syncId, r);
+  }
+  for (const r of remoteRecords) {
+    if (r.syncId) remoteBySyncId.set(r.syncId, r);
+  }
+
+  // Records that exist on both sides
+  for (const [syncId, localRecord] of localBySyncId) {
+    const remoteRecord = remoteBySyncId.get(syncId);
+
+    if (remoteRecord) {
+      const winner = resolveConflict(localRecord, remoteRecord, conflictStrategy);
+
+      if (winner === 'local' && (direction === 'push' || direction === 'both')) {
+        result.toPush.push({ local: localRecord, remote: remoteRecord });
+      } else if (winner === 'remote' && (direction === 'pull' || direction === 'both')) {
+        result.toPull.push({ local: localRecord, remote: remoteRecord });
+      }
+    } else if (direction === 'push' || direction === 'both') {
+      result.toCreateRemote.push(localRecord);
+    }
+  }
+
+  // Records that only exist on the remote
+  for (const [syncId] of remoteBySyncId) {
+    if (!localBySyncId.has(syncId)) {
+      if (direction === 'pull' || direction === 'both') {
+        result.toCreateLocal.push(remoteBySyncId.get(syncId));
+      }
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Determine the winner of a conflict.
+ * @returns {'local' | 'remote' | 'equal'}
+ */
+function resolveConflict(localRecord, remoteRecord, strategy) {
+  const localTime = new Date(localRecord.updatedAt).getTime();
+  const remoteTime = new Date(remoteRecord.updatedAt).getTime();
+
+  if (localTime === remoteTime) return 'equal';
+
+  switch (strategy) {
+    case 'local_wins':
+      return 'local';
+    case 'remote_wins':
+      return 'remote';
+    case 'latest':
+    default:
+      return localTime > remoteTime ? 'local' : 'remote';
+  }
+}
+
+module.exports = { compareRecords, resolveConflict };

package/server/src/utils/fetcher.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const { paginate, DEFAULT_PAGE_SIZE, normalizePageSize } = require('./pagination');
+
+/**
+ * Fetch ONE page of local records from the Strapi document service.
+ * Returns { records, hasMore, total }.
+ */
+async function fetchLocalPage(strapi, uid, { fields, lastSyncAt, page = 1, pageSize = DEFAULT_PAGE_SIZE } = {}) {
+  const size = normalizePageSize(pageSize);
+  const params = {
+    start: (page - 1) * size,
+    limit: size,
+    sort: 'updatedAt:asc',
+  };
+
+  if (lastSyncAt) {
+    params.filters = { updatedAt: { $gt: lastSyncAt } };
+  }
+
+  if (fields && fields.length > 0) {
+    params.fields = [...new Set([...fields, 'syncId', 'updatedAt'])];
+  }
+
+  const records = (await strapi.documents(uid).findMany(params)) || [];
+  return { records, hasMore: records.length === size };
+}
+
+/**
+ * Fetch ONE page of remote records via the standard Strapi REST API.
+ * Returns { records, hasMore, total, pageCount }.
+ */
+async function fetchRemotePage(remoteConfig, uid, { fields, lastSyncAt, page = 1, pageSize = DEFAULT_PAGE_SIZE } = {}) {
+  const size = normalizePageSize(pageSize);
+  const { baseUrl, apiToken } = remoteConfig;
+  const pluralName = uidToPluralEndpoint(uid);
+  const url = new URL(`/api/${pluralName}`, baseUrl);
+
+  if (fields && fields.length > 0) {
+    const allFields = [...new Set([...fields, 'syncId', 'updatedAt'])];
+    allFields.forEach((f, i) => {
+      url.searchParams.set(`fields[${i}]`, f);
+    });
+  }
+
+  if (lastSyncAt) {
+    url.searchParams.set('filters[updatedAt][$gt]', lastSyncAt);
+  }
+
+  url.searchParams.set('pagination[page]', String(page));
+  url.searchParams.set('pagination[pageSize]', String(size));
+  url.searchParams.set('sort', 'updatedAt:asc');
+
+  const response = await fetch(url.toString(), {
+    method: 'GET',
+    headers: {
+      Authorization: `Bearer ${apiToken}`,
+      'Content-Type': 'application/json',
+    },
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Remote fetch failed for ${uid}: ${response.status} – ${text}`);
+  }
+
+  const json = await response.json();
+  const records = json.data || [];
+  const meta = json.meta?.pagination;
+  const pageCount = meta?.pageCount;
+  const total = meta?.total;
+  const hasMore = pageCount ? page < pageCount : records.length === size;
+
+  return { records, hasMore, total, pageCount };
+}
+
+/**
+ * Async iterator over all local pages.
+ */
+function iterateLocalPages(strapi, uid, options = {}) {
+  return paginate(
+    (page, pageSize) => fetchLocalPage(strapi, uid, { ...options, page, pageSize }),
+    { pageSize: options.pageSize }
+  );
+}
+
+/**
+ * Async iterator over all remote pages.
+ */
+function iterateRemotePages(remoteConfig, uid, options = {}) {
+  return paginate(
+    (page, pageSize) => fetchRemotePage(remoteConfig, uid, { ...options, page, pageSize }),
+    { pageSize: options.pageSize }
+  );
+}
+
+/**
+ * Back-compat: fetch ALL local records (aggregates pages). Prefer the
+ * iterator variant for large datasets.
+ */
+async function fetchLocalRecords(strapi, uid, options = {}) {
+  const out = [];
+  for await (const { records } of iterateLocalPages(strapi, uid, options)) {
+    out.push(...records);
+  }
+  return out;
+}
+
+/**
+ * Back-compat: fetch ALL remote records (aggregates pages). Prefer the
+ * iterator variant for large datasets.
+ */
+async function fetchRemoteRecords(remoteConfig, uid, options = {}) {
+  const out = [];
+  for await (const { records } of iterateRemotePages(remoteConfig, uid, options)) {
+    out.push(...records);
+  }
+  return out;
+}
+
+/**
+ * Convert a content-type UID to its plural REST endpoint name.
+ * e.g. "api::product.product" → "products"
+ */
+function uidToPluralEndpoint(uid) {
+  const parts = uid.split('.');
+  const modelName = parts[parts.length - 1];
+  if (modelName.endsWith('s')) return modelName;
+  if (modelName.endsWith('y')) return modelName.slice(0, -1) + 'ies';
+  return modelName + 's';
+}
+
+module.exports = {
+  fetchLocalRecords,
+  fetchRemoteRecords,
+  fetchLocalPage,
+  fetchRemotePage,
+  iterateLocalPages,
+  iterateRemotePages,
+  uidToPluralEndpoint,
+};
+

package/server/src/utils/hmac.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const crypto = require('crypto');
+
+/**
+ * Generate an HMAC-SHA256 signature for a payload.
+ */
+function generateSignature(payload, secret, timestamp) {
+  const message = `${timestamp}.${typeof payload === 'string' ? payload : JSON.stringify(payload)}`;
+  return crypto.createHmac('sha256', secret).update(message).digest('hex');
+}
+
+/**
+ * Verify an HMAC-SHA256 signature.
+ * Rejects if timestamp is older than toleranceMs (default 5 minutes).
+ */
+function verifySignature(payload, secret, signature, timestamp, toleranceMs = 300000) {
+  const now = Date.now();
+  const ts = parseInt(timestamp, 10);
+
+  if (isNaN(ts) || Math.abs(now - ts) > toleranceMs) {
+    return false;
+  }
+
+  const expected = generateSignature(payload, secret, timestamp);
+
+  try {
+    return crypto.timingSafeEqual(
+      Buffer.from(expected, 'hex'),
+      Buffer.from(signature, 'hex')
+    );
+  } catch {
+    return false;
+  }
+}
+
+module.exports = { generateSignature, verifySignature };

package/server/src/utils/pagination.js

@@ -0,0 +1,51 @@
+'use strict';
+
+/**
+ * Generic pagination helpers used by both the local Document Service path
+ * and the remote REST API path. Keeping these centralized means the sync
+ * engine can process arbitrarily large content types in bounded memory.
+ */
+
+const DEFAULT_PAGE_SIZE = 100;
+const MIN_PAGE_SIZE = 1;
+const MAX_PAGE_SIZE = 5000;
+
+function normalizePageSize(size) {
+  const n = Number(size);
+  if (!Number.isFinite(n)) return DEFAULT_PAGE_SIZE;
+  return Math.max(MIN_PAGE_SIZE, Math.min(MAX_PAGE_SIZE, Math.floor(n)));
+}
+
+/**
+ * Walk a paginated source page-by-page.
+ *
+ *   fetchPage(page, pageSize) -> { records, hasMore, total? }
+ *
+ * The helper yields each page's records so callers can process/apply them
+ * without ever holding the full result set in memory.
+ */
+async function* paginate(fetchPage, { pageSize = DEFAULT_PAGE_SIZE, maxPages } = {}) {
+  const size = normalizePageSize(pageSize);
+  let page = 1;
+  while (true) {
+    const result = await fetchPage(page, size);
+    const records = Array.isArray(result) ? result : (result?.records || []);
+    const hasMore = Array.isArray(result)
+      ? records.length === size
+      : !!result?.hasMore;
+
+    yield { page, pageSize: size, records, total: result?.total };
+
+    if (!hasMore || records.length === 0) break;
+    if (maxPages && page >= maxPages) break;
+    page += 1;
+  }
+}
+
+module.exports = {
+  DEFAULT_PAGE_SIZE,
+  MIN_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+  normalizePageSize,
+  paginate,
+};

package/server/src/utils/sync-guard.js

@@ -0,0 +1,29 @@
+'use strict';
+
+/**
+ * In-memory guard to prevent infinite sync loops.
+ *
+ * Before applying a record received from a remote instance we call
+ * markAsRemoteUpdate(key).  The afterCreate / afterUpdate lifecycle
+ * hook then calls isRemoteUpdate(key) — if it returns true the hook
+ * skips pushing the record back to the remote.
+ */
+const _active = new Set();
+
+module.exports = {
+  markAsRemoteUpdate(key) {
+    _active.add(key);
+  },
+
+  isRemoteUpdate(key) {
+    if (_active.has(key)) {
+      _active.delete(key);
+      return true;
+    }
+    return false;
+  },
+
+  clear() {
+    _active.clear();
+  },
+};

package/server/src/utils/sync-id.js

@@ -0,0 +1,16 @@
+'use strict';
+
+const crypto = require('crypto');
+
+function generateSyncId() {
+  return crypto.randomUUID();
+}
+
+function ensureSyncId(data) {
+  if (!data.syncId) {
+    data.syncId = generateSyncId();
+  }
+  return data;
+}
+
+module.exports = { generateSyncId, ensureSyncId };