strapi-content-sync-pro 1.0.0

package/server/src/services/sync-profiles.js

@@ -0,0 +1,380 @@
+'use strict';
+
+const STORE_KEY = 'sync-profiles';
+
+/**
+ * Sync Profiles Service
+ * 
+ * A profile defines WHAT and HOW to sync for a content type:
+ * - direction: push, pull, both
+ * - conflictStrategy: latest, local_wins, remote_wins
+ * - fieldPolicies: per-field direction overrides (advanced mode)
+ * 
+ * Execution settings (WHEN to sync) are managed separately in sync-execution service.
+ * 
+ * Profile structure:
+ * {
+ *   id: string,
+ *   name: string,
+ *   contentType: string (uid),
+ *   direction: 'push' | 'pull' | 'both',
+ *   conflictStrategy: 'latest' | 'local_wins' | 'remote_wins',
+ *   isActive: boolean,
+ *   isSimple: boolean (false = advanced mode with field policies),
+ *   fieldPolicies: [{ field, direction }],
+ *   createdAt: ISO string,
+ *   updatedAt: ISO string
+ * }
+ */
+module.exports = ({ strapi }) => {
+  function getStore() {
+    return strapi.store({ type: 'plugin', name: 'strapi-content-sync-pro' });
+  }
+
+  function generateId() {
+    return `profile_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
+  }
+
+  const VALID_DIRECTIONS = ['push', 'pull', 'both', 'none'];
+  const VALID_CONFLICT_STRATEGIES = ['latest', 'local_wins', 'remote_wins'];
+
+  return {
+    /**
+     * Get all sync profiles
+     */
+    async getProfiles() {
+      const store = getStore();
+      const data = await store.get({ key: STORE_KEY });
+      return data || [];
+    },
+
+    /**
+     * Get a single profile by ID
+     */
+    async getProfile(id) {
+      const profiles = await this.getProfiles();
+      return profiles.find((p) => p.id === id) || null;
+    },
+
+    /**
+     * Get active profile for a content type
+     */
+    async getActiveProfileForContentType(contentTypeUid) {
+      const profiles = await this.getProfiles();
+      return profiles.find((p) => p.contentType === contentTypeUid && p.isActive) || null;
+    },
+
+    /**
+     * Get all profiles for a content type
+     */
+    async getProfilesForContentType(contentTypeUid) {
+      const profiles = await this.getProfiles();
+      return profiles.filter((p) => p.contentType === contentTypeUid);
+    },
+
+    /**
+     * Auto-generate default profiles for a content type
+     * Creates: Full Push, Full Pull, Bidirectional (Merge)
+     */
+    async autoGenerateProfiles(contentTypeUid) {
+      const existingProfiles = await this.getProfilesForContentType(contentTypeUid);
+      if (existingProfiles.length > 0) {
+        return existingProfiles; // Don't regenerate if profiles exist
+      }
+
+      const contentType = strapi.contentTypes[contentTypeUid];
+      const displayName = contentType?.info?.displayName || contentTypeUid;
+
+      const defaultProfiles = [
+        {
+          name: `${displayName} - Full Push`,
+          contentType: contentTypeUid,
+          direction: 'push',
+          conflictStrategy: 'local_wins',
+          isActive: false,
+          isSimple: true,
+          fieldPolicies: [],
+        },
+        {
+          name: `${displayName} - Full Pull`,
+          contentType: contentTypeUid,
+          direction: 'pull',
+          conflictStrategy: 'remote_wins',
+          isActive: false,
+          isSimple: true,
+          fieldPolicies: [],
+        },
+        {
+          name: `${displayName} - Bidirectional`,
+          contentType: contentTypeUid,
+          direction: 'both',
+          conflictStrategy: 'latest',
+          isActive: true, // Default active profile
+          isSimple: true,
+          fieldPolicies: [],
+        },
+      ];
+
+      const created = [];
+      for (const profileData of defaultProfiles) {
+        const profile = await this.createProfile(profileData);
+        created.push(profile);
+      }
+
+      return created;
+    },
+
+    /**
+     * Create a new sync profile
+     */
+    async createProfile(profileData) {
+      const store = getStore();
+      const profiles = await this.getProfiles();
+
+      if (!profileData.name) {
+        throw new Error('Profile name is required');
+      }
+      if (!profileData.contentType) {
+        throw new Error('Content type is required');
+      }
+
+      // Validate direction
+      if (profileData.direction && !['push', 'pull', 'both'].includes(profileData.direction)) {
+        throw new Error(`Invalid direction "${profileData.direction}"`);
+      }
+
+      // Validate conflict strategy
+      if (profileData.conflictStrategy && !VALID_CONFLICT_STRATEGIES.includes(profileData.conflictStrategy)) {
+        throw new Error(`Invalid conflict strategy "${profileData.conflictStrategy}"`);
+      }
+
+      // Validate field policies
+      if (profileData.fieldPolicies) {
+        for (const fp of profileData.fieldPolicies) {
+          if (!fp.field) {
+            throw new Error('Each field policy must have a field name');
+          }
+          if (fp.direction && !VALID_DIRECTIONS.includes(fp.direction)) {
+            throw new Error(`Invalid direction "${fp.direction}" for field "${fp.field}"`);
+          }
+        }
+      }
+
+      const newProfile = {
+        id: generateId(),
+        name: profileData.name,
+        contentType: profileData.contentType,
+        direction: profileData.direction || 'both',
+        conflictStrategy: profileData.conflictStrategy || 'latest',
+        isActive: profileData.isActive || false,
+        isSimple: profileData.isSimple !== false, // Default to simple mode
+        fieldPolicies: (profileData.fieldPolicies || []).map((fp) => ({
+          field: fp.field,
+          direction: fp.direction || 'both',
+        })),
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      };
+
+      // If this profile is set as active, deactivate others for same content type
+      if (newProfile.isActive) {
+        profiles.forEach((p) => {
+          if (p.contentType === newProfile.contentType) {
+            p.isActive = false;
+          }
+        });
+      }
+
+      profiles.push(newProfile);
+      await store.set({ key: STORE_KEY, value: profiles });
+
+      return newProfile;
+    },
+
+    /**
+     * Update an existing sync profile
+     */
+    async updateProfile(id, updates) {
+      const store = getStore();
+      const profiles = await this.getProfiles();
+      const index = profiles.findIndex((p) => p.id === id);
+
+      if (index === -1) {
+        throw new Error(`Profile with id "${id}" not found`);
+      }
+
+      // Validate direction
+      if (updates.direction && !['push', 'pull', 'both'].includes(updates.direction)) {
+        throw new Error(`Invalid direction "${updates.direction}"`);
+      }
+
+      // Validate conflict strategy
+      if (updates.conflictStrategy && !VALID_CONFLICT_STRATEGIES.includes(updates.conflictStrategy)) {
+        throw new Error(`Invalid conflict strategy "${updates.conflictStrategy}"`);
+      }
+
+      // Validate field policies if provided
+      if (updates.fieldPolicies) {
+        for (const fp of updates.fieldPolicies) {
+          if (!fp.field) {
+            throw new Error('Each field policy must have a field name');
+          }
+          if (fp.direction && !VALID_DIRECTIONS.includes(fp.direction)) {
+            throw new Error(`Invalid direction "${fp.direction}" for field "${fp.field}"`);
+          }
+        }
+      }
+
+      // If setting this profile as active, deactivate others for same content type
+      if (updates.isActive) {
+        const contentType = updates.contentType || profiles[index].contentType;
+        profiles.forEach((p) => {
+          if (p.contentType === contentType && p.id !== id) {
+            p.isActive = false;
+          }
+        });
+      }
+
+      const updatedProfile = {
+        ...profiles[index],
+        ...updates,
+        id: profiles[index].id, // prevent id change
+        createdAt: profiles[index].createdAt, // preserve creation date
+        updatedAt: new Date().toISOString(),
+      };
+
+      if (updates.fieldPolicies) {
+        updatedProfile.fieldPolicies = updates.fieldPolicies.map((fp) => ({
+          field: fp.field,
+          direction: fp.direction || 'both',
+        }));
+      }
+
+      profiles[index] = updatedProfile;
+      await store.set({ key: STORE_KEY, value: profiles });
+
+      return updatedProfile;
+    },
+
+    /**
+     * Delete a sync profile
+     */
+    async deleteProfile(id) {
+      const store = getStore();
+      const profiles = await this.getProfiles();
+      const filtered = profiles.filter((p) => p.id !== id);
+
+      if (filtered.length === profiles.length) {
+        throw new Error(`Profile with id "${id}" not found`);
+      }
+
+      await store.set({ key: STORE_KEY, value: filtered });
+      return { success: true };
+    },
+
+    /**
+     * Create a simple preset profile
+     */
+    async createSimpleProfile(contentTypeUid, preset) {
+      const contentType = strapi.contentTypes[contentTypeUid];
+      const displayName = contentType?.info?.displayName || contentTypeUid;
+
+      const presets = {
+        full_push: {
+          name: `${displayName} - Full Push`,
+          direction: 'push',
+          conflictStrategy: 'local_wins',
+        },
+        full_pull: {
+          name: `${displayName} - Full Pull`,
+          direction: 'pull',
+          conflictStrategy: 'remote_wins',
+        },
+        bidirectional: {
+          name: `${displayName} - Bidirectional`,
+          direction: 'both',
+          conflictStrategy: 'latest',
+        },
+      };
+
+      const presetConfig = presets[preset];
+      if (!presetConfig) {
+        throw new Error(`Invalid preset "${preset}". Valid presets: ${Object.keys(presets).join(', ')}`);
+      }
+
+      return this.createProfile({
+        ...presetConfig,
+        contentType: contentTypeUid,
+        isSimple: true,
+        isActive: false,
+        fieldPolicies: [],
+      });
+    },
+
+    /**
+     * Get field policies for a content type (from active profile)
+     * Returns a map: { fieldName: 'push' | 'pull' | 'both' | 'none' }
+     */
+    async getFieldPoliciesForContentType(contentTypeUid) {
+      const activeProfile = await this.getActiveProfileForContentType(contentTypeUid);
+      if (!activeProfile || activeProfile.isSimple) {
+        return null; // No field policies for simple profiles
+      }
+
+      const policyMap = {};
+      for (const fp of activeProfile.fieldPolicies) {
+        policyMap[fp.field] = fp.direction;
+      }
+      return policyMap;
+    },
+
+    /**
+     * Get sync configuration for a content type (from active profile)
+     */
+    async getSyncConfigForContentType(contentTypeUid) {
+      const activeProfile = await this.getActiveProfileForContentType(contentTypeUid);
+      if (!activeProfile) {
+        return null;
+      }
+
+      return {
+        direction: activeProfile.direction,
+        conflictStrategy: activeProfile.conflictStrategy,
+        fieldPolicies: activeProfile.isSimple ? null : await this.getFieldPoliciesForContentType(contentTypeUid),
+      };
+    },
+
+    /**
+     * Filter fields based on policies for a given direction
+     */
+    filterFieldsByPolicy(record, fieldPolicies, syncDirection) {
+      if (!fieldPolicies) {
+        return record; // No policies, return all fields
+      }
+
+      const filtered = {};
+      for (const [field, value] of Object.entries(record)) {
+        const policy = fieldPolicies[field];
+
+        // If no policy defined for field, include it (default to 'both')
+        if (!policy || policy === 'both') {
+          filtered[field] = value;
+          continue;
+        }
+
+        // Include field if policy matches sync direction
+        if (policy === syncDirection) {
+          filtered[field] = value;
+          continue;
+        }
+
+        // Always include id and metadata fields
+        if (['id', 'documentId', 'syncId', 'createdAt', 'updatedAt'].includes(field)) {
+          filtered[field] = value;
+        }
+      }
+
+      return filtered;
+    },
+  };
+};

package/server/src/services/sync.js

@@ -0,0 +1,248 @@
+'use strict';
+
+const { fetchLocalRecords, fetchRemoteRecords } = require('../utils/fetcher');
+const { compareRecords } = require('../utils/comparator');
+const { applyLocal, applyRemote } = require('../utils/applier');
+
+const LAST_SYNC_STORE_KEY = 'last-sync-timestamps';
+
+module.exports = ({ strapi }) => {
+  function getStore() {
+    return strapi.store({ type: 'plugin', name: 'strapi-content-sync-pro' });
+  }
+
+  function plugin() {
+    return strapi.plugin('strapi-content-sync-pro');
+  }
+
+  async function getLastSyncTimestamps() {
+    const store = getStore();
+    return (await store.get({ key: LAST_SYNC_STORE_KEY })) || {};
+  }
+
+  async function setLastSyncTimestamp(uid, timestamp) {
+    const store = getStore();
+    const timestamps = await getLastSyncTimestamps();
+    timestamps[uid] = timestamp;
+    await store.set({ key: LAST_SYNC_STORE_KEY, value: timestamps });
+  }
+
+  return {
+    /**
+     * Step 6 + 7 + 10 — Execute a manual / incremental sync for every
+     * enabled content type.
+     * 
+     * Now supports field-level policies from Sync Profiles.
+     */
+    async syncNow() {
+      const logService = plugin().service('syncLog');
+      const configService = plugin().service('config');
+      const syncConfigService = plugin().service('syncConfig');
+      const syncProfilesService = plugin().service('syncProfiles');
+      const executionService = plugin().service('syncExecution');
+
+      const remoteConfig = await configService.getConfig({ safe: false });
+      if (!remoteConfig || !remoteConfig.baseUrl) {
+        throw new Error('Remote server not configured');
+      }
+
+      const syncConfig = await syncConfigService.getSyncConfig();
+      const enabledTypes = (syncConfig.contentTypes || []).filter((ct) => ct.enabled);
+
+      if (enabledTypes.length === 0) {
+        throw new Error('No content types configured for sync');
+      }
+
+      // Pagination — remote + local fetches are chunked to keep memory bounded
+      // for large datasets. Page size is a global setting tunable in the Sync tab.
+      const globalExec = (await executionService.getGlobalSettings?.()) || {};
+      const pageSize = Number(globalExec.syncPageSize) || 100;
+
+      const timestamps = await getLastSyncTimestamps();
+      const conflictStrategy = syncConfig.conflictStrategy || 'latest';
+      const results = [];
+
+      for (const ctConfig of enabledTypes) {
+        const { uid, direction, fields } = ctConfig;
+        const lastSyncAt = timestamps[uid] || null;
+        const syncStartTime = new Date().toISOString();
+
+        // Get field-level policies from active profile (if any)
+        const fieldPolicies = await syncProfilesService.getFieldPoliciesForContentType(uid);
+
+        try {
+          // Both sides are fetched in pages of `pageSize` records under the
+          // hood (see utils/fetcher.js). We aggregate per content-type because
+          // the comparator needs the full set to diff by syncId, but each
+          // network/DB call still only returns a bounded chunk.
+          const localRecords = await fetchLocalRecords(strapi, uid, { fields, lastSyncAt, pageSize });
+          const remoteRecords = await fetchRemoteRecords(remoteConfig, uid, { fields, lastSyncAt, pageSize });
+
+          const diff = compareRecords(localRecords, remoteRecords, {
+            direction,
+            conflictStrategy,
+          });
+
+          let pushed = 0;
+          let pulled = 0;
+          let errors = 0;
+
+          // Apply field policies to records before pushing/pulling
+          for (const { local } of diff.toPush) {
+            try {
+              const filteredRecord = syncProfilesService.filterFieldsByPolicy(local, fieldPolicies, 'push');
+              await applyRemote(remoteConfig, uid, filteredRecord, fields);
+              pushed++;
+            } catch (err) {
+              errors++;
+              await logService.log({ action: 'push', contentType: uid, syncId: local.syncId, direction: 'push', status: 'error', message: err.message });
+            }
+          }
+
+          for (const { remote } of diff.toPull) {
+            try {
+              const filteredRecord = syncProfilesService.filterFieldsByPolicy(remote, fieldPolicies, 'pull');
+              await applyLocal(strapi, uid, filteredRecord, fields);
+              pulled++;
+            } catch (err) {
+              errors++;
+              await logService.log({ action: 'pull', contentType: uid, syncId: remote.syncId, direction: 'pull', status: 'error', message: err.message });
+            }
+          }
+
+          for (const record of diff.toCreateRemote) {
+            try {
+              const filteredRecord = syncProfilesService.filterFieldsByPolicy(record, fieldPolicies, 'push');
+              await applyRemote(remoteConfig, uid, filteredRecord, fields);
+              pushed++;
+            } catch (err) {
+              errors++;
+              await logService.log({ action: 'create_remote', contentType: uid, syncId: record.syncId, direction: 'push', status: 'error', message: err.message });
+            }
+          }
+
+          for (const record of diff.toCreateLocal) {
+            try {
+              const filteredRecord = syncProfilesService.filterFieldsByPolicy(record, fieldPolicies, 'pull');
+              await applyLocal(strapi, uid, filteredRecord, fields);
+              pulled++;
+            } catch (err) {
+              errors++;
+              await logService.log({ action: 'create_local', contentType: uid, syncId: record.syncId, direction: 'pull', status: 'error', message: err.message });
+            }
+          }
+
+          await setLastSyncTimestamp(uid, syncStartTime);
+
+          const summary = { uid, pushed, pulled, errors, hasFieldPolicies: !!fieldPolicies };
+          results.push(summary);
+
+          await logService.log({
+            action: 'sync_complete',
+            contentType: uid,
+            direction,
+            status: errors > 0 ? 'partial' : 'success',
+            message: `Pushed: ${pushed}, Pulled: ${pulled}, Errors: ${errors}${fieldPolicies ? ' (with field policies)' : ''}`,
+            details: summary,
+          });
+        } catch (err) {
+          results.push({ uid, error: err.message });
+          await logService.log({
+            action: 'sync_error',
+            contentType: uid,
+            direction,
+            status: 'error',
+            message: err.message,
+          });
+        }
+      }
+
+      return { syncedAt: new Date().toISOString(), results };
+    },
+
+    /**
+     * Step 8 — Push a single record to the remote (called by lifecycle hooks).
+     * Now supports field-level policies.
+     */
+    async pushRecord(uid, record) {
+      const configService = plugin().service('config');
+      const logService = plugin().service('syncLog');
+      const syncProfilesService = plugin().service('syncProfiles');
+
+      const remoteConfig = await configService.getConfig({ safe: false });
+      if (!remoteConfig || !remoteConfig.baseUrl) return;
+
+      const syncConfigService = plugin().service('syncConfig');
+      const syncConfig = await syncConfigService.getSyncConfig();
+      const ctConfig = (syncConfig.contentTypes || []).find(
+        (ct) => ct.uid === uid && ct.enabled
+      );
+
+      if (!ctConfig) return;
+      if (ctConfig.direction === 'pull') return;
+
+      // Get field-level policies from active profile (if any)
+      const fieldPolicies = await syncProfilesService.getFieldPoliciesForContentType(uid);
+      const filteredRecord = syncProfilesService.filterFieldsByPolicy(record, fieldPolicies, 'push');
+
+      try {
+        await applyRemote(remoteConfig, uid, filteredRecord, ctConfig.fields);
+        await logService.log({
+          action: 'event_push',
+          contentType: uid,
+          syncId: record.syncId,
+          direction: 'push',
+          status: 'success',
+          message: `Record ${record.syncId} pushed to remote${fieldPolicies ? ' (with field policies)' : ''}`,
+        });
+      } catch (err) {
+        await logService.log({
+          action: 'event_push',
+          contentType: uid,
+          syncId: record.syncId,
+          direction: 'push',
+          status: 'error',
+          message: err.message,
+        });
+      }
+    },
+
+    /**
+     * Step 9 — Receive a record pushed from a remote instance.
+     * Now supports field-level policies.
+     */
+    async receiveRecord(uid, data, syncId) {
+      const logService = plugin().service('syncLog');
+      const syncProfilesService = plugin().service('syncProfiles');
+
+      // Get field-level policies from active profile (if any)
+      const fieldPolicies = await syncProfilesService.getFieldPoliciesForContentType(uid);
+      const filteredData = syncProfilesService.filterFieldsByPolicy(data, fieldPolicies, 'pull');
+
+      try {
+        await applyLocal(strapi, uid, { ...filteredData, syncId }, []);
+
+        await logService.log({
+          action: 'receive',
+          contentType: uid,
+          syncId,
+          direction: 'pull',
+          status: 'success',
+          message: `Record ${syncId} received from remote${fieldPolicies ? ' (with field policies)' : ''}`,
+        });
+
+        return { success: true };
+      } catch (err) {
+        await logService.log({
+          action: 'receive',
+          contentType: uid,
+          syncId,
+          direction: 'pull',
+          status: 'error',
+          message: err.message,
+        });
+        throw err;
+      }
+    },
+  };
+};