psdev-task-manager 1.0.2 → 1.0.3

package/backend/elevated-modules.js

@@ -0,0 +1,22 @@
+const { items, collections } = require('@wix/data');
+const { auth } = require('@wix/essentials');
+
+// @wix/data does not support suppressAuth currently, so we need to elevate it
+const wixData = {
+  insert: auth.elevate(items.insert),
+  update: auth.elevate(items.update),
+  bulkInsert: auth.elevate(items.bulkInsert),
+  query: auth.elevate(items.query),
+  save: auth.elevate(items.save),
+  remove: auth.elevate(items.remove),
+  get: auth.elevate(items.get),
+  //TODO: add other methods here as needed
+};
+
+const wixCollections = {
+  getDataCollection: auth.elevate(collections.getDataCollection),
+  createDataCollection: auth.elevate(collections.createDataCollection),
+  updateDataCollection: auth.elevate(collections.updateDataCollection),
+};
+
+module.exports = { wixData, wixCollections };

package/backend/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./tasks'),
+  ...require('./elevated-modules'),
+};

package/backend/tasks/consts.js

@@ -0,0 +1,25 @@
+const TASK_STATUS = {
+  PENDING: 'pending',
+  IN_PROGRESS: 'in_progress',
+  SUCCESS: 'success',
+  FAILED: 'failed',
+  SKIPPED: 'skipped',
+};
+
+const TASK_TYPE = {
+  SCHEDULED: 'scheduled',
+  EVENT: 'event',
+};
+
+const TASKS_CONCURRENCY_LIMIT = 1;
+const TASK_MAX_TRIES = 3;
+
+const CRON_JOB_MAX_DURATION_SEC = 240; // 4 minutes is the velo limit for a cron job run @see https://wix.slack.com/archives/C02T75TDG2C/p1747750017241869
+
+module.exports = {
+  TASK_STATUS,
+  TASK_TYPE,
+  TASKS_CONCURRENCY_LIMIT,
+  TASK_MAX_TRIES,
+  CRON_JOB_MAX_DURATION_SEC,
+};

package/backend/tasks/fieldTypeDetector.js

@@ -0,0 +1,231 @@
+/**
+ * Utility functions for detecting field types based on values
+ */
+
+/**
+ * Checks if a string is a valid URL
+ * @param {string} str - The string to check
+ * @returns {boolean} - Whether the string is a valid URL
+ */
+function isValidUrl(str) {
+  if (typeof str !== 'string') return false;
+
+  try {
+    const url = new URL(str);
+    return ['http:', 'https:', 'ftp:', 'ftps:'].includes(url.protocol);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Checks if a string represents a date/datetime
+ * @param {string} str - The string to check
+ * @returns {boolean} - Whether the string represents a valid date
+ */
+function isDateString(str) {
+  if (typeof str !== 'string') return false;
+
+  // Common date patterns
+  const datePatterns = [
+    /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?$/, // ISO 8601: 2025-06-24T11:26:00.000Z
+    /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$/, // ISO with timezone: 2025-06-24T11:26:00+01:00
+    /^\d{4}-\d{2}-\d{2}$/, // Date only: 2025-06-24
+    /^\d{2}\/\d{2}\/\d{4}$/, // US format: 06/24/2025
+    /^\d{2}-\d{2}-\d{4}$/, // Alternative format: 24-06-2025
+  ];
+
+  if (!datePatterns.some(pattern => pattern.test(str))) {
+    return false;
+  }
+
+  // Validate that it's actually a valid date
+  const date = new Date(str);
+  return !isNaN(date.getTime());
+}
+
+/**
+ * Checks if a string represents an email address
+ * @param {string} str - The string to check
+ * @returns {boolean} - Whether the string is a valid email
+ */
+function isEmail(str) {
+  if (typeof str !== 'string') return false;
+
+  const emailPattern = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+  return emailPattern.test(str);
+}
+
+/**
+ * Checks if a string represents a phone number
+ * @param {string} str - The string to check
+ * @returns {boolean} - Whether the string looks like a phone number
+ */
+function isPhoneNumber(str) {
+  if (typeof str !== 'string') return false;
+
+  // Remove all non-alphanumeric characters for pattern matching
+  const cleanStr = str.replace(/[\s().-]/g, '');
+
+  // Basic phone number patterns (international and common formats)
+  const phonePatterns = [
+    /^\+\d{1,3}\d{6,14}$/, // International: +1234567890
+    /^\d{10}$/, // Simple 10 digits: 1234567890
+    /^\d{11}$/, // 11 digits with country code: 11234567890
+  ];
+
+  // Also check original string for formatted patterns
+  const formattedPatterns = [
+    /^\+\d{1,3}\s?\(\d{3}\)\s?\d{3}[\s-]?\d{4}$/, // +1 (555) 123-4567
+    /^\(\d{3}\)\s?\d{3}[\s-]?\d{4}$/, // (555) 123-4567
+    /^\d{3}[\s-]?\d{3}[\s-]?\d{4}$/, // 555-123-4567 or 555 123 4567
+    /^\+\d{1,3}[\s-]?\d{6,14}$/, // +1-1234567890 or +1 1234567890
+  ];
+
+  return (
+    phonePatterns.some(pattern => pattern.test(cleanStr)) ||
+    formattedPatterns.some(pattern => pattern.test(str))
+  );
+}
+
+/**
+ * Detects the appropriate Wix Data field type based on a value
+ * @param {*} value - The value to analyze
+ * @returns {string} - The Wix Data field type
+ */
+function detectFieldType(value) {
+  // Handle null/undefined
+  if (value === null || value === undefined) {
+    return 'TEXT'; // Default to TEXT for null values
+  }
+
+  // Handle primitive types first
+  if (typeof value === 'boolean') {
+    return 'BOOLEAN';
+  }
+
+  if (typeof value === 'number') {
+    return 'NUMBER';
+  }
+
+  if (value instanceof Date) {
+    return 'DATETIME';
+  }
+
+  // Handle arrays
+  if (Array.isArray(value)) {
+    return 'ARRAY';
+  }
+
+  // Handle objects (but not arrays or dates)
+  if (typeof value === 'object') {
+    return 'OBJECT';
+  }
+
+  // Handle strings with special patterns
+  if (typeof value === 'string') {
+    // Check for empty strings
+    if (value.trim() === '') {
+      return 'TEXT';
+    }
+
+    // Check for date strings first (most specific)
+    if (isDateString(value)) {
+      return 'DATETIME';
+    }
+
+    // Check for URLs
+    if (isValidUrl(value)) {
+      return 'URL';
+    }
+
+    // Check for email addresses
+    if (isEmail(value)) {
+      return 'EMAIL';
+    }
+
+    // Check for phone numbers
+    if (isPhoneNumber(value)) {
+      return 'PHONE';
+    }
+
+    // Check for very long text (might be better as rich text)
+    if (value.length > 500) {
+      return 'RICHTEXT';
+    }
+
+    // Default to TEXT for other strings
+    return 'TEXT';
+  }
+
+  // Fallback for unknown types
+  return 'TEXT';
+}
+
+/**
+ * Analyzes multiple values to determine the best field type
+ * Useful when you have an array of values and want to determine the most appropriate type
+ * @param {Array} values - Array of values to analyze
+ * @returns {string} - The most appropriate Wix Data field type
+ */
+function detectFieldTypeFromMultipleValues(values) {
+  if (!Array.isArray(values) || values.length === 0) {
+    return 'TEXT';
+  }
+
+  // Filter out null/undefined values for analysis
+  const validValues = values.filter(v => v !== null && v !== undefined);
+
+  if (validValues.length === 0) {
+    return 'TEXT';
+  }
+
+  // Get types for all valid values
+  const types = validValues.map(detectFieldType);
+  const uniqueTypes = [...new Set(types)];
+
+  // If all values have the same type, use that type
+  if (uniqueTypes.length === 1) {
+    return uniqueTypes[0];
+  }
+
+  // Handle mixed types - prioritize more specific types
+  const typePriority = {
+    BOOLEAN: 8,
+    NUMBER: 7,
+    DATETIME: 6,
+    URL: 5,
+    EMAIL: 4,
+    PHONE: 3,
+    ARRAY: 2,
+    OBJECT: 2,
+    RICHTEXT: 1,
+    TEXT: 0,
+  };
+
+  // Find the highest priority type that appears
+  const sortedTypes = uniqueTypes.sort((a, b) => (typePriority[b] || 0) - (typePriority[a] || 0));
+
+  // If we have mixed primitive types with TEXT, prefer the more specific type
+  // unless it's a very small percentage of the data
+  const primaryType = sortedTypes[0];
+  const primaryTypeCount = types.filter(t => t === primaryType).length;
+  const primaryTypePercentage = primaryTypeCount / types.length;
+
+  // If the primary type represents at least 70% of the data, use it
+  if (primaryTypePercentage >= 0.7) {
+    return primaryType;
+  }
+
+  // Otherwise, default to TEXT for mixed data
+  return 'TEXT';
+}
+
+module.exports = {
+  isValidUrl,
+  isDateString,
+  isEmail,
+  isPhoneNumber,
+  detectFieldType,
+  detectFieldTypeFromMultipleValues,
+};

package/backend/tasks/index.js

@@ -0,0 +1,10 @@
+module.exports = {
+  ...require('../tasksCollections'),
+  ...require('./parentChildTasks'),
+
+  ...require('./taskManager'),
+  ...require('./consts'),
+  ...require('./resultsSaver'),
+  ...require('./utils'),
+  ...require('./fieldTypeDetector'),
+};

package/backend/tasks/parentChildTasks/childScheduler.js

@@ -0,0 +1,66 @@
+const { insertNewTask, bulkInsertTasks } = require('../utils');
+
+const { createChildTaskPayload } = require('./utils');
+
+const scheduleChildren = ({
+  parentTaskId,
+  parentTaskData,
+  expectedChildTasks,
+  scheduledChildren,
+}) => {
+  const scheduleSingleChild = ({ taskName, taskData, parentTaskId }) => {
+    const childTask = createChildTaskPayload({
+      taskName,
+      taskData,
+      parentTaskId,
+    });
+    return insertNewTask(childTask);
+  };
+
+  const scheduleChildrenInBulk = ({
+    parentTaskId,
+    parentTaskData: { previousTaskOutput: _previousTaskOutput, ...restParentData },
+    childTasks,
+  }) => {
+    const tasksToInsert = childTasks.map(({ name, data }) =>
+      createChildTaskPayload({
+        taskName: name,
+        taskData: { ...restParentData, ...data },
+        parentTaskId,
+      })
+    );
+    //Host needs to keep in mind Velo limitations of bulkInsert 1000 items max and 500kb max item size
+    //Ref: https://dev.wix.com/docs/velo/apis/wix-data/bulk-insert
+    return bulkInsertTasks(tasksToInsert);
+  };
+
+  const getPreviousChildOutput = childIndex => {
+    const isFirstChild = childIndex === 0 || !scheduledChildren?.length;
+    if (isFirstChild) {
+      return undefined;
+    }
+    const lastScheduledChild = scheduledChildren[scheduledChildren.length - 1];
+    return lastScheduledChild?.data?.output;
+  };
+
+  return {
+    sequentially: childIndex =>
+      scheduleSingleChild({
+        taskName: expectedChildTasks[childIndex].name,
+        taskData: {
+          ...parentTaskData,
+          ...expectedChildTasks[childIndex].data,
+          previousTaskOutput: getPreviousChildOutput(childIndex),
+        },
+        parentTaskId,
+      }),
+    inBulk: () =>
+      scheduleChildrenInBulk({
+        parentTaskId,
+        parentTaskData,
+        childTasks: expectedChildTasks,
+      }),
+  };
+};
+
+module.exports = { scheduleChildren };

package/backend/tasks/parentChildTasks/handler.js

@@ -0,0 +1,77 @@
+const { getTaskConfig, getTaskScheduledChildren, getExpectedChildTasks } = require('../utils');
+
+const { scheduleChildren } = require('./childScheduler');
+const {
+  hasFailedChildAndMarkedParent,
+  hasUnfinishedChild,
+  areChildrenSucceededAndMarkedParent,
+} = require('./parentStatusManager');
+const { shouldScheduleNextChild: _shouldScheduleNextChild } = require('./utils');
+const scheduleChildTasksAndUpdateParent = async (task, tasksConfig) => {
+  const { _id: parentTaskId, name: parentTaskName, data: parentTaskData } = task;
+  const parentTaskConfig = getTaskConfig(parentTaskName, tasksConfig);
+  const expectedChildTasks = getExpectedChildTasks(task, parentTaskConfig);
+  const scheduledChildren = await getTaskScheduledChildren(parentTaskId);
+  const childrenScheduler = scheduleChildren({
+    parentTaskId,
+    parentTaskData,
+    expectedChildTasks,
+    scheduledChildren,
+  });
+  if (!scheduledChildren.length) {
+    console.log(
+      `No scheduled children found for task with id ${parentTaskId}, started scheduling children`
+    );
+    if (parentTaskConfig.scheduleChildrenSequentially) {
+      console.log('scheduling children sequentially');
+      return childrenScheduler.sequentially(0);
+    } else {
+      console.log('scheduling children in bulk');
+      return childrenScheduler.inBulk();
+    }
+  }
+
+  if (hasUnfinishedChild(scheduledChildren)) {
+    console.log(
+      `[Children Unfinished]: Found unfinished children for task with id ${parentTaskId}, quitting parent processing`
+    );
+    return;
+  }
+
+  if (await hasFailedChildAndMarkedParent(task, scheduledChildren)) {
+    console.log(
+      `[Children Failed]: Found Failed children for task with id ${parentTaskId}, quitting parent processing`
+    );
+    return;
+  }
+
+  if (
+    await areChildrenSucceededAndMarkedParent({
+      parent: task,
+      scheduledChildren: scheduledChildren,
+      expectedChildrenCount: expectedChildTasks.length,
+    })
+  ) {
+    console.log(
+      `[Children Completed Successfully]: All children for task with id ${parentTaskId} completed Successfully, quitting parent processing.`
+    );
+    return;
+  }
+  const shouldScheduleNextChild = _shouldScheduleNextChild({
+    parentTaskConfig,
+    scheduledChildren,
+    expectedChildTasks,
+  });
+  if (shouldScheduleNextChild) {
+    const toScheduleChildIndex = scheduledChildren.length;
+    console.log(
+      `[Sequential Scheduling]: Scheduling child with index ${toScheduleChildIndex} for task with id ${parentTaskId}`
+    );
+    return childrenScheduler.sequentially(toScheduleChildIndex);
+  }
+  throw new Error(
+    `[scheduleChildTasksAndUpdateParent] - Something went wrong with task: ${task.name} this line should not be reached`
+  );
+};
+
+module.exports = { scheduleChildTasksAndUpdateParent };

package/backend/tasks/parentChildTasks/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('./handler'),
+};

package/backend/tasks/parentChildTasks/parentStatusManager.js

@@ -0,0 +1,41 @@
+const { TASK_STATUS } = require('../consts');
+const { markTask } = require('../utils');
+
+const { isPermanentlyFailed, isCompletedSuccessfully, isUnfinishedTask } = require('./utils');
+
+const hasFailedChildAndMarkedParent = async (parent, children) => {
+  const failedChild = children.find(child => isPermanentlyFailed(child));
+  if (failedChild) {
+    console.log('Child task failed, so marking parent as failed!!');
+    await markTask({
+      task: parent,
+      status: TASK_STATUS.FAILED,
+      error: failedChild.error,
+    });
+    return true;
+  }
+  return false;
+};
+
+const hasUnfinishedChild = children => children.some(child => isUnfinishedTask(child));
+
+const areChildrenSucceededAndMarkedParent = async ({
+  parent,
+  scheduledChildren,
+  expectedChildrenCount,
+}) => {
+  const allChildrenSucceeded =
+    scheduledChildren.length === expectedChildrenCount &&
+    scheduledChildren.every(isCompletedSuccessfully);
+  if (allChildrenSucceeded) {
+    await markTask({ task: parent, status: TASK_STATUS.SUCCESS });
+    return true;
+  }
+  return false;
+};
+
+module.exports = {
+  hasFailedChildAndMarkedParent,
+  hasUnfinishedChild,
+  areChildrenSucceededAndMarkedParent,
+};

package/backend/tasks/parentChildTasks/utils.js

@@ -0,0 +1,28 @@
+const { TASK_TYPE, TASK_STATUS, TASK_MAX_TRIES } = require('../consts');
+
+const isCompletedSuccessfully = task =>
+  [TASK_STATUS.SUCCESS, TASK_STATUS.SKIPPED].includes(task.status);
+
+const isPermanentlyFailed = task =>
+  task.status === TASK_STATUS.FAILED && task.amountOfRetries === TASK_MAX_TRIES;
+const isFinishedTask = task => isCompletedSuccessfully(task) || isPermanentlyFailed(task);
+
+const isUnfinishedTask = task => !isCompletedSuccessfully(task) && !isPermanentlyFailed(task);
+const createChildTaskPayload = ({ taskName, taskData, parentTaskId }) => ({
+  name: taskName,
+  data: taskData,
+  type: TASK_TYPE.SCHEDULED,
+  parentTaskId,
+});
+const shouldScheduleNextChild = ({ parentTaskConfig, scheduledChildren, expectedChildTasks }) =>
+  parentTaskConfig.scheduleChildrenSequentially &&
+  scheduledChildren.length < expectedChildTasks.length;
+
+module.exports = {
+  createChildTaskPayload,
+  isPermanentlyFailed,
+  isCompletedSuccessfully,
+  isFinishedTask,
+  isUnfinishedTask,
+  shouldScheduleNextChild,
+};

package/backend/tasks/resultsSaver.js

@@ -0,0 +1,124 @@
+const { wixData, wixCollections } = require('../elevated-modules');
+
+const { detectFieldType } = require('./fieldTypeDetector');
+const { createCollectionIfMissing } = require('./utils');
+
+/**
+ * Creates a results collection for a specific task if it doesn't exist
+ * @param {string} taskName - The name of the task
+ * @returns {Promise<string>} - The collection ID
+ */
+function createResultsCollectionIfMissing(taskName) {
+  const collectionId = `${taskName}_results`;
+
+  const resultsFields = [
+    { key: 'taskId', type: 'TEXT' },
+    { key: 'taskName', type: 'TEXT' },
+    { key: 'status', type: 'TEXT' },
+    { key: 'result', type: 'OBJECT' },
+  ];
+
+  return createCollectionIfMissing(collectionId, resultsFields, null, 'Results');
+}
+
+/**
+ * Adds fields dynamically to a results collection based on the result object structure
+ * @param {string} collectionId - The collection ID
+ * @param {Object} result - The result object to analyze for fields
+ */
+async function addDynamicFieldsToCollection(collectionId, result) {
+  if (!result || typeof result !== 'object') {
+    console.log(`[addDynamicFieldsToCollection] No valid result object to analyze for fields`);
+    return;
+  }
+
+  try {
+    const collection = await wixCollections.getDataCollection(collectionId);
+    const existingFields = new Set(collection.fields.map(field => field.key));
+    const newFields = [];
+
+    Object.keys(result).forEach(key => {
+      if (!existingFields.has(key)) {
+        const value = result[key];
+        const fieldType = detectFieldType(value);
+        newFields.push({ key, type: fieldType });
+      }
+    });
+
+    if (newFields.length > 0) {
+      console.log(
+        `[addDynamicFieldsToCollection] Adding ${newFields.length} new fields to collection ${collectionId}`
+      );
+
+      const updatedFields = [...collection.fields, ...newFields];
+      await wixCollections.updateDataCollection({
+        _id: collectionId,
+        revision: collection.revision,
+        fields: updatedFields,
+      });
+
+      console.log(
+        `[addDynamicFieldsToCollection] Successfully added new fields: ${newFields.map(f => f.key).join(', ')}`
+      );
+    }
+  } catch (error) {
+    // If updating fields fails, we'll continue anyway - the result will still be saved in the 'result' OBJECT field
+    console.error(
+      `[addDynamicFieldsToCollection] Failed to add dynamic fields to collection ${collectionId}: ${error.message}`
+    );
+  }
+}
+
+/**
+ * Saves task results to a dedicated collection
+ * @param {Object} task - The task object
+ * @param {string} status - The task status
+ * @returns {Promise<string>} - The inserted result ID
+ */
+async function saveTaskResultToAdditionalCollection(task, status, result) {
+  try {
+    const { _id: taskId, name: taskName } = task;
+
+    console.log(`[saveTaskResult] Saving result for task ${taskName}`);
+
+    const collectionId = await createResultsCollectionIfMissing(taskName);
+
+    await addDynamicFieldsToCollection(collectionId, result);
+
+    const dataToInsert = {
+      taskId,
+      taskName,
+      status,
+      result,
+      // Add individual fields from result for easier querying
+      ...(result && typeof result === 'object' ? result : {}),
+    };
+
+    const insertedResult = await wixData.insert(collectionId, dataToInsert);
+
+    console.log(
+      `[saveTaskResult] Successfully saved result for task ${taskName} with ID ${insertedResult._id}`
+    );
+    return insertedResult._id;
+  } catch (error) {
+    const errMsg = `[saveTaskResult] Failed to save result for task ${task.name}: ${error.message}`;
+    console.error(errMsg);
+    throw new Error(errMsg);
+  }
+}
+
+/**
+ * Checks if a task configuration has results saving enabled
+ * @param {Object} taskConfig - The task configuration
+ * @returns {boolean} - Whether results saving is enabled
+ */
+function shouldSaveResultsToAdditionalCollection(taskConfig) {
+  return taskConfig && taskConfig.saveResults === true;
+}
+
+module.exports = {
+  createResultsCollectionIfMissing,
+  addDynamicFieldsToCollection,
+  saveTaskResultToAdditionalCollection,
+  shouldSaveResultsToAdditionalCollection,
+};

package/backend/tasks/taskManager.js

@@ -0,0 +1,141 @@
+const { withDuration } = require('../../public/performance/duration'); //relative path to needed file,  so we won't break site due to frontend modules imports inside backend
+
+const { TASK_STATUS, CRON_JOB_MAX_DURATION_SEC } = require('./consts');
+const { scheduleChildTasksAndUpdateParent } = require('./parentChildTasks/handler');
+const {
+  saveTaskResultToAdditionalCollection,
+  shouldSaveResultsToAdditionalCollection,
+} = require('./resultsSaver');
+const {
+  getTaskConfig,
+  markTask,
+  insertNewTask,
+  getScheduledTasks,
+  isParentTask,
+  getTasksToProcess,
+  bulkInsertTasks,
+} = require('./utils');
+
+function taskManager() {
+  const processTask = (task, tasksConfig) =>
+    withDuration(
+      'processTask',
+      async () => {
+        const taskName = task.name;
+        const didFailBefore = task.status === TASK_STATUS.FAILED;
+        let saveResults = false;
+        try {
+          console.log(`[processTask] for ${taskName} Started`);
+          const taskConfig = getTaskConfig(taskName, tasksConfig);
+          saveResults = shouldSaveResultsToAdditionalCollection(taskConfig);
+          await markTask({ task, status: TASK_STATUS.IN_PROGRESS });
+          if (isParentTask(task, taskConfig)) {
+            await scheduleChildTasksAndUpdateParent(task, tasksConfig);
+            //TODO: handle collecting children outputs and use it, e.g csv generation
+            return task; // we return here, as we let upper line handle parent task status update;
+          }
+          const identifier = await taskConfig.getIdentifier(task);
+          if (taskConfig.shouldSkipCheck(identifier)) {
+            console.log(`[${taskName}] Task for identifier ${identifier} will be skipped`);
+            await markTask({
+              task,
+              status: TASK_STATUS.SKIPPED,
+              error: `${taskName} skipped`,
+            });
+            console.log(`[${taskName}] Task for identifier ${identifier} is skipped`);
+            return task;
+          }
+
+          const result = await taskConfig.process(identifier);
+          //setting the processing output to the task data, so it can be used by the parent task if needed
+          task.data.output = result;
+
+          await markTask({ task, status: TASK_STATUS.SUCCESS });
+
+          if (saveResults) {
+            await saveTaskResultToAdditionalCollection(task, TASK_STATUS.SUCCESS, result);
+          }
+
+          console.log(
+            `[processTask] for ${taskName} Task with identifier ${identifier} completed successfully`
+          );
+          return task;
+        } catch (error) {
+          const errMsg = `[processTask] for ${taskName} Task failed with error: ${error}`;
+          console.error(errMsg);
+          const amountOfRetries = didFailBefore ? task.amountOfRetries + 1 : task.amountOfRetries; // this is the first time it fails
+          await markTask({
+            task,
+            error,
+            status: TASK_STATUS.FAILED,
+            amountOfRetries,
+          });
+          if (saveResults) {
+            await saveTaskResultToAdditionalCollection(task, TASK_STATUS.FAILED, { error });
+          }
+          throw new Error(errMsg);
+        }
+      },
+      console
+    );
+  /**
+   * @description Processing tasks based on how many the cron job tick can handle and running them sequentially to be as safe as possible
+   * */
+  const processTasksBasedOnVeloLimit = async (scheduledTasks, tasksConfig) => {
+    console.log(
+      `processTasksBasedOnVeloLimit: ${JSON.stringify(scheduledTasks)} ${JSON.stringify(tasksConfig)}`
+    );
+    const toProcessTasks = getTasksToProcess({
+      scheduledTasks,
+      tasksConfig,
+      maxDuration: CRON_JOB_MAX_DURATION_SEC,
+    });
+    console.log(`processTasksBasedOnVeloLimit: toProcessTasks: ${JSON.stringify(toProcessTasks)}`);
+    console.log(
+      `[processTasksBasedOnVeloLimit] started processing: ${toProcessTasks.length} tasks`
+    );
+    for (const task of toProcessTasks) {
+      // want to run tasks sequentially, but still don't want if one fails to stop others from running to gain the most from each job tick
+      try {
+        await processTask(task, tasksConfig);
+      } catch (error) {
+        console.error(
+          `[processTasksBasedOnVeloLimit] failed to process task ${JSON.stringify(
+            task
+          )} with error: ${error}`
+        );
+      }
+    }
+    console.log(
+      `[processTasksBasedOnVeloLimit] finished processing: ${toProcessTasks.length} tasks`
+    );
+  };
+  const runScheduledTasks = async tasksConfig => {
+    try {
+      const scheduledTasks = await getScheduledTasks();
+      console.log(`runScheduledTasks: scheduledTasks: ${JSON.stringify(scheduledTasks)}`);
+      console.log(`runScheduledTasks: tasksConfig: ${JSON.stringify(tasksConfig)}`);
+      if (scheduledTasks.length) {
+        await processTasksBasedOnVeloLimit(scheduledTasks, tasksConfig);
+      } else {
+        console.log(
+          '[runScheduledTasks] No Tasks to process',
+          JSON.stringify({ scheduledTasksCount: scheduledTasks.length })
+        );
+      }
+    } catch (error) {
+      console.error(`[runScheduledTasks] failed with error: ${error}`);
+      throw error;
+    }
+  };
+
+  return {
+    schedule: insertNewTask,
+    scheduleInBulk: bulkInsertTasks,
+    processTask,
+    runScheduledTasks: (...args) =>
+      withDuration('runScheduledTasks', () => runScheduledTasks(...args), console),
+  };
+}
+
+module.exports = { taskManager };

package/backend/tasks/utils.js

@@ -0,0 +1,262 @@
+const { COLLECTIONS, COLLECTIONS_FIELDS } = require('../../public/consts'); // need relative import to not break site due to frontend modules imports inside backend
+const { wixData, wixCollections } = require('../elevated-modules');
+
+const { TASK_STATUS, TASK_TYPE, TASK_MAX_TRIES } = require('./consts');
+
+/**
+ * Generic function to create a collection if it doesn't exist
+ * @param {string} collectionId - The ID of the collection to create
+ * @param {Array} fields - Array of field definitions for the collection
+ * @param {Object} permissions - Permissions object for the collection
+ * @param {string} collectionType - Description of the collection type for logging
+ * @returns {Promise<string>} - The collection ID
+ */
+async function createCollectionIfMissing(
+  collectionId,
+  fields,
+  permissions = null,
+  collectionType = 'Collection'
+) {
+  console.log(`[createCollectionIfMissing] Checking for collection with ID: ${collectionId}`);
+
+  const defaultPermissions = { insert: 'ADMIN', update: 'ADMIN', remove: 'ADMIN', read: 'ADMIN' };
+  const collectionPermissions = permissions || defaultPermissions;
+
+  try {
+    // try to get the collection by id, if it doesn't exist, this will throw an error
+    await wixCollections.getDataCollection(collectionId);
+    console.log(
+      `[createCollectionIfMissing] ${collectionType} collection ${collectionId} already exists`
+    );
+    return collectionId;
+  } catch (e) {
+    if (e.message.includes('WDE0025')) {
+      // an error that indicates that the collection does not exist.
+      console.log(
+        `[createCollectionIfMissing] ${collectionType} collection ${collectionId} does not exist. Creating...`
+      );
+      const createDataCollectionOptions = {
+        id: collectionId,
+        fields,
+        permissions: collectionPermissions,
+      };
+
+      if (collectionType === 'singleItem') {
+        createDataCollectionOptions.plugins = [
+          {
+            type: "SINGLE_ITEM",
+            singleItemOptions: {},
+          },
+        ];
+      }
+      await wixCollections.createDataCollection(createDataCollectionOptions);
+      console.log(
+        `[createCollectionIfMissing] ${collectionType} collection ${collectionId} created successfully`
+      );
+      return collectionId;
+    } else {
+      // unexpected error
+      console.error(
+        `[createCollectionIfMissing] While trying to get collection with id: ${collectionId} an unexpected error occurred: ${e}`
+      );
+      throw e;
+    }
+  }
+}
+
+const getTaskConfig = (taskName, tasksConfig) => {
+  const taskConfig = Object.values(tasksConfig).find(t => t.name === taskName);
+  if (!taskConfig) {
+    throw new Error(`unknown task name ${taskName}`);
+  }
+  return taskConfig;
+};
+
+const updateTask = async task => {
+  try {
+    await wixData.update(COLLECTIONS.TASKS, task);
+  } catch (err) {
+    const errMsg = `[updateTask] for ${JSON.stringify(task)} failed with error: ${err.message}`;
+    console.error(errMsg);
+    throw new Error(errMsg);
+  }
+};
+const markTask = async ({ task, status, error, amountOfRetries }) => {
+  const toMarkTask = {
+    ...task,
+    ...(status && { status }),
+    ...(error && { error: error.message ?? error }),
+    ...(amountOfRetries !== undefined && { amountOfRetries }),
+  };
+  await updateTask(toMarkTask);
+};
+
+const validateTask = (task, methodName) => {
+  if (
+    typeof task.name !== 'string' ||
+    !task.name ||
+    typeof task.data !== 'object' ||
+    !task.data ||
+    typeof task.type !== 'string' ||
+    !task.type ||
+    (task.parentTaskId !== undefined &&
+      (typeof task.parentTaskId !== 'string' || !task.parentTaskId))
+  ) {
+    throw new Error(`[${methodName}] Invalid params: ${JSON.stringify(task)}`);
+  }
+};
+
+const insertNewTask = async ({ name, data, type, parentTaskId }) => {
+  await createCollectionIfMissing(COLLECTIONS.TASKS, COLLECTIONS_FIELDS.TASKS);
+  try {
+    validateTask({ name, data, type, parentTaskId }, 'insertNewTask');
+    console.log('inserting pending task', name);
+    const insertedTask = await wixData.insert(COLLECTIONS.TASKS, {
+      name,
+      parentTaskId,
+      data,
+      status: TASK_STATUS.PENDING,
+      type,
+      amountOfRetries: 0,
+    });
+    console.log(`Task ${name} inserted successfully`);
+    return insertedTask._id;
+  } catch (err) {
+    const errMsg = `[insertNewTask] Failed for Task ${name} with error ${err.message}`;
+    console.error(errMsg);
+    throw new Error(errMsg);
+  }
+};
+const bulkInsertTasks = async tasks => {
+  await createCollectionIfMissing(COLLECTIONS.TASKS, COLLECTIONS_FIELDS.TASKS);
+
+  // Validate tasks before attempting database operations
+  const toInsertTasks = tasks.map(task => {
+    validateTask(task, 'bulkInsertTasks');
+    return {
+      ...task,
+      status: TASK_STATUS.PENDING,
+      amountOfRetries: 0,
+    };
+  });
+
+  try {
+    const insertedTasks = await wixData.bulkInsert(COLLECTIONS.TASKS, toInsertTasks);
+    return insertedTasks.insertedItemIds;
+  } catch (err) {
+    const errMsg = `[bulkInsertTasks] Failed for Tasks with error ${err.message}`;
+    console.error(errMsg);
+    throw new Error(errMsg);
+  }
+};
+
+const getExpectedChildTasks = (task, taskConfig) => {
+  const childTasks =
+    typeof taskConfig.childTasks === 'function'
+      ? taskConfig.childTasks(task)
+      : taskConfig.childTasks;
+  return childTasks;
+};
+const isParentTask = (task, taskConfig) => {
+  if (!('childTasks' in taskConfig)) {
+    return false;
+  }
+  const childTasks = getExpectedChildTasks(task, taskConfig);
+  if (!Array.isArray(childTasks) || childTasks.length === 0) {
+    throw new Error('taskConfig.childTasks must be a non-empty array if defined');
+  }
+  return true;
+};
+const filterScheduledTasksByStatus = (tasks, tasksConfig) => {
+  console.log(
+    `filterScheduledTasksByStatus: ${JSON.stringify(tasks)} ${JSON.stringify(tasksConfig)}`
+  );
+  const filtered = tasks.filter(task => {
+    if (task.status === TASK_STATUS.IN_PROGRESS) {
+      //Only include parent tasks that are in progress
+      console.log(`filterScheduledTasksByStatus: task: ${JSON.stringify(task)}`);
+      console.log(`filterScheduledTasksByStatus: tasksConfig: ${JSON.stringify(tasksConfig)}`);
+      return isParentTask(task, tasksConfig[task.name]);
+    }
+    return [TASK_STATUS.PENDING, TASK_STATUS.FAILED].includes(task.status);
+  });
+  console.log(`filterScheduledTasksByStatus: filtered: ${JSON.stringify(filtered)}`);
+  return filtered;
+};
+const getScheduledTasks = async () => {
+  try {
+    const baseQuery = wixData.query(COLLECTIONS.TASKS);
+    const scheduledTasksQuery = baseQuery
+      .eq('type', TASK_TYPE.SCHEDULED)
+      .hasSome('status', [TASK_STATUS.PENDING, TASK_STATUS.FAILED, TASK_STATUS.IN_PROGRESS]);
+    const eventBasedTasksQuery = baseQuery
+      .eq('type', TASK_TYPE.EVENT)
+      .eq('status', TASK_STATUS.FAILED);
+    return await scheduledTasksQuery
+      .or(eventBasedTasksQuery)
+      .lt('amountOfRetries', TASK_MAX_TRIES)
+      .ascending('_updatedDate')
+      .limit(1000)
+      .find()
+      .then(result => result.items);
+  } catch (error) {
+    throw new Error(`[getScheduledTasks] failed with error: ${error.message}`);
+  }
+};
+const getTaskScheduledChildren = async parentTaskId => {
+  try {
+    return await wixData
+      .query(COLLECTIONS.TASKS)
+      .eq('parentTaskId', parentTaskId)
+      .ascending('_createdDate')
+      .limit(1000)
+      .find()
+      .then(result => result.items);
+  } catch (error) {
+    console.error(
+      `[getTaskScheduledChildren] failed for task [${parentTaskId}] with error: ${error.message}`
+    );
+    throw error;
+  }
+};
+
+const getTasksToProcess = ({ scheduledTasks, tasksConfig, maxDuration }) => {
+  console.log('getTasksToProcess:', { scheduledTasks, tasksConfig, maxDuration });
+  const tasksToProcess = [];
+  let totalDuration = 0;
+  console.log(`tasksConfig is : ${JSON.stringify(tasksConfig)}`);
+  const filteredScheduledTasks = filterScheduledTasksByStatus(scheduledTasks, tasksConfig);
+  console.log(`filteredScheduledTasks is  : ${JSON.stringify(filteredScheduledTasks)}`);
+  for (const task of filteredScheduledTasks) {
+    console.log(`task is : ${JSON.stringify(task)}`);
+    const taskConfig = getTaskConfig(task.name, tasksConfig);
+    console.log(`taskConfig is : ${JSON.stringify(taskConfig)}`);
+    const estimated = taskConfig?.estimatedDurationSec;
+    if (!estimated) {
+      const errMsg = `[getTasksToProcess] estimatedDurationSec is not defined for task ${task.name}`;
+      console.error(errMsg);
+      throw new Error(errMsg);
+    }
+    const buffered = estimated * 1.5; // 1.5x buffer to be safe
+
+    if (totalDuration + buffered > maxDuration) break;
+    totalDuration += buffered;
+    tasksToProcess.push(task);
+  }
+  console.log('getTasksToProcess: tasksToProcess:', tasksToProcess);
+  return tasksToProcess;
+};
+
+module.exports = {
+  createCollectionIfMissing,
+  getTaskConfig,
+  updateTask,
+  markTask,
+  insertNewTask,
+  bulkInsertTasks,
+  getScheduledTasks,
+  getTaskScheduledChildren,
+  isParentTask,
+  getTasksToProcess,
+  getExpectedChildTasks,
+};

package/index.js

@@ -1,7 +1,4 @@
-// Main entry point for psdev-task-manager
-const script = require('./script');
-
-// Export all functions from script.js
 module.exports = {
-  ...script,
+  ...require('./backend'),
+  ...require('./public')
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "psdev-task-manager",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Task manager library",
   "keywords": [
     "task-manager"
@@ -19,5 +19,9 @@
   "main": "index.js",
   "scripts": {
     "test": "jest"
+  },
+  "dependencies": {
+    "@wix/data": "^1.0.273",
+    "@wix/essentials": "^0.1.27"
   }
 }

package/public/consts.js

@@ -0,0 +1,29 @@
+const COLLECTIONS = {
+  TASKS: 'Tasks',
+};
+
+const COLLECTIONS_FIELDS = {
+  TASKS: [
+    { key: 'name', type: 'TEXT' },
+    { key: 'data', type: 'OBJECT' },
+    { key: 'type', type: 'TEXT', enum: ['event', 'scheduled'] },
+    {
+      key: 'status',
+      type: 'TEXT',
+      enum: ['pending', 'in_progress', 'success', 'skipped', 'failed'],
+    },
+    { key: 'error', type: 'TEXT' },
+    { key: 'amountOfRetries', type: 'NUMBER' },
+    {
+      key: 'parentTaskId',
+      type: 'REFERENCE',
+      typeMetadata: { reference: { referencedCollectionId: 'Tasks' } },
+    },
+    { key: 'Task Result', type: 'TEXT' },
+  ],
+};
+
+module.exports = {
+  COLLECTIONS,
+  COLLECTIONS_FIELDS,
+};

package/public/index.js

@@ -0,0 +1,6 @@
+// Export all functions from script.js
+module.exports = {
+  ...require('./performance'),
+  ...require('./consts'),
+
+};

package/public/performance/duration.js

@@ -0,0 +1,14 @@
+async function withDuration(functionName, func, logger) {
+  const start = Date.now();
+  const result = await func();
+  logger.log(
+    `${functionName} duration: ${
+      Date.now() - start
+    }ms and in human readble format: ${(Date.now() - start) / 1000} s`
+  );
+  return result;
+}
+
+module.exports = {
+  withDuration,
+};

package/public/performance/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('./duration'),
+};

package/script.js

@@ -1,7 +0,0 @@
-const myFunction = () => {
-  console.log("Hello World");
-};
-
-module.exports = {
-  myFunction,
-};