ti2 1.0.93 → 1.0.95

package/controllers/app.js

@@ -1,18 +1,33 @@
-const jwt = require('jwt-promise');
+const jwt = require('jsonwebtoken');
 const { omit } = require('ramda');
 const assert = require('assert');
 const { Umzug, SequelizeStorage } = require('umzug');
 const path = require('path');
 const Sequelize = require('sequelize');
-const fs = require('fs').promises;
+const fs = require('fs'); // Changed to synchronous fs for initial load
+const yaml = require('js-yaml');
 const bb = require('bluebird');
 const R = require('ramda');
 
+// Load OpenAPI schema
+let openApiSchema = null;
+try {
+  const schemaPath = path.join(__dirname, '..', 'api.yml'); // Assuming api.yml is in the parent directory of controllers
+  const schemaFile = fs.readFileSync(schemaPath, 'utf8');
+  openApiSchema = yaml.load(schemaFile);
+} catch (e) {
+  console.error('Failed to load OpenAPI schema:', e);
+  // Handle error appropriately, perhaps by preventing the app from starting
+  // or by using a default empty schema to prevent crashes later.
+  // For now, we'll let it potentially crash if openApiSchema is null and accessed.
+}
+
 const sqldb = require('../models');
 const {
   queue,
   addJob,
   jobStatus,
+  removeJob,
 } = require('../worker/queue');
 
 const { env: { jwtSecret } } = process;
@@ -96,7 +111,7 @@ const createAppToken = async (req, res, next) => {
     if (userAppKeyDup) await userAppKeyDup.destroy();
     const newAppKey = await sqldb.UserAppKey.create(payload);
     // create any cronjobs related to the app
-    await bb.each(req.app.plugins, async plugin => {
+        await bb.each(req.app.plugins, async plugin => {
       if (Array.isArray(plugin.jobs)) {
         const validJobs = plugin.jobs.filter(job => Boolean(job.cron) && Boolean(job.method));
         await bb.each(validJobs, async job => {
@@ -151,6 +166,7 @@ const createAppToken = async (req, res, next) => {
       }
     });
 
+
     return res.json({ value: newAppKey.get('id').toString() });
   } catch (err) {
     return next(err);
@@ -297,7 +313,7 @@ const migrateApp = async ({ integrationId, action }) => {
     'migrations',
   );
   try {
-    await fs.access(migrationsPath);
+    await fs.promises.access(migrationsPath);
   } catch (err) {
     throw Error(`Could not find any migrations for ${integrationId} on ${migrationsPath}`);
   }
@@ -332,12 +348,10 @@ const migrateApp = async ({ integrationId, action }) => {
 const getAppScheduledJobs = async ({
   integrationId,
   userId,
-  hint,
 }) => {
   const where = R.reject(R.isNil)({
     pluginName: integrationId,
     userId,
-    hint,
   });
   const jobs = await sqldb.CronJobs.findAll({
     where,
@@ -487,6 +501,252 @@ const getAppToken = async (req, res, next) => {
   return res.json({ token: await userAppKey.token });
 };
 
+const createCronjob = async (req, res, next) => {
+  const {
+    params: {
+      userId,
+    },
+    body: {
+      method,
+      url,
+      cron,
+      callbackUrl,
+      body,
+      removeOnComplete,
+    },
+  } = req;
+
+  let transaction;
+
+  try {
+    // Validate required fields
+    if (!method || !url || !cron) {
+      const error = new Error('method, url, and cron are required fields');
+      error.status = 400;
+      throw error;
+    }
+
+    // Validate URL path
+    const validUrlPattern = /^\/[\w\-\/\.]+$/;
+    if (!validUrlPattern.test(url)) {
+      const error = new Error('Invalid URL path');
+      error.status = 400;
+      throw error;
+    }
+  
+    // Helper function to find a matching path in the OpenAPI schema
+    const findMatchingOpenApiPath = (requestUrl, schemaPaths) => {
+      if (!schemaPaths) return null;
+      for (const schemaPathKey in schemaPaths) {
+        // Convert schemaPathKey to a regex: e.g., /products/{id} -> ^/products/[^/]+$
+        const regexPattern = '^' + schemaPathKey.replace(/{[^}]+}/g, '[^/]+') + '$';
+        const regex = new RegExp(regexPattern);
+        if (regex.test(requestUrl)) {
+          return schemaPathKey; // Return the original schema path key
+        }
+      }
+      return null;
+    };
+
+    const matchedSchemaPathKey = findMatchingOpenApiPath(url, openApiSchema ? openApiSchema.paths : null);
+
+    // Validate URL and method against OpenAPI schema
+    if (!openApiSchema || !matchedSchemaPathKey || !openApiSchema.paths[matchedSchemaPathKey][method.toLowerCase()]) {
+      const errorMessage = openApiSchema && openApiSchema.paths ?
+        `Invalid URL ('${url}') or method ('${method}') combination not found in API schema, or schema path not matched.` :
+        'API schema not loaded, cannot validate URL and method.';
+      const error = new Error(errorMessage);
+      error.status = 400;
+      throw error;
+    }
+
+    // Create a user token for the job test
+    const jobToken = jwt.sign({ userId }, process.env.jwtSecret);
+
+    // Start transaction
+    transaction = await sqldb.sequelize.transaction();
+
+    // Create job in database
+    const cronJobData = {
+      userId,
+      // bullJobId will be set by the hook
+      cron,
+      method,
+      url,
+      token: jobToken,
+      body: { // Ensure body includes callbackUrl if present, or is an empty object if body is null/undefined
+        ...(body || {}),
+        ...(callbackUrl ? { callbackUrl } : {}),
+      },
+      removeOnComplete: removeOnComplete || false, // Ensure it defaults to false if not provided
+    };
+    
+    const cronJob = await sqldb.ApiCronJobs.create(cronJobData, { transaction });
+
+    await transaction.commit();
+    // The cronJob instance returned here should have bullJobId populated by the hook
+    return res.json(R.omit(['token'], cronJob.get({ plain:true })));
+  } catch (err) {
+    if (transaction) await transaction.rollback();
+    // No need to manually cleanup bullJobId here.
+    // If hook failed, transaction is rolled back by the hook re-throwing,
+    // or by create itself failing.
+    // If create itself failed before hook, no bull job was created.
+    return next(err);
+  }
+};
+
+const listCronjobs = async (req, res, next) => {
+  const {
+    params: {
+      userId,
+    },
+  } = req;
+
+  try {
+    // Get all jobs from Bull queue
+    const bullJobs = await queue.getJobs(['active', 'wait', 'delayed']);
+    const bullJobIds = new Set(bullJobs.map(job => job.id));
+
+    // Get repeatable jobs
+    const repeatableJobs = await queue.getRepeatableJobs();
+    const repeatablePatterns = new Set(repeatableJobs.map(job => job.cron));
+
+    // Get jobs from database
+    const jobs = await sqldb.ApiCronJobs.findAll({
+      where: {
+        userId,
+      },
+      attributes: { exclude: ['token'] },
+      raw: true,
+    });
+
+    // Add inQueue status to jobs
+    const jobsWithStatus = jobs.map(job => {
+      const jobIdParts = job.bullJobId.split(':');
+      const isRepeatJob = jobIdParts[0] === 'repeat';
+      let isInQueue = false;
+
+      if (isRepeatJob) {
+        isInQueue = repeatablePatterns.has(job.cron);
+      } else {
+        isInQueue = bullJobIds.has(job.bullJobId);
+      }
+      return { ...job, inQueue: isInQueue };
+    });
+
+    return res.json({ jobs: jobsWithStatus });
+  } catch (err) {
+    return next(err);
+  }
+};
+
+const deleteCronjob = async (req, res, next) => {
+  const {
+    params: {
+      userId,
+      id, // Changed from jobId to id
+    },
+  } = req;
+
+  // Extract token from Authorization header
+  const getToken = (req) => {
+    if (
+      !req.header('Authorization') ||
+      req.header('Authorization').substring(0, 7) !== 'Bearer '
+    ) return null;
+    return req.header('Authorization').split(' ')[1];
+  };
+  
+  const token = getToken(req);
+  let transaction; // Add transaction variable for deleteCronjob
+
+  try {
+    // Check if user is admin
+    const isAdmin = token === process.env.adminKey;
+
+    // For non-admin users, verify token
+    if (!isAdmin) {
+      try {
+        const decoded = jwt.verify(token, process.env.jwtSecret);
+        // Verify userId in token matches requested userId
+        if (!decoded || !decoded.userId || decoded.userId !== userId) {
+          throw new Error('User ID mismatch');
+        }
+      } catch (err) {
+        return res.status(403).json({
+          status: 403,
+          message: 'Forbidden'
+        });
+      }
+    }
+
+    // Find the cronjob in the database
+    const cronJob = await sqldb.ApiCronJobs.findOne({
+      where: {
+        userId,
+        id, // Changed from bullJobId to id
+      },
+    });
+
+    // If cronjob doesn't exist, return 404
+    if (!cronJob) {
+      const error = new Error('Cronjob not found');
+      error.status = 404;
+      return next(error);
+    }
+
+    // For non-admin users, verify ownership of the cronjob
+    if (!isAdmin && userId !== cronJob.userId) {
+      return res.status(403).json({
+        status: 403,
+        message: 'Forbidden'
+      });
+    }
+
+    // Start transaction
+    transaction = await sqldb.sequelize.transaction();
+
+    // Remove from database. The beforeDestroy hook in ApiCronJobs model
+    // will handle removing the job from the Bull queue.
+    // Pass the transaction to ensure atomicity.
+    // The destroy operation will only succeed if the beforeDestroy hook (including removeJob) succeeds.
+    const destroyedRows = await sqldb.ApiCronJobs.destroy({
+      where: {
+        // id, // cronJob instance is already fetched and verified
+        id: cronJob.id, // Use the id from the fetched cronJob instance
+        userId: cronJob.userId, // Ensure we are deleting the correct user's job
+      },
+      transaction, // Pass transaction to destroy
+    });
+
+    // It's good practice to check if any rows were actually deleted,
+    // though in this flow, cronJob existence is checked prior.
+    if (destroyedRows === 0) {
+        // This case should ideally not be reached if cronJob was found
+        if (transaction) await transaction.rollback();
+        const error = new Error('Cronjob not found for deletion, though it was fetched prior.');
+        error.status = 404; // Or 500 for internal inconsistency
+        return next(error);
+    }
+    
+    await transaction.commit();
+    // Return success response
+    return res.json({ success: true });
+  } catch (err) {
+    if (transaction) await transaction.rollback(); // Rollback transaction on error
+    // Pass along errors with status
+    if (err.status) {
+      return next(err);
+    }
+    
+    // Otherwise, create a generic 500 error
+    const error = new Error(err.message || 'Internal Server Error');
+    error.status = 500;
+    return next(error); // Changed from next(err) to next(the new error object) for consistency
+  }
+};
+
 module.exports = plugins => ({
   createAppToken,
   getAppToken,
@@ -504,4 +764,7 @@ module.exports = plugins => ({
   tokenTemplate,
   validateAppToken: validateAppToken(plugins),
   getAffiliates: getAffiliates(plugins),
+  createCronjob,
+  deleteCronjob,
+  listCronjobs,
 });

package/index.js

@@ -191,7 +191,7 @@ module.exports = async ({
     app.use(async (req, res, next) => {
       const startHrTime = process.hrtime();
       const body = composeBodyFromReq(req);
-      req.customBody = body;
+      req.requestBody = R.clone(body);
       ti2Events.emit('request.start', body);
       req.requestId = body.requestId;
       res.on('finish', async () => {
@@ -201,7 +201,7 @@ module.exports = async ({
           10,
         );
         ti2Events.emit('request.end', {
-          ...body,
+          ...req.requestBody,
           cacheKey: req.cacheKey,
           responseTimeInMs,
           responseStatusCode: res.statusCode,
@@ -312,9 +312,8 @@ module.exports = async ({
             addJob({
               type: 'callback',
               payload: callbackBody,
-            }).then(jobId => {
-              console.debug('created callback job', jobId);
-            });
+              removeOnComplete: true,
+            }).then();
             return originalJson.call(this, data);
           };
         }
@@ -328,7 +327,7 @@ module.exports = async ({
           ...cacheSettings['*'],
           ...(currentPlugin ? R.keys(filteredPluginCacheSettings) : []),
         ];
-        const body = req.customBody;
+        const body = req.requestBody;
         if (cachingOperations.indexOf(body.operationId) > -1) {
           const cacheBody = R.omit(['requestId', 'date'], body);
           const cacheKey = hash(cacheBody);

package/migrations/20240130000000-add-token-to-cronjobs.js

@@ -0,0 +1,12 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('CronJobs', 'token', {
+      type: Sequelize.STRING,
+      allowNull: true
+    });
+  },
+
+  down: async (queryInterface) => {
+    await queryInterface.removeColumn('CronJobs', 'token');
+  }
+};

package/migrations/20250528_create_apiCronjobs.js

@@ -0,0 +1,61 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    // Create a new table with the updated schema
+    await queryInterface.createTable('ApiCronJobs', {
+      id: {
+        type: Sequelize.UUID,
+        defaultValue: Sequelize.UUIDV4,
+        primaryKey: true,
+      },
+      userId: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      bullJobId: {
+        type: Sequelize.STRING,
+        allowNull: true,
+      },
+      cron: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      method: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      url: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      callbackUrl: {
+        type: Sequelize.STRING,
+        allowNull: true,
+      },
+      token: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      body: {
+        type: Sequelize.JSON,
+        allowNull: true,
+      },
+      removeOnComplete: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false,
+        defaultValue: false,
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+    });
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    await queryInterface.dropTable('ApiCronJobs');
+  },
+};

package/models/apiCronJobs.js

@@ -0,0 +1,96 @@
+const Sequelize = require('sequelize');
+const db = require('./db');
+
+const ApiCronJobs = db.define('ApiCronJobs', {
+  id: {
+    type: Sequelize.UUID,
+    defaultValue: Sequelize.UUIDV4,
+    primaryKey: true,
+  },
+  userId: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  bullJobId: {
+    type: Sequelize.STRING,
+    // It might be null initially until the afterCreate hook successfully adds the job
+    allowNull: true,
+  },
+  cron: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  method: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  url: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  token: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  body: {
+    type: Sequelize.JSON,
+    allowNull: true, // Assuming body can be optional
+  },
+  removeOnComplete: {
+    type: Sequelize.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  }
+}, {
+  hooks: {
+    afterCreate: async (apiCronJob, options) => {
+      const { addJob } = require('../worker/queue');
+      const jobPayload = {
+        type: 'api',
+        method: apiCronJob.method,
+        url: apiCronJob.url,
+        headers: {
+          'content-type': 'application/json',
+          'Authorization': `Bearer ${apiCronJob.token}`,
+        },
+        payload: (apiCronJob.body || {}),
+      };
+
+        const jobParams = {
+          repeat: {
+            cron: apiCronJob.cron,
+          },
+          removeOnComplete: apiCronJob.removeOnComplete,
+        };
+
+      try {
+        // The addJob must be part of the same transaction that created apiCronJob
+        const bullJobId = await addJob(jobPayload, jobParams, options.transaction);
+        // Update the instance with the bullJobId and save it within the same transaction
+        // Note: This will trigger update hooks if any are defined, which is generally fine.
+        // apiCronJob.bullJobId = bullJobId;
+        await apiCronJob.update({ bullJobId }, { transaction: options.transaction, hooks: false });
+      } catch (err) {
+        // If addJob or the subsequent save fails, the transaction should be rolled back
+        // by the controller that initiated it.
+        // We re-throw the error to ensure the transaction is rolled back.
+        console.error('Failed to create Bull job or save bullJobId:', err);
+        throw err; // This will trigger the rollback in the controller
+      }
+    },
+    beforeDestroy: async (apiCronJob, options) => {
+      const { removeJob } = require('../worker/queue');
+      try {
+        // The removeJob must be part of the same transaction
+        await removeJob(apiCronJob.bullJobId, options.transaction);
+      } catch (err) {
+        // If removeJob fails, the transaction should be rolled back.
+        console.error('Failed to remove Bull job:', err);
+        throw err; // This will trigger the rollback in the controller
+      }
+    },
+  },
+});
+
+module.exports = ApiCronJobs;
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ti2",
-  "version": "1.0.93",
+  "version": "1.0.95",
   "description": "Tourist Industry Exchange (TI2)",
   "main": "index.js",
   "scripts": {
@@ -28,7 +28,7 @@
     "axios": "^0.27.2",
     "axios-curlirize": "^1.3.7",
     "bluebird": "^3.7.2",
-    "bull": "^4.8.2",
+    "bull": "^4.16.5",
     "commander": "^9.2.0",
     "compression": "^1.7.4",
     "eventemitter2": "^6.4.9",

package/test/utils.js

@@ -9,7 +9,10 @@ const Plugin = require('./plugin');
 const slugify = require('./slugify');
 const sqldb = require('../models/db');
 
-const { env: { adminKey } } = process;
+const jwt = require('jsonwebtoken');
+const { env: { adminKey, jwtSecret } } = process;
+
+const createUserToken = (userId) => jwt.sign({ userId }, jwtSecret);
 
 afterAll(async () => {
   await sqldb.connectionManager.close();
@@ -51,8 +54,14 @@ module.exports = async (appParams = {}) => {
     } else {
       resp = await request(app)[verb](url).query(query).send(payload);
     }
-    // if (resp.statusCode !== 200) assert.strictEqual(resp, `${url}-${expectStatusCode}`);
-    assert.strictEqual(`${url}-${resp.statusCode}`, `${url}-${expectStatusCode}`);
+    if (resp.statusCode !== expectStatusCode) {
+      const error = new Error();
+      error.response = {
+        status: resp.statusCode,
+        data: resp.body
+      };
+      throw error;
+    }
     if (rawResponse) return resp;
     return resp.body;
   };
@@ -62,9 +71,9 @@ module.exports = async (appParams = {}) => {
   const doApiPut = params => doApi({ ...params, verb: 'put' });
   const doApiDelete = params => doApi({ ...params, verb: 'delete' });
 
-  const appSetup = async ({ userId: userIdParam } = {}) => {
+  const appSetup = async ({ userId: userIdParam, appName: appNameParam } = {}) => {
     const userId = userIdParam || chance.guid();
-    const appName1 = slugify(
+    const appName1 = appNameParam || slugify(
       chance.company(),
     ).toLowerCase();
     const newApp = {
@@ -77,31 +86,49 @@ module.exports = async (appParams = {}) => {
       endpoint: chance.url(),
       apiKey: apiKey1,
     };
-    const { body: { value: appKey } } = await request(app)
-      .post('/app')
-      .set('Authorization', `Bearer ${adminKey}`)
-      .send(newApp);
+    
+    let appKey;
+    
+    // Only create a new app if appNameParam is not provided
+    if (!appNameParam) {
+      const { body: { value: newAppKey } } = await request(app)
+        .post('/app')
+        .set('Authorization', `Bearer ${adminKey}`)
+        .send(newApp);
+      appKey = newAppKey;
+    } else {
+      // If appNameParam is provided, we'll use the adminKey directly
+      // This avoids the need for a GET /app/{appName} endpoint
+      appKey = adminKey;
+    }
+    
+    // Create user token for the app
+    const hint = token.apiKey.split('-')[0];
     await request(app)
       .post(`/${newApp.name}/${userId}`)
       .set('Authorization', `Bearer ${appKey}`)
       .send({
-        tokenHint: token.apiKey.split('-')[0],
+        tokenHint: hint,
         token,
       });
+    
     return {
       newApp,
       token,
       appKey,
       userId,
+      hint,
     };
   };
 
   return {
+    app,
     appSetup,
     doApiDelete,
     doApiGet,
     doApiPost,
     doApiPut,
+    createUserToken,
     slugify,
     plugins: app.plugins,
   };

package/worker/__tests__/job-callback.js

@@ -45,28 +45,6 @@ describe('worker: Callback job handling', () => {
     }));
   });
   
-  it('should include the correct structure for callback jobs', () => {
-    // This test verifies the worker/index.js handles callback jobs correctly
-    // by checking the code structure without actually running it
-    
-    // Read the worker code
-    const workerCode = require('fs').readFileSync(require('path').resolve(__dirname, '../index.js'), 'utf8');
-    
-    // Verify the worker code includes handling for callback jobs
-    expect(workerCode).toContain('if (type === \'callback\')');
-    
-    // Verify it extracts the necessary parameters
-    expect(workerCode).toContain('callbackUrl,');
-    expect(workerCode).toContain('request,');
-    expect(workerCode).toContain('result,');
-    
-    // Verify it calls the sendCallback function
-    expect(workerCode).toContain('sendCallback');
-    
-    // Verify it returns the expected result structure
-    expect(workerCode).toContain('return { success: true }');
-  });
-  
   it('should use the callback module to send callbacks', async () => {
     // Get a reference to the mocked axios.post function
     const axios = require('axios');