spice-js 2.6.39 → 2.6.40

package/build/utility/RestHelper.js

@@ -3,12 +3,14 @@
 exports.__esModule = true;
 exports.default = void 0;
 
-var _flat = _interopRequireDefault(require("flat"));
-
 var _lodash = _interopRequireDefault(require("lodash"));
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+function _getRequireWildcardCache() { if (typeof WeakMap !== "function") return null; var cache = new WeakMap(); _getRequireWildcardCache = function _getRequireWildcardCache() { return cache; }; return cache; }
+
+function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
 
 function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
@@ -75,8 +77,12 @@ class RestHelper {
         var content;
 
         if (download_type == "csv") {
+          var {
+            flatten
+          } = yield Promise.resolve().then(() => _interopRequireWildcard(require("flat")));
+
           var items = _lodash.default.map(ctx.data, item => {
-            return (0, _flat.default)(item);
+            return flatten(item);
           });
 
           var fields = _lodash.default.union(_lodash.default.keys(_lodash.default.first(items)), _lodash.default.keys(_lodash.default.last(items)), _lodash.default.keys(_lodash.default.nth(items.length / 2)));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spice-js",
-  "version": "2.6.39",
+  "version": "2.6.40",
   "description": "spice",
   "main": "build/index.js",
   "repository": {
@@ -14,28 +14,28 @@
   "author": "Chad Fraser <chad@sonover.com> (http://sonover.com)",
   "license": "ISC",
   "dependencies": {
-    "agenda": "^3.1.0",
+    "agenda": "^5.0.0",
     "co": "^4.6.0",
-    "dotenv": "^8.2.0",
-    "flat": "^5.0.0",
-    "fs-extra": "^8.1.0",
+    "dotenv": "^16.4.5",
+    "flat": "^6.0.1",
+    "fs-extra": "^11.2.0",
     "json2csv": "^4.5.4",
-    "juice": "^6.0.0",
+    "juice": "^11.0.0",
     "kcors": "^2.2.2",
-    "koa-body": "^4.1.1",
-    "koa-convert": "^1.2.0",
-    "koa-qs": "^2.0.0",
-    "koa-socket-2": "^1.2.0",
+    "koa-body": "^6.0.1",
+    "koa-convert": "^2.0.0",
+    "koa-qs": "^3.0.0",
+    "koa-socket-2": "^2.0.0",
     "koa-static": "^5.0.0",
     "koa-validation": "^0.1.9",
     "koa-views": "^6.2.3",
-    "koa2-swagger-ui": "^5.5.1",
+    "koa2-swagger-ui": "^5.10.0",
     "lodash": "^4.17.15",
     "node-cache": "^5.1.2",
     "nunjucks": "^3.2.0",
-    "open": "7.0.0",
+    "open": "10.1.0",
     "parse-comments": "^1.0.0",
-    "regenerator-runtime": "^0.13.7",
+    "regenerator-runtime": "^0.14.1",
     "socket.io-client": "^4.0.1",
     "sonover-date": "^1.0.1",
     "uuid": "^3.3.3",

package/src/artificial_intelligence/Ai.js

@@ -0,0 +1,244 @@
+import _ from "lodash";
+const crypto = require("crypto");
+const path = require("path");
+import fs, { promises, WriteStream } from "fs";
+import { pipeline } from "stream/promises";
+import { spawn } from "child_process";
+
+export default class AI {
+  constructor(args = {}) {
+    this.provider = args.provider || spice.config.ai.default_driver;
+    console.log(this.provider, args.provider, spice.config.ai.default_driver);
+    this.options = _.merge(
+      spice.config.ai.drivers[this.provider],
+      args.options
+    );
+    //console.log(this.provider_name, spice.config.ai.providers)
+    this.Provider = spice.config.ai.providers[this.provider];
+  }
+
+  createUniqueDirectory() {
+    const timestamp = new Date().toISOString().replace(/[:.-]/g, "");
+    const randomString = crypto.randomBytes(4).toString("hex");
+    const uniqueDirName = `job_${timestamp}_${randomString}`;
+    const project_root = path.resolve(spice.root_path, "..");
+    const storageDir = path.join(project_root, "storage/jobs");
+    const uniqueDir = path.join(storageDir, uniqueDirName);
+
+    if (!fs.existsSync(storageDir)) {
+      fs.mkdirSync(storageDir, { recursive: true });
+    }
+    fs.mkdirSync(uniqueDir);
+
+    return uniqueDir;
+  }
+
+  async extractOrStoreFile(fileStream, outputDir, supportedFileTypes) {
+    // Ensure the output directory exists
+    if (!fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir, { recursive: true });
+    }
+
+    // Create a temporary file to store the input
+    const tempFilePath = path.join(outputDir, "temp_input");
+    await pipeline(fileStream, fs.createWriteStream(tempFilePath));
+
+    try {
+      // Check file type
+      const fileType = await this.getFileType(tempFilePath);
+
+      if (_.includes(supportedFileTypes, fileType)) {
+        // If it's a JPEG or PNG, just move it to the output directory
+        const timestamp = new Date().toISOString().replace(/[:.-]/g, "");
+        const extension = fileType === "jpeg" ? "jpg" : fileType;
+        const outputPath = path.join(outputDir, `${timestamp}.${extension}`);
+        fs.renameSync(tempFilePath, outputPath);
+        return [
+          {
+            type: fileType,
+            media_type: this.getMimeType(fileType),
+            path: outputPath,
+          },
+        ];
+      } else {
+        // If it's a PDF, extract pages
+        if (fileType === "pdf") {
+          const outputPrefix = path.join(outputDir, "page");
+
+          // Use pdftoppm (part of poppler-utils) to convert PDF to images
+          await new Promise((resolve, reject) => {
+            const process = spawn("pdftoppm", [
+              "-jpeg",
+              tempFilePath,
+              outputPrefix,
+            ]);
+            process.on("close", (code) => {
+              if (code === 0) resolve();
+              else
+                reject(new Error(`pdftoppm process exited with code ${code}`));
+            });
+          });
+
+          // Get a list of extracted images
+          const files = fs.readdirSync(outputDir);
+          const images = files.filter(
+            (file) => file.startsWith("page") && file.endsWith(".jpg")
+          );
+
+          return images.map((image) => {
+            return {
+              type: "jpeg",
+              media_type: this.getMimeType("jpeg"),
+              path: path.join(outputDir, image),
+            };
+          });
+        } else {
+          throw new Error(
+            "Unsupported file type. Only PDF, JPEG, and PNG are supported."
+          );
+        }
+      }
+    } catch (error) {
+      console.error("Error processing file:", error);
+      throw error;
+    } finally {
+      // Clean up the temporary input file
+      if (fs.existsSync(tempFilePath)) {
+        fs.unlinkSync(tempFilePath);
+      }
+    }
+  }
+
+  getMimeType(fileType) {
+    // Mapping of file extensions to MIME types
+    const mimeTypes = {
+      jpeg: "image/jpeg",
+      jpg: "image/jpeg", // Adding 'jpg' as it's a common alias for 'jpeg'
+      png: "image/png",
+      gif: "image/gif",
+      bmp: "image/bmp",
+      webp: "image/webp",
+      svg: "image/svg+xml",
+      pdf: "application/pdf",
+      txt: "text/plain",
+      html: "text/html",
+      css: "text/css",
+      js: "application/javascript",
+      json: "application/json",
+      xml: "application/xml",
+      csv: "text/csv",
+      mp3: "audio/mpeg",
+      wav: "audio/wav",
+      mp4: "video/mp4",
+      avi: "video/x-msvideo",
+      mov: "video/quicktime",
+      zip: "application/zip",
+      rar: "application/vnd.rar",
+      "7z": "application/x-7z-compressed",
+      // Add more MIME types as needed
+    };
+
+    // Ensure the fileType is a string and convert it to lowercase
+    if (typeof fileType !== "string") {
+      throw new TypeError("fileType must be a string");
+    }
+
+    const ext = fileType.toLowerCase();
+
+    // Return the MIME type if found, otherwise return the default
+    return mimeTypes[ext] || "application/octet-stream";
+  }
+
+  async getFileType(filePath) {
+    return new Promise((resolve, reject) => {
+      const fileSignature = Buffer.alloc(8);
+      fs.open(filePath, "r", (err, fd) => {
+        if (err) reject(err);
+        fs.read(fd, fileSignature, 0, 8, 0, (err) => {
+          if (err) reject(err);
+          fs.close(fd, (err) => {
+            if (err) console.error("Error closing file:", err);
+          });
+
+          const hex = fileSignature.toString("hex");
+          if (hex.startsWith("ffd8")) {
+            resolve("jpeg");
+          } else if (hex.startsWith("89504e47")) {
+            resolve("png");
+          } else if (fileSignature.toString("ascii").startsWith("%PDF")) {
+            resolve("pdf");
+          } else {
+            resolve("unknown");
+          }
+        });
+      });
+    });
+  }
+
+  async deleteFolderRecursive(dirPath) {
+    if (fs.existsSync(dirPath)) {
+      for (const entry of fs.readdirSync(dirPath)) {
+        const entryPath = path.join(dirPath, entry);
+        if (fs.lstatSync(entryPath).isDirectory()) {
+          await this.deleteFolderRecursive(entryPath);
+        } else {
+          await fs.promises.unlink(entryPath);
+        }
+      }
+      await fs.promises.rmdir(dirPath);
+    }
+  }
+
+  async processFiles(files) {
+    let types = this.getSupportedFileTypes();
+    const outputDir = this.createUniqueDirectory();
+
+    return (
+      await Promise.all(
+        files.map(async (fileStream) => {
+          //let type = await this.getFileType(fileStream);
+          const imagePaths = await this.extractOrStoreFile(
+            fileStream,
+            outputDir,
+            types
+          );
+          return imagePaths;
+        })
+      )
+    ).flat();
+  }
+
+  getSupportedFileTypes(options) {
+    if (this.Provider === undefined) {
+      throw new Error(`AI provider ${this.provider} is not defined`);
+    }
+    let provider = new this.Provider(this.options);
+    return provider.getSupportedTypes();
+  }
+
+  async prompt(prompt, { files, system_message, prompt_options }, options) {
+    let context_files = await this.processFiles(files);
+    try {
+      if (options) {
+        this.options = _.merge(this.options, options);
+      }
+      //console.log(this.Provider);
+      if (this.Provider === undefined) {
+        throw new Error(`AI provider ${this.provider} is not defined`);
+      }
+      let provider = new this.Provider(this.options);
+      return await provider.prompt(prompt, {
+        prompt_options,
+        system_message,
+        files: context_files,
+      });
+    } catch (e) {
+      console.error(e);
+      return { error: e.message };
+    } finally {
+      for (let context_file of context_files) {
+        await this.deleteFolderRecursive(context_file);
+      }
+    }
+  }
+}

package/src/config/ai.js

@@ -0,0 +1,3 @@
+module.exports = {
+    
+}

package/src/index.js

@@ -7,7 +7,7 @@ global.spice = {};
 require("dotenv").config({ path: path.resolve("./") });
 
 var Validate = require("koa-validation");
-const koaBody = require("koa-body");
+const { koaBody } = require("koa-body");
 spice.parent_app_dir = path.resolve("./");
 spice.root_path = path.resolve("./build/");
 spice.module_root_path = path.resolve(__dirname);
@@ -33,6 +33,7 @@ export { default as MailDebug } from "./mail/providers/Debug";
 export { default as DebugStorage } from "./storage/providers/Debug";
 export { default as LocalStorage } from "./storage/providers/Local";
 export { default as Storage } from "./storage/Storage";
+export { default as AI } from "./artificial_intelligence/Ai.js";
 
 export { default as MailFile } from "./mail/providers/File";
 export { default as SpiceCache } from "./cache/providers/SpiceCache";

package/src/mail/providers/File.js

@@ -1,32 +1,32 @@
-let fs = require('fs-extra');
-var path = require('path');
-var open = require("open");
+let fs = require("fs-extra");
+var path = require("path");
 
-export default class File{
-    
-	constructor(args={}){
-        this.sender = args.sender;
-        this.subject = args.subject;
-        this.htmlText = args.htmlText;
-        this.dest = path.join(path.resolve('./'),'storage/mail');
-        if(args.path){
-            this.dest = path.resolve(args.path)
-        }
+export default class File {
+  constructor(args = {}) {
+    this.sender = args.sender;
+    this.subject = args.subject;
+    this.htmlText = args.htmlText;
+    this.dest = path.join(path.resolve("./"), "storage/mail");
+    if (args.path) {
+      this.dest = path.resolve(args.path);
     }
+  }
 
-    async send(htmlText, options={}){
+  async send(htmlText, options = {}) {
+    const openModule = await import("open");
+    const open = openModule.default;
+    let email = {
+      to: options.to,
+      from: options.sender || this.sender,
+      subject: options.subject || this.subject,
+      htmlText: htmlText || this.htmlText,
+    };
 
-        let email = {
-            to: options.to,
-            from: options.sender || this.sender,
-            subject: options.subject || this.subject,
-            htmlText: htmlText || this.htmlText
-        }
-        
-        let time = new Date().getTime();
-        fs.ensureDirSync(`${this.dest}/${email.subject}/${time}`);
-        let full_path = `${this.dest}/${email.subject}/${time}/to:${email.to} from:${email.from}.html`;
-        fs.writeFileSync(full_path, email.htmlText);
-        open(full_path);
-    }
-}
+    let time = new Date().getTime();
+    fs.ensureDirSync(`${this.dest}/${email.subject}/${time}`);
+    let full_path = `${this.dest}/${email.subject}/${time}/to:${email.to} from:${email.from}.html`;
+    fs.writeFileSync(full_path, email.htmlText);
+
+    open(full_path);
+  }
+}