stacks 0.58.73 → 0.59.1

package/core/email/src/server/converter.ts

@@ -1,305 +0,0 @@
-/* eslint-disable eslint-comments/no-unlimited-disable */
-/* eslint-disable */
-
-const AWS = require('aws-sdk')
-const mime = require('mime')
-const parser = require('mailparser').simpleParser
-
-// Initialize S3.
-const s3 = new AWS.S3({
-  apiVersion: '2006-03-01',
-})
-
-// This lambda will read RAW emails and convert them in easy to read formats
-// like: HTML and text
-exports.handler = (event: any) => {
-  // 1. We need to process the path received by S3 since AWS dose escape the
-  // string in a special way. They escape the string in a HTML style
-  // but for whatever reason they convert spaces in to +ses.
-  const s3_key = event.Records[0].s3.object.key
-
-  // 2. So first we convert the + in to spaces
-  const plus_to_space = s3_key.replace(/\+/g, ' ')
-
-  // 3. And then we unescape the string, other wise we lose real + characters
-  const unescaped_key = decodeURIComponent(plus_to_space)
-
-  // 4. This JS object will contain all the data within the chain
-  const container = {
-    bucket: event.Records[0].s3.bucket.name,
-    key: unescaped_key,
-    parsed: {
-      html: '',
-      text: '',
-      attachments: [],
-    },
-  }
-
-  // ->	Start the chain.
-  load_the_email(container)
-    .then((container) => {
-      return remove_extension(container)
-    }).then((container) => {
-      return parse_the_email(container)
-    }).then((container) => {
-      return save_text(container)
-    }).then((container) => {
-      return save_html(container)
-    }).then((container) => {
-      return save_attachments(container)
-    }).then((container) => {
-      return true
-    }).catch((error) => {
-      console.error(error)
-
-      return false
-    })
-}
-
-// Load the email that triggered the function from S3
-function load_the_email(container: any) {
-  return new Promise((resolve, reject) => {
-    console.info('load_the_email')
-
-    // 1. Set the query.
-    const params = {
-      Bucket: container.bucket,
-      Key: container.key,
-    }
-
-    // ->	Execute the query
-    s3.getObject(params, (error: any, data: any) => {
-      // 1. Check for internal errors
-      if (error)
-        return reject(error)
-
-      // 2. Save the email for the next promise
-      container.raw_email = data.Body
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    })
-  })
-}
-
-// Since the raw email is saved with an extension we need to remove it
-// before we can save other converted versions.
-function remove_extension(container: any) {
-  return new Promise((resolve, reject) => {
-    console.info('remove_extension')
-
-    // 1. Split the string.
-    const tmp = container.key.split('.')
-
-    // 2. Remove the last element from the array which is `eml`.
-    tmp.pop()
-
-    // 2. Join the array back to how it was, minus the extension.
-    // We need to do this, since we don't know how many dots
-    // were in the email title for example.
-    container.key = tmp.join('.')
-
-    // ->	Move to the next chain.
-    return resolve(container)
-  })
-}
-
-// Convert the raw email in to HTML and Text, and extract also the attachments
-function parse_the_email(container: any) {
-  return new Promise((resolve, reject) => {
-    // 1. Parse the email and extract all the necessary
-    parser(container.raw_email, (error: any, parsed: any) => {
-      // 1. Check for internal errors
-      if (error)
-        return reject(error)
-
-      // 2. Save the parsed email for the next promise.
-      container.parsed.html = parsed.html
-      container.parsed.text = parsed.text
-      container.parsed.attachments = parsed.attachments
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    })
-  })
-}
-
-// After the conversion we save the Text version to S3.
-function save_text(container: any) {
-  return new Promise((resolve, reject) => {
-    console.info('save_text')
-
-    // 1. Set the query
-    const params = {
-      Bucket: container.bucket,
-      Key: `${container.key}.txt`,
-      Body: container.parsed.text,
-    }
-
-    // ->	Execute the query.
-    s3.putObject(params, (error: any, data: any) => {
-      // 1. Check for internal errors.
-      if (error)
-        return reject(error)
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    })
-  })
-}
-
-// Then if we have a HTML version we try to save that.
-function save_html(container: any) {
-  return new Promise((resolve, reject) => {
-    // <>> When the body of an email have only one version, meaning it
-    // does have only the pure text version and no HTML.
-    // Nodemailer won't generate the HTML for you, it just grabs
-    // what is in the Email body.
-    // So, this value will be false, when there is no HTML content in
-    // the email, and thus we skip this step.
-    if (!container.parsed.html) {
-      console.info('save_html - skipped')
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    }
-
-    console.info('save_html')
-
-    // 1. Set the query.
-    const params = {
-      Bucket: container.bucket,
-      Key: `${container.key}.html`,
-      Body: container.parsed.html,
-    }
-
-    // ->	Execute the query.
-    s3.putObject(params, (error: any, data: any) => {
-      // 1. Check for internal errors.
-      if (error)
-        return reject(error)
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    })
-  })
-}
-
-// Last thing to do is to save the attachments if there are any. If the
-// array is empty the loop will skip itself.
-function save_attachments(container: any) {
-  return new Promise((resolve, reject) => {
-    console.info('save_attachments')
-
-    // Start the loop which save all the attachments.
-    loop(1, (error: any) => {
-      // <<> Check if there was an error.
-      if (error) {
-        // ->	Stop everything and surface the error.
-        return reject(container)
-      }
-
-      // ->	Move to the next chain.
-      return resolve(container)
-    })
-
-    // This loop will upload all the individual attachments found in a email
-    function loop(count: any, callback: any) {
-      // 1. Pop the last element in the array if any.
-      const file = container.parsed.attachments.pop()
-
-      // 2. If there is nothing left then we stop the loop.
-      if (!file)
-        return callback()
-
-      // 3. Get the file name which also contain the file extension.
-      let file_name = file.filename
-
-      // 4. An email attachment is not required to have a name, this
-      // mean we need to check if we have a file name and, if not
-      // we create a general name, so all attachments can be saved
-      // and accounted for.
-      if (!file_name) {
-        // 1. Set the generic name with the count name so we give
-        // unique names to each attachment.
-        file_name = `Mail Attachment ${count}`
-
-        // 2. We only count the times we had to set a generic name.
-        count++
-      }
-
-      // 5. Split the string to and get the last element of the array
-      // which should be the file extension.
-      const extension = file_name.split('.').pop()
-
-      // 6. Then we check if the last element is an extension.
-      const is_extension = mime.getType(extension)
-
-      // 7. If it isn't, we get one from the content-type header.
-      if (!is_extension) {
-        // 1. Make sure we have a content type, since I'm sure
-        // some emails won't have it if they were malformed.
-        if (file.contentType) {
-          // 1. Try to find out the file extension from the
-          // content-type found in the email message.
-          let file_type = mime.getExtension(file.contentType)
-
-          // 2. If the extension was not found, then we set a
-          // default one which will let the user know
-          // that we don't know the type of the file, while
-          // also setting the type to txt (for convenience)
-          // so it can be open by any editor and inspected.
-          if (!file_type)
-            file_type = 'unknown_extension.txt'
-
-          // 3. Attach the extension to the file.
-          file_name += `.${file_type}`
-        }
-      }
-
-      // 8. Then save the buffer of the attachment.
-      const file_body = file.content
-
-      // 9. Split the S3 Key (path) so we can remove the last element. Since
-      // we don't want the object name, we care only about the path.
-      const tmp = container.key.split('/')
-
-      // 10. Now remove the last element from the array which is the
-      // file name that contains the raw email, which we don't want.
-      tmp.pop()
-
-      // 11. After all this, we recombine the array in to a single
-      // string which becomes again the S3 Key minus the file name.
-      const path = tmp.join('/')
-
-      // 12. This variable is used if there are two files of the same name.
-      let cid = ''
-
-      // 13. If the CID is set then it means that two files have the same names
-      if (file.cid) {
-        // 1. Add the CID in front of the name of the file so they won't be overwritten
-        cid = `${file.cid} - `
-      }
-
-      // 14. Create the full key path with the object at the end.
-      const key = 	`${path}/attachments/${cid}${file_name}`
-
-      // 15. Set the query.
-      const params = {
-        Bucket: container.bucket,
-        Key: key,
-        Body: file_body,
-      }
-
-      // ->	Execute the query.
-      s3.putObject(params, (error: any, data: any) => {
-        // 1. Check for internal errors.
-        if (error)
-          return callback(error)
-
-        // ->	Move to the next chain.
-        return loop(count, callback)
-      })
-    }
-  })
-}

package/core/email/src/server/inbound.ts

@@ -1,406 +0,0 @@
-/* eslint-disable eslint-comments/no-unlimited-disable */
-/* eslint-disable */
-import type Stream from 'node:stream'
-import * as AWS from 'aws-sdk'
-import moment from 'moment'
-import { simpleParser as parser } from 'mailparser'
-import { log } from '@stacksjs/logging'
-
-// Initialize S3
-const s3 = new AWS.S3({
-  apiVersion: '2006-03-01',
-})
-
-// Initialize SES
-const ses = new AWS.SES({
-  apiVersion: '2010-12-01',
-})
-
-// This variable will hold all the domain available in SES so we don't
-// have to query SES each time the Lambda runs, we query SES only when
-// the lambda starts from scratch to circumvent the 1 sec request limit.
-const domains: string[] = []
-
-interface Container {
-  bucket: string
-  unescaped_key: string
-  escaped_key: string
-  domains: string[]
-  folder: string
-  // wip
-  raw_email?: Buffer | Stream | string
-  to?: string
-  from?: string
-  date?: Date | string
-  subject?: string
-  message_id?: string
-  to_domain?: string
-  to_account?: string
-  from_domain?: string
-  from_account?: string
-  path?: string
-}
-
-interface S3Event {
-  Records: {
-    s3: {
-      bucket: {
-        name: string
-      }
-      object: {
-        key: string
-      }
-    }
-  }[]
-}
-
-// This Lambda will filter all the incoming emails based on their From and To field
-export function handler(event: S3Event): Promise<any> {
-  // if (!event.Records.length)
-  //   throw new Error('No records in the event')
-
-  // This JS object will contain all the data within the chain.
-  const container: Container = {
-    bucket: event.Records[0]?.s3.bucket.name || '',
-    unescaped_key: '',
-    escaped_key: event.Records[0]?.s3.object.key || '',
-    domains,
-    folder: 'Sent',
-  }
-
-  get_email_domains(container)
-    .then((container) => {
-      return unescape_key(container)
-    }).then((container) => {
-      return load_the_email(container)
-    }).then((container) => {
-      return parse_the_email(container)
-    }).then((container) => {
-      return format_time(container)
-    }).then((container) => {
-      return extract_data(container)
-    }).then((container) => {
-      return where_to_save(container)
-    }).then((container) => {
-      return create_s3_boject_key(container)
-    }).then((container) => {
-      return copy_email_to_inbox(container)
-    }).then((container) => {
-      return copy_email_to_today(container)
-    }).then((container) => {
-      return delete_the_email(container)
-    }).then(() => {
-      return true
-    }).catch((error) => {
-      console.error(error)
-
-      return false
-    })
-
-  return Promise.resolve()
-}
-
-// List all the emails added to SES, so we can use them to decided where to
-// store the emails. If in the Inbox, or sent. This is important when you
-// upload emails by hand to back them up.
-//
-// Without doing this everything goes to the Inbox folder.
-function get_email_domains(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    // If we have already the domains we don't query SES anymore since you
-    // can only query SES once a second. To solve this limitation
-    // we grab the data once, and save it in to memory.
-    if (container.domains.length)
-      return resolve(container)
-
-    log.info('get_email_domains')
-
-    const params = {
-      IdentityType: 'Domain',
-      MaxItems: 100,
-    }
-
-    ses.listIdentities(params, (error, data) => {
-      if (error) {
-        console.error(params)
-        return reject(error)
-      }
-
-      container.domains = data.Identities
-
-      return resolve(container)
-    })
-  })
-}
-
-// We need to process the path received by S3 since AWS dose escape the
-// string in a special way. They escape the string in a HTML style
-// but for whatever reason they convert spaces in to +ses.
-function unescape_key(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('unescape_key')
-
-    // First we convert the + in to spaces
-    const plus_to_space = container.escaped_key.replace(/\+/g, ' ')
-
-    // And then we unescape the string, other wise we lose real + characters
-    const unescaped_key = decodeURIComponent(plus_to_space)
-
-    container.unescaped_key = unescaped_key
-
-    return resolve(container)
-  })
-}
-
-// Load the email that we received from SES.
-function load_the_email(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('load_the_email')
-
-    const params = {
-      Bucket: container.bucket,
-      Key: container.unescaped_key,
-    }
-
-    s3.getObject(params, (error, data) => {
-      if (error) {
-        console.error(params)
-        return reject(error)
-      }
-
-      // Save the email for the next promise
-      // @ts-expect-error - refactor later
-      container.raw_email = data.Body
-
-      return resolve(container)
-    })
-  })
-}
-
-// Once the raw email is loaded we parse it with one goal in mind, get
-// the date the of the email. This way we don't rely on the SES date, but
-// on the real date the email was created.
-//
-// This way we can even load in to the system old emails as long as they
-// are in the standard raw email format, and not some proprietary solution.
-//
-// That why will be organized with the time the emails were created, and not
-// received in to the system.
-function parse_the_email(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('parse_the_email')
-
-    if (!container.raw_email)
-      return reject(Error('No raw email to parse'))
-
-    // Parse the email and extract all that is necessary
-    parser(container.raw_email, (error, data) => {
-      // Check for internal errors
-      if (error) {
-        console.error(data)
-        return reject(error)
-      }
-
-      // Save the parsed email for the next promise.
-      container.date = data.date
-      container.from = data.from?.value[0]?.address
-      // container.to = data.to?.value[0]?.address
-      container.to = data.from?.value[0]?.address
-      container.subject = data.subject || 'No Subject'
-      container.message_id = data.messageId
-
-      return resolve(container)
-    })
-  })
-}
-
-// We format the date and time to make sure that when the folder is saved
-// in S3, the object will sort one after the other which makes it much
-// easier to see the latest emails.
-function format_time(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('format_time')
-
-    // Format the date found in the email message itself
-    container.date = moment(container.date).format('YYYY-MM-DD HH:mm:ss Z')
-
-    return resolve(container)
-  })
-}
-
-// Extract all the data necessary to organize the incoming emails.
-function extract_data(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('extract_data')
-
-    if (!container.to)
-      return reject(Error('No To field in the email'))
-
-    // Since the email string can come in a form of:
-    //  Name Last <name@example.com>
-    // We have to extract just the email address, and discard the rest
-    let tmp_to
-    if (container?.to) {
-      const matchResult = container.to.match(/(?:[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x21\x23-\x5B\x5D-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])*")@(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-zA-Z0-9-]*[a-zA-Z0-9]:(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x21-\x5A\x53-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])+)\])/gm);
-      if (matchResult) {
-        tmp_to = matchResult[0].split('@');
-        // Continue with your logic using tmp_to
-      }
-    }
-
-    let tmp_from
-    if (container?.from) {
-      const matchResult = container.from.match(/(?:[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x21\x23-\x5B\x5D-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])*")@(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-zA-Z0-9-]*[a-zA-Z0-9]:(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x21-\x5A\x53-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])+)\])/gm)
-      if (matchResult) {
-        tmp_from = matchResult[0].split('@');
-        // Continue with your logic using tmp_to
-      }
-    }
-
-    if (!tmp_to || !tmp_from)
-      return reject(Error('No To or From field in the email'))
-
-    // Get the domain name of the receiving end, so we can group
-    // emails by all the domain that were added to SES.
-    container.to_domain = tmp_to[1]
-
-    // Based on the email name, we replace all the + characters, that
-    // can be used to organize ones on-line accounts in to /, this way
-    // we can build a S3 patch which will automatically organize
-    // all the email in structured folder.
-    container.to_account = tmp_to[0]?.replace(/\+/g, '/')
-
-    // Get the domain name of the email which in our case will become the company name
-    container.from_domain = tmp_from[1]
-
-    // Get the name of who sent us the email
-    container.from_account = tmp_from[0]
-
-    // Set all the strings to lower case, because some times different
-    // on-line services will have your email in all caps. For example
-    // Priceline will do this.
-    //
-    // Since in the next step we compare the domain that is in SES
-    // with the data from the email, wee need to have it all in the
-    // same format for the if() statement to work correctly and
-    // generate the right path where to save the email.
-    container.to_domain = container.to_domain?.toLowerCase()
-    container.to_account = container.to_account?.toLowerCase()
-    container.from_domain = container.from_domain?.toLowerCase()
-    container.from_account = container.from_account?.toLowerCase()
-
-    // S3 objects have a limit of how they they can be named
-    // so we remove everything but...
-    container.subject = container.subject?.replace(/[^a-zA-Z0-9 &@:,$=+?;]/g, '_')
-
-    return resolve(container)
-  })
-}
-
-// Once we have the domains from SES, and we know the value of the To
-// field, we can decide where the emails should be saved.
-function where_to_save(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('where_to_save')
-
-    // Since by default we assume that the email should be saved
-    // in the Sent folder, we need to loop over the SES domains
-    // and check if there is a match with the To: domain found in
-    // the email.
-    container.domains.forEach((domain) => {
-      if (domain === container.to_domain)
-        container.folder = 'Inbox'
-    })
-
-    return resolve(container)
-  })
-}
-
-// Create Key path for the S3 object to be saved to.
-function create_s3_boject_key(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('create_s3_boject_key')
-
-    // create the path where the email needs to be moved for house keeping
-    container.path = `${container.to_domain}/${container.to_account}/${container.from_domain}/${container.from_account}/${container.date} - ${container.subject}/` + 'email.eml'
-
-    return resolve(container)
-  })
-}
-
-// Copy the email to a new location - we don't put the email that we
-// already have in memory since the system requires a COPY action and not
-// a PUT action.
-// TODO: refactor below
-// WARNING: We are using the escaped_key value, because there is a
-// known bug in the AWS JS SDK which won't unescape the
-// string, so you have to do it - AWS is aware of this issue.
-function copy_email_to_inbox(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('copy_email_to_inbox')
-
-    const params = {
-      Bucket: container.bucket,
-      CopySource: `${container.bucket}/${container.escaped_key}`,
-      Key: `${container.folder}/${container.path}`,
-    }
-
-    s3.copyObject(params, (error, data) => {
-      if (error) {
-        console.error(params)
-        return reject(error)
-      }
-
-      return resolve(container)
-    })
-  })
-}
-
-// Similarly to the previous promise, but in this case we COPY the email to
-// the Today folder to showcase all the latest emails that were received
-// within 24h.
-//
-// There is a Lifecycle rule set on the bucket to delete any object inside
-// the Today folder if it is older then 24h.
-//
-// This way it is easier to see all the latest emails.
-function copy_email_to_today(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('copy_email_to_today')
-
-    const params = {
-      Bucket: container.bucket,
-      CopySource: `${container.bucket}/${container.escaped_key}`,
-      Key: `today/${container.path}`,
-    }
-
-    s3.copyObject(params, (error, data) => {
-      if (error) {
-        console.error(params)
-        return reject(error)
-      }
-
-      return resolve(container)
-    })
-  })
-}
-
-function delete_the_email(container: Container): Promise<Container> {
-  return new Promise((resolve, reject) => {
-    log.info('delete_the_email')
-
-    const params = {
-      Bucket: container.bucket,
-      Key: container.unescaped_key,
-    }
-
-    s3.deleteObject(params, (error, data) => {
-      if (error) {
-        console.error(params)
-        return reject(error)
-      }
-
-      return resolve(container)
-    })
-  })
-}