velocious 1.0.0 → 1.0.2

package/src/http-server/client/request-buffer/index.mjs

@@ -0,0 +1,229 @@
+import {EventEmitter} from "events"
+import FormDataPart from "./form-data-part.mjs"
+import Header from "./header.mjs"
+import Incorporator from "incorporator"
+import logger from "../../../logger.mjs"
+import ParamsToObject from "../params-to-object.mjs"
+import querystring from "querystring"
+
+export default class RequestBuffer {
+  bodyLength = 0
+  data = []
+  events = new EventEmitter()
+  headers = []
+  headersByName = {}
+  params = {}
+  postBody = ""
+  postBodyChars = []
+  readingBody = false
+  state = "status"
+
+  constructor({configuration}) {
+    this.configuration = configuration
+  }
+
+  feed(data) {
+    for (const char of data) {
+      if (this.readingBody) this.bodyLength += 1
+
+      switch(this.state) {
+        case "status":
+        case "headers":
+        case "multi-part-form-data":
+        case "multi-part-form-data-header":
+          this.data.push(char)
+
+          if (char == 10) {
+            const line = String.fromCharCode.apply(null, this.data)
+
+            this.data = []
+            this.parse(line)
+          }
+
+          break
+        case "multi-part-form-data-body":
+          const body = this.formDataPart.body
+
+          body.push(char)
+
+          const possibleBoundaryEndPosition = body.length - this.boundaryLineEnd.length
+          const possibleBoundaryEndChars = body.slice(possibleBoundaryEndPosition, body.length)
+          const possibleBoundaryEnd = String.fromCharCode.apply(null, possibleBoundaryEndChars)
+
+          const possibleBoundaryNextPosition = body.length - this.boundaryLineNext.length
+          const possibleBoundaryNextChars = body.slice(possibleBoundaryNextPosition, body.length)
+          const possibleBoundaryNext = String.fromCharCode.apply(null, possibleBoundaryNextChars)
+
+          if (possibleBoundaryEnd == this.boundaryLineEnd) {
+            this.formDataPart.removeFromBody(possibleBoundaryEnd)
+            this.formDataPartDone()
+            this.completeRequest()
+          } else if (possibleBoundaryNext == this.boundaryLineNext) {
+            this.formDataPart.removeFromBody(possibleBoundaryNext)
+            this.formDataPartDone()
+            this.newFormDataPart()
+          } else if (this.contentLength && this.bodyLength >= this.contentLength) {
+            this.formDataPartDone()
+            this.completeRequest()
+          } else if (this.formDataPart.contentLength && this.bodyLength >= this.formDataPart.contentLength) {
+            this.formDataPartDone()
+
+            throw new Error("stub")
+          }
+
+          break
+        case "post-body":
+          this.bodyLength += 1
+          this.postBodyChars.push(char)
+
+          if (this.contentLength && this.postBodyChars.length >= this.contentLength) {
+            this.postRequestDone()
+          }
+
+          break
+        default:
+          console.error(`Unknown state: ${this.state}`)
+      }
+    }
+  }
+
+  getHeader = (name) => this.headersByName[name.toLowerCase().trim()]
+
+  formDataPartDone() {
+    const formDataPart = this.formDataPart
+
+    this.formDataPart = undefined
+    formDataPart.finish()
+
+    this.events.emit("form-data-part", formDataPart)
+  }
+
+  isMultiPartyFormData = () => this.multiPartyFormData
+
+  newFormDataPart() {
+    this.formDataPart = new FormDataPart()
+    this.setState("multi-part-form-data-header")
+  }
+
+  parse(line) {
+    if (this.state == "status") {
+      this.parseStatusLine(line)
+    } else if (this.state == "headers") {
+      this.parseHeader(line)
+    } else if (this.state == "multi-part-form-data") {
+      if (line == this.boundaryLine) {
+        this.newFormDataPart()
+      } else if (line == "\r\n") {
+        this.setState("done")
+      } else {
+        throw new Error(`Expected boundary line but didn't get it: ${line}`)
+      }
+    } else if (this.state == "multi-part-form-data-header") {
+      const header = this.readHeaderFromLine(line)
+
+      if (header) {
+        this.formDataPart.addHeader(header)
+        this.state == "multi-part-form-data"
+      } else if (line == "\r\n") {
+        this.setState("multi-part-form-data-body")
+      }
+    } else {
+      throw new Error(`Unknown state: ${this.state}`)
+    }
+  }
+
+  readHeaderFromLine(line) {
+    let match
+
+    if (match = line.match(/^(.+): (.+)\r\n/)) {
+      const header = new Header(match[1], match[2])
+
+      return header
+    }
+  }
+
+  parseHeader(line) {
+    const header = this.readHeaderFromLine(line)
+
+    if (header) {
+      this.headersByName[header.formattedName] = header
+
+      if (header.formattedName == "content-length") this.contentLength = parseInt(header.value)
+
+      this.events.emit("header", header)
+    } else if (line == "\r\n") {
+      if (this.httpMethod.toUpperCase() == "GET") {
+        this.completeRequest()
+      } else if (this.httpMethod.toUpperCase() == "POST") {
+        this.readingBody = true
+
+        const match = this.getHeader("content-type").value.match(/^multipart\/form-data;\s*boundary=(.+)$/i)
+
+        if (match) {
+          this.boundary = match[1]
+          this.boundaryLine = `--${this.boundary}\r\n`
+          this.boundaryLineNext = `\r\n--${this.boundary}\r\n`
+          this.boundaryLineEnd = `\r\n--${this.boundary}--`
+          this.multiPartyFormData = true
+          this.setState("multi-part-form-data")
+        } else {
+          this.setState("post-body")
+        }
+      } else {
+        throw new Error(`Unknown HTTP method: ${this.httpMethod}`)
+      }
+    }
+  }
+
+  parseStatusLine(line) {
+    const match = line.match(/^(GET|POST) (.+?) HTTP\/1\.1\r\n/)
+
+    if (!match) {
+      throw new Error(`Couldn't match status line from: ${line}`)
+    }
+
+    this.httpMethod = match[1]
+    this.path = match[2]
+    this.setState("headers")
+  }
+
+  postRequestDone() {
+    this.postBody += String.fromCharCode.apply(null, this.postBodyChars)
+    this.parseQueryStringPostParams()
+    this.completeRequest()
+  }
+
+  setState(newState) {
+    logger(this, `Changing state from ${this.state} to ${newState}`)
+
+    this.state = newState
+  }
+
+  completeRequest = () => {
+    this.state = "completed"
+
+    if (this.getHeader("content-type")?.value?.startsWith("application/json")) {
+      this.parseApplicationJsonParams()
+    } else if (this.multiPartyFormData) {
+      // Done after each new form data part
+    }
+
+    this.events.emit("completed")
+  }
+
+  parseApplicationJsonParams() {
+    const newParams = JSON.parse(this.postBody)
+    const incorporator = new Incorporator({objects: [this.params, newParams]})
+
+    incorporator.merge()
+  }
+
+  parseQueryStringPostParams() {
+    const unparsedParams = querystring.parse(this.postBody)
+    const paramsToObject = new ParamsToObject(unparsedParams)
+    const newParams = paramsToObject.toObject()
+    const incorporator = new Incorporator({objects: [this.params, newParams]})
+
+    incorporator.merge()
+  }
+}

package/src/http-server/client/request-parser.mjs

@@ -0,0 +1,47 @@
+import {digg} from "diggerize"
+import {EventEmitter} from "events"
+import Incorporator from "incorporator"
+import ParamsToObject from "./params-to-object.mjs"
+import RequestBuffer from "./request-buffer/index.mjs"
+
+export default class VelociousHttpServerClientRequestParser {
+  constructor({configuration}) {
+    if (!configuration) throw new Error("No configuration given")
+
+    this.configuration = configuration
+    this.data = []
+    this.events = new EventEmitter()
+    this.params = {}
+
+    this.requestBuffer = new RequestBuffer({configuration})
+    this.requestBuffer.events.on("completed", this.requestDone)
+    this.requestBuffer.events.on("form-data-part", this.onFormDataPart)
+    this.requestBuffer.events.on("request-done", this.requestDone)
+  }
+
+  onFormDataPart = (formDataPart) => {
+    const unorderedParams = {}
+
+    unorderedParams[formDataPart.getName()] = formDataPart.getValue()
+
+    const paramsToObject = new ParamsToObject(unorderedParams)
+    const newParams = paramsToObject.toObject()
+    const incorporator = new Incorporator({objects: [this.params, newParams]})
+
+    incorporator.merge()
+  }
+
+  feed = (data) => this.requestBuffer.feed(data)
+  getHeader = (name) => this.requestBuffer.getHeader(name)?.value
+  getHttpMethod = () => digg(this, "requestBuffer", "httpMethod")
+  getHost = () => this.requestBuffer.getHeader("host")?.value
+  getPath = () => digg(this, "requestBuffer", "path")
+
+  requestDone = () => {
+    const incorporator = new Incorporator({objects: [this.params, this.requestBuffer.params]})
+
+    incorporator.merge()
+    this.state = "done"
+    this.events.emit("done")
+  }
+}

package/src/http-server/client/{request-runner.cjs → request-runner.mjs}

@@ -1,9 +1,9 @@
-const EventEmitter = require("events")
-const logger = require("../../logger.cjs")
-const Response = require("./response.cjs")
-const RoutesResolver = require("../../routes/resolver.cjs")
+import EventEmitter from "events"
+import logger from "../../logger.mjs"
+import Response from "./response.mjs"
+import RoutesResolver from "../../routes/resolver.mjs"
 
-module.exports = class VelociousHttpServerClientRequestRunner {
+export default class VelociousHttpServerClientRequestRunner {
   events = new EventEmitter()
 
   constructor({configuration, request}) {

package/src/http-server/client/request.mjs

@@ -0,0 +1,15 @@
+import {digg} from "diggerize"
+import RequestParser from "./request-parser.mjs"
+
+export default class VelociousHttpServerClientRequest {
+  constructor({configuration}) {
+    this.configuration = configuration
+    this.requestParser = new RequestParser({configuration})
+  }
+
+  feed = (data) => this.requestParser.feed(data)
+  httpMethod = () => this.requestParser.getHttpMethod()
+  host = () => this.requestParser.getHost()
+  path = () => this.requestParser.getPath()
+  params = () => digg(this, "requestParser", "params")
+}

package/src/http-server/client/{response.cjs → response.mjs}

@@ -1,4 +1,4 @@
-module.exports = class VelociousHttpServerClientResponse {
+export default class VelociousHttpServerClientResponse {
   body = undefined
   headers = {}
 

package/src/http-server/index.mjs

@@ -0,0 +1,137 @@
+import {digg} from "diggerize"
+import logger from "../logger.mjs"
+import Net from "net"
+import ServerClient from "./server-client.mjs"
+import WorkerHandler from "./worker-handler/index.mjs"
+
+export default class VelociousHttpServer {
+  clientCount = 0
+  clients = {}
+  workerCount = 0
+  workerHandlers = []
+
+  constructor({configuration, host, maxWorkers, port}) {
+    this.configuration = configuration
+    this.host = host || "0.0.0.0"
+    this.port = port || 3006
+    this.maxWorkers = maxWorkers || 16
+  }
+
+  async start() {
+    await this._ensureAtLeastOneWorker()
+    this.netServer = new Net.Server()
+    this.netServer.on("connection", this.onConnection)
+    await this._netServerListen()
+  }
+
+  _netServerListen() {
+    return new Promise((resolve, reject) => {
+      try {
+        this.netServer.listen(this.port, this.host, () => {
+          logger(this, `Velocious listening on ${this.host}:${this.port}`)
+          resolve()
+        })
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  async _ensureAtLeastOneWorker() {
+    if (this.workerHandlers.length == 0) {
+      await this.spawnWorker()
+    }
+  }
+
+  isActive() {
+    return this.netServer.listening
+  }
+
+  async stopClients() {
+    for (const clientCount in this.clients) {
+      const client = this.clients[clientCount]
+
+      await client.close()
+    }
+  }
+
+  stopServer() {
+    return new Promise((resolve, reject) => {
+      this.netServer.close((error) => {
+        if (error) {
+          reject(error)
+        } else {
+          resolve()
+        }
+      })
+    })
+  }
+
+  async stop() {
+    await this.stopClients()
+    await this.stopServer()
+  }
+
+  onConnection = (socket) => {
+    const clientCount = this.clientCount
+
+    logger(this, `New client ${clientCount}`)
+
+    this.clientCount++
+
+    const workerHandler = this.workerHandlerToUse()
+    const client = new ServerClient({
+      clientCount,
+      configuration: this.configuration,
+      socket
+    })
+
+    client.events.on("close", this.onClientClose)
+
+    logger(this, `Gave client ${clientCount} to worker ${workerHandler.workerCount}`)
+
+    workerHandler.addSocketConnection(client)
+
+    this.clients[clientCount] = client
+  }
+
+  onClientClose = (client) => {
+    const clientCount = digg(client, "clientCount")
+    const oldClientsLength = Object.keys(this.clients).length
+
+    delete this.clients[clientCount]
+
+    const newClientsLength = Object.keys(this.clients).length
+
+    if (newClientsLength != (oldClientsLength - 1)) {
+      console.error(`Expected client to have been removed but length didn't change from ${oldClientsLength} to ${oldClientsLength - 1}`)
+    }
+  }
+
+  async spawnWorker() {
+    const workerCount = this.workerCount
+
+    this.workerCount++
+
+    const workerHandler = new WorkerHandler({
+      configuration: this.configuration,
+      workerCount
+    })
+
+    await workerHandler.start()
+    this.workerHandlers.push(workerHandler)
+  }
+
+  workerHandlerToUse() {
+    logger(this, `Worker handlers length: ${this.workerHandlers.length}`)
+
+    const randomWorkerNumber = parseInt(Math.random() * this.workerHandlers.length)
+    const workerHandler = this.workerHandlers[randomWorkerNumber]
+
+    if (!workerHandler) {
+      throw new Error(`No workerHandler by that number: ${randomWorkerNumber}`)
+    }
+
+    return workerHandler
+  }
+}

package/src/http-server/server-client.mjs

@@ -0,0 +1,47 @@
+import EventEmitter from "events"
+import logger from "../logger.mjs"
+
+export default class ServerClient {
+  events = new EventEmitter()
+
+  constructor({configuration, socket, clientCount}) {
+    if (!configuration) throw new Error("No configuration given")
+
+    this.configuration = configuration
+    this.socket = socket
+    this.clientCount = clientCount
+
+    socket.on("end", this.onSocketEnd)
+  }
+
+  listen = () => this.socket.on("data", this.onSocketData)
+
+  close() {
+    return new Promise((resolve, reject) => {
+      this.socket.destroy()
+      this.events.emit("close", this)
+      resolve()
+    })
+  }
+
+  onSocketData = (chunk) => {
+    logger(this, `Socket ${this.clientCount}: ${chunk}`)
+
+    this.worker.postMessage({
+      command: "clientWrite",
+      chunk,
+      clientCount: this.clientCount
+    })
+  }
+
+  onSocketEnd = () => {
+    logger(this, `Socket ${this.clientCount} end`)
+    this.events.emit("close", this)
+  }
+
+  send(data) {
+    logger(this, "Send", data)
+
+    this.socket.write(data)
+  }
+}

package/src/http-server/worker-handler/index.mjs

@@ -0,0 +1,79 @@
+import {digg, digs} from "diggerize"
+import {dirname} from "path"
+import {fileURLToPath} from "url"
+import logger from "../../logger.mjs"
+import {Worker} from "worker_threads"
+
+export default class VelociousHttpServerWorker {
+  constructor({configuration, workerCount}) {
+    this.configuration = configuration
+    this.clients = {}
+    this.workerCount = workerCount
+  }
+
+  async start() {
+    return new Promise((resolve) => {
+      const {debug, directory} = digs(this.configuration, "debug", "directory")
+      const __filename = fileURLToPath(import.meta.url)
+      const __dirname = dirname(__filename)
+
+      this.onStartCallback = resolve
+      this.worker = new Worker(`${__dirname}/worker-script.mjs`, {
+        workerData: {
+          debug,
+          directory,
+          workerCount: this.workerCount
+        }
+      })
+      this.worker.on("error", this.onWorkerError)
+      this.worker.on("exit", this.onWorkerExit)
+      this.worker.on("message", this.onWorkerMessage)
+    })
+  }
+
+  addSocketConnection(client) {
+    const clientCount = digg(client, "clientCount")
+
+    client.socket.on("end", () => {
+      logger(this, `Removing ${clientCount} from clients`)
+      delete this.clients[clientCount]
+    })
+
+    client.worker = this.worker
+    client.listen()
+
+    this.clients[clientCount] = client
+    this.worker.postMessage({command: "newClient", clientCount})
+  }
+
+  onWorkerError = (error) => {
+    throw error // Throws original error with backtrace and everything into the console
+  }
+
+  onWorkerExit = (code) => {
+    if (code !== 0) {
+      throw new Error(`Client worker stopped with exit code ${code}`)
+    }
+  }
+
+  onWorkerMessage = (data) => {
+    logger(this, `Worker message`, data)
+
+    const {command} = digs(data, "command")
+
+    if (command == "started") {
+      this.onStartCallback()
+      this.onStartCallback = null
+    } else if (command == "clientOutput") {
+      logger(this, "CLIENT OUTPUT", data)
+
+      const {clientCount, output} = digs(data, "clientCount", "output")
+
+      logger(this, "CLIENT OUTPUT", data)
+
+      this.clients[clientCount].send(output)
+    } else {
+      throw new Error(`Unknown command: ${command}`)
+    }
+  }
+}

package/src/http-server/worker-handler/worker-script.mjs

@@ -0,0 +1,4 @@
+import {workerData, parentPort} from "worker_threads"
+import WorkerThread from "./worker-thread.mjs"
+
+new WorkerThread({parentPort, workerData})

package/src/http-server/worker-handler/{worker-thread.cjs → worker-thread.mjs}

@@ -1,26 +1,33 @@
-const Client = require("../client/index.cjs")
-const Configuration = require("../../configuration.cjs")
-const {digg, digs} = require("diggerize")
-const errorLogger = require("../../error-logger")
-const logger = require("../../logger.cjs")
-
-module.exports = class VelociousHttpServerWorkerHandlerWorkerThread {
+import Application from "../../application.mjs"
+import Client from "../client/index.mjs"
+import DatabasePool from "../../database/pool/index.mjs"
+import {digg, digs} from "diggerize"
+import errorLogger from "../../error-logger.mjs"
+import logger from "../../logger.mjs"
+
+export default class VelociousHttpServerWorkerHandlerWorkerThread {
   constructor({parentPort, workerData}) {
     const {debug, directory, workerCount} = digs(workerData, "debug", "directory", "workerCount")
 
+    this.application = new Application({debug, directory})
+    this.databasePool = DatabasePool.current()
     this.clients = {}
-    this.configuration = new Configuration({debug, directory})
+    this.configuration = this.application.configuration
     this.parentPort = parentPort
     this.workerCount = workerCount
 
-    parentPort.on("message", errorLogger((data) => this.onCommand(data)))
+    parentPort.on("message", errorLogger(this.onCommand))
 
     logger(this, `Worker ${workerCount} started`)
 
-    parentPort.postMessage({command: "started"})
+    this.application.initialize().then(() => {
+      this.databasePool.connect().then(() => {
+        parentPort.postMessage({command: "started"})
+      })
+    })
   }
 
-  onCommand(data) {
+  onCommand = (data) => {
     logger(this, `Worker ${this.workerCount} received command`, data)
 
     const {command} = data

package/src/{logger.cjs → logger.mjs}

@@ -1,6 +1,6 @@
-const {digg} = require("diggerize")
+import {digg} from "diggerize"
 
-module.exports = function log(object, ...messages) {
+export default function log(object, ...messages) {
   if (!object.configuration) console.error(`No configuration on ${object.constructor.name}`)
 
   if (object.configuration?.debug) {

package/src/routes/base-route.mjs

@@ -0,0 +1,34 @@
+import GetRoute from "./get-route.mjs"
+import ResourceRoute from "./resource-route.mjs"
+
+var VelociousBaseRoute
+
+export function initBaseRoute() {
+  if (VelociousBaseRoute) return
+
+  VelociousBaseRoute = class VelociousBaseRoute {
+    routes = []
+
+    get(name, args) {
+      const route = new GetRoute({name, args})
+
+      this.routes.push(route)
+    }
+
+    matchWithPath(_path) {
+      throw new Error(`No 'matchWithPath' implemented on ${this.constructor.name}`)
+    }
+
+    resources(name, callback) {
+      const route = new ResourceRoute({name})
+
+      this.routes.push(route)
+
+      if (callback) {
+        callback(route)
+      }
+    }
+  }
+}
+
+export {VelociousBaseRoute as default}

package/src/routes/{get-route.cjs → get-route.mjs}

@@ -1,7 +1,9 @@
-const BaseRoute = require("./base-route.cjs")
-const escapeStringRegexp = require("escape-string-regexp")
+import BaseRoute, {initBaseRoute} from "./base-route.mjs"
+import escapeStringRegexp from "escape-string-regexp"
 
-module.exports = class VelociousRouteGetRoute extends BaseRoute {
+initBaseRoute()
+
+export default class VelociousRouteGetRoute extends BaseRoute {
   constructor({name}) {
     super()
     this.name = name