serene-core-server 0.1.1

package/src/services/api-usage/api-usage-base-service.ts

@@ -0,0 +1,95 @@
+import { PrismaClient } from '@/prisma/client'
+import { CustomError } from '../../types/errors'
+import { TechModel } from '../../models/tech/tech-model'
+import { RateLimitedApiModel } from '../../models/tech/rate-limited-api-model'
+import { RateLimitedApiEventModel } from '../../models/tech/rate-limited-api-event-model'
+
+export class ApiUsageBaseService {
+
+  // Consts
+  clName = 'ApiUsageBaseService'
+
+  // Models
+  rateLimitedApiEventModel = new RateLimitedApiEventModel()
+  rateLimitedApiModel = new RateLimitedApiModel()
+  techModel = new TechModel()
+
+  // Code
+  async isRateLimited(
+          prisma: PrismaClient,
+          variantName: string | undefined = undefined,
+          techId: string | undefined = undefined) {
+
+    // Debug
+    const fnName = `${this.clName}.isRateLimited()`
+
+    // Validate
+    if (techId == null &&
+        variantName == null) {
+
+      throw new CustomError(`Either a techId or variantName must be specified`)
+    }
+
+    // Get the techId for the Apollo.io API
+    if (techId == null &&
+        variantName != null) {
+
+      const tech = await
+              this.techModel.getByVariantName(
+                prisma,
+                variantName)
+
+      techId = tech.id
+    }
+
+    if (techId == null) {
+      throw new CustomError(`techId not found`)
+    }
+
+    // Get API rate limiting data
+    const rateLimitedApi = await
+            this.rateLimitedApiModel.getByTechId(
+              prisma,
+              techId)
+
+    // Return null if not a rate-limited tech
+    if (rateLimitedApi == null) {
+      return null
+    }
+
+    // Determine if rate-limited
+    const eventCount = await
+            this.rateLimitedApiEventModel.getUserSentForLastSpecifiedMinutes(
+              prisma,
+              1)  // For the last minute
+
+    // console.log(`${fnName}: is eventCount: ${eventCount} >= ` +
+    //   `rateLimitedApi.ratePerMinute: ${rateLimitedApi.ratePerMinute}`)
+
+    // Is the API currently rate limited?
+    var isRateLimited = false
+    var waitSeconds: number | null = 0
+
+    if (eventCount >= rateLimitedApi.ratePerMinute) {
+      isRateLimited = true
+
+      // Get last record within the window
+      waitSeconds = await
+        this.rateLimitedApiEventModel.getWaitSecondsSinceLastSpecifiedMinutes(
+          prisma,
+          1)  // For the last minute
+
+      if (waitSeconds == null ||
+          waitSeconds === 0) {
+
+        isRateLimited = false
+      }
+    }
+
+    return {
+      isRateLimited: isRateLimited,
+      waitSeconds: waitSeconds,
+      rateLimitedApiId: rateLimitedApi.id
+    }
+  }
+}

package/src/services/console/service.ts

@@ -0,0 +1,24 @@
+import chalk from 'chalk'
+import * as readline from 'node:readline/promises'
+import { stdin as input, stdout } from 'node:process'
+
+// Note: this class's console services can clash with other CLI libraries.
+
+class ConsoleService {
+  private rl = readline.createInterface({ input })
+
+  async askQuestion(query: string): Promise<string> {
+
+    // Reuse the class-level interface
+    stdout.write(chalk.bold.cyan(query))
+    const answer = await this.rl.question('')
+    return answer
+  }
+
+  close(): void {
+    this.rl.close()
+  }
+}
+
+// Export a single instance so that the interface is only created once
+export const consoleService = new ConsoleService()

package/src/services/db/pg/pg-estimate-size-service.ts

@@ -0,0 +1,119 @@
+export type PgRecordEstimateInput = {
+  numberFields?: number[]
+  booleanFields?: boolean[]
+  stringFields?: string[]
+  jsonFields?: any[]
+
+  bigStringValues?: any[]
+  bigJsonValues?: any[]
+}
+
+export class PgEstimateSizeService {
+
+  // Consts
+  clName = 'PgEstimateSizeService'
+
+  // Code
+  estimateRecordSize(
+    records: PgRecordEstimateInput[]
+  ): { totalInline: number; totalToast: number; total: number } {
+
+    // Debug
+    const fnName = `${this.clName}.estimateRecordSize()`
+
+    // console.log(`${fnName}: starting..`)
+
+    // Consts and initial vars
+    const TOAST_THRESHOLD = 2000  // bytes
+    const ROW_OVERHEAD = 24
+    const TOAST_POINTER_SIZE = 20
+
+    let totalInline = 0
+    let totalToast = 0
+
+    // Per record
+    for (const record of records) {
+
+      // Debug
+      // console.log(`${fnName}: record: ` + JSON.stringify(record))
+
+      // Start with initial rowSize
+      let rowSize = ROW_OVERHEAD
+
+      // Estimate primitive sizes
+      rowSize += (record.numberFields?.length ?? 0) * 8
+      rowSize += (record.booleanFields?.length ?? 0) * 1
+
+      if (record.stringFields) {
+        for (const str of record.stringFields) {
+          if (str != null) {
+            rowSize += Buffer.byteLength(str, 'utf8')
+          }
+        }
+      }
+
+      if (record.jsonFields) {
+        for (const json of record.jsonFields) {
+          if (json != null) {
+            rowSize += Buffer.byteLength(JSON.stringify(json), 'utf8')
+          }
+        }
+      }
+
+      // Estimate for bigStringValues
+      if (record.bigStringValues) {
+
+        for (const bigStringValue of record.bigStringValues) {
+
+          if (bigStringValue == null) {
+            continue
+          }
+
+          const strSize = Buffer.byteLength(bigStringValue, 'utf8')
+
+          if (strSize > TOAST_THRESHOLD) {
+            rowSize += TOAST_POINTER_SIZE
+            totalToast += strSize
+          } else {
+            rowSize += strSize
+          }
+
+          totalInline += rowSize
+        }
+      }
+
+      // Estimate for bigJsonValues
+      if (record.bigJsonValues) {
+
+        for (const bigJsonValue of record.bigJsonValues) {
+
+          if (bigJsonValue == null) {
+            continue
+          }
+
+          const jsonStr = JSON.stringify(bigJsonValue)
+          const jsonSize = Buffer.byteLength(jsonStr, 'utf8')
+
+          if (jsonSize > TOAST_THRESHOLD) {
+            rowSize += TOAST_POINTER_SIZE
+            totalToast += jsonSize
+          } else {
+            rowSize += jsonSize
+          }
+
+          totalInline += rowSize
+        }
+      }
+    }
+
+    // Debug
+    // console.log(`${fnName}: returning..`)
+
+    // Return
+    return {
+      totalInline,
+      totalToast,
+      total: totalInline + totalToast
+    }
+  }
+}

package/src/services/files/types.ts

@@ -0,0 +1,7 @@
+export interface WalkDirConfig {
+
+  recursive: boolean,
+  fileExts?: string[]
+  ignoreDirs?: Set<string>
+  ignoreFilePatterns?: RegExp[]
+}

package/src/services/files/walk-dir-service.ts

@@ -0,0 +1,135 @@
+const fs = require('fs')
+const path = require('path')
+import { WalkDirConfig } from './types'
+
+export class WalkDirService {
+
+  // Consts
+  clName = 'WalkDirService'
+
+  // Code
+  private normalizePath(p: string) {
+    return path.resolve(p)
+  }
+
+  private isPathIgnored(
+            fullPath: string,
+            root: string,
+            config: WalkDirConfig): boolean {
+
+    // Validate
+    if (config.ignoreDirs == null) {
+      return false
+    }
+
+    // Is path ignored?
+    const normalizedPath = this.normalizePath(fullPath)
+    const relative = path.relative(root, normalizedPath)
+    const segments = relative.split(path.sep)
+
+    return segments.some((s: string) => config.ignoreDirs?.has(s))
+  }
+
+  private isFileIgnored(
+            fullPath: string,
+            root: string,
+            config: WalkDirConfig): boolean {
+
+    // Validate
+    if (config.ignoreFilePatterns == null) {
+      return false
+    }
+
+    // Is path ignored?
+    if (this.isPathIgnored(
+          fullPath,
+          root,
+          config)) return true
+
+    const filename = path.basename(fullPath)
+
+    return config.ignoreFilePatterns.some((re) => re.test(filename))
+  }
+
+  async walkDir(
+          dir: string,
+          fileList: string[] = [],
+          config: WalkDirConfig,
+          root?: string): Promise<string[]> {
+
+    // Debug
+    const fnName = `${this.clName}.walkDir()`
+
+    // Init root
+    if (root == null) {
+      root = dir
+    }
+
+    // Ignore this path?
+    const normalizedDir = this.normalizePath(dir)
+
+    if (config.ignoreDirs != null) {
+
+      if (this.isPathIgnored(
+            normalizedDir,
+            root,
+            config)) return []
+    }
+
+    // Read the dir
+    const files = await
+            fs.promises.readdir(
+              dir,
+              { withFileTypes: true })
+
+    // Walk files
+    for (const file of files) {
+
+      // Get filePath
+      const filePath = path.join(dir, file.name)
+
+      // If a directory, call this function (if recursive)
+      if (file.isDirectory()) {
+
+        // Directories
+        if (config.recursive === true) {
+
+          await this.walkDir(
+                  filePath,
+                  fileList,
+                  config,
+                  root)
+
+          continue
+        }
+
+      }
+
+      if (config.ignoreFilePatterns != null &&
+          this.isFileIgnored(
+              filePath,
+              root,
+              config)) {
+
+        continue
+      }
+
+      // Files
+      // In fileExts list?
+      if (config.fileExts != null &&
+          !config.fileExts.includes(path.extname(filePath))) {
+
+        continue
+      }
+
+      // Add file to list
+      fileList.push(filePath)
+    }
+
+    // Debug
+    // console.log(`${fnName}: fileList: ${fileList.length}`)
+
+    // Return file list
+    return fileList
+  }
+}