spiceflow 0.0.1

package/src/browser.ts

@@ -0,0 +1,142 @@
+import { JsonRpcRequest, JsonRpcResponse } from './jsonRpc';
+import { EventSourceParserStream } from 'eventsource-parser/stream';
+import { generate } from 'fast-glob/out/managers/tasks';
+
+import superjson from 'superjson';
+
+type NextRpcCall = (...params: any[]) => any | AsyncGenerator<any>;
+
+let nextId = 1;
+
+export function createRpcFetcher({
+  url,
+  method,
+  isGenerator,
+}: {
+  url: string;
+  method: string;
+  isGenerator?: boolean;
+}): NextRpcCall {
+  const controller = new AbortController();
+  if (isGenerator) {
+    const generator = async function* rpcFetchGenerator(...args) {
+      const { json, meta } = superjson.serialize(args);
+      const res = await fetch(url, {
+        method: 'POST',
+        headers: {
+          Accept: 'text/event-stream',
+          'Content-Type': 'application/json',
+        },
+        signal: controller.signal,
+        body: JSON.stringify({
+          jsonrpc: '2.0',
+          id: nextId++,
+          method,
+          params: json as any[],
+          meta,
+        } satisfies JsonRpcRequest),
+      });
+      if (res.status >= 400) {
+        const json = await res.json();
+        await handleJsonrpcError({ status: res.status, json });
+      }
+
+      if (!res.body) {
+        throw new Error('No response body for generator action');
+      }
+      const eventStream = res.body
+        .pipeThrough(new TextDecoderStream())
+        .pipeThrough(new EventSourceParserStream())
+        .getReader();
+      let isClosed = false;
+      try {
+        while (true) {
+          const { value: event, done } = await eventStream.read();
+
+          if (done) {
+            isClosed = true;
+            break;
+          }
+          if (!event) continue;
+
+          if (event.data === '[DONE]') {
+            continue;
+          }
+
+          const json = JSON.parse(event.data);
+
+          const { jsonrpc, id, result, meta, error } = json as JsonRpcResponse;
+          await handleJsonrpcError({ status: res.status, json });
+          const deserialized = superjson.deserialize({
+            json: result,
+            meta,
+          });
+          yield deserialized;
+        }
+      } finally {
+        // if user calls return() in the generator, we need to close the stream
+
+        if (!isClosed) {
+          // if stream is still open, abort controller
+          controller.abort();
+          eventStream.cancel();
+        }
+      }
+    };
+
+    return generator;
+  }
+  async function rpcFetch(...args) {
+    const { json: argsJson, meta } = superjson.serialize(args);
+    const res = await fetch(url, {
+      method: 'POST',
+      body: JSON.stringify(
+        {
+          jsonrpc: '2.0',
+          id: nextId++,
+          method,
+          params: argsJson as any[],
+          meta,
+        } satisfies JsonRpcRequest,
+        null,
+        2,
+      ),
+      headers: {
+        'content-type': 'application/json',
+      },
+    });
+    const json = await res.json();
+    await handleJsonrpcError({ status: res.status, json });
+    {
+      const deserialized = superjson.deserialize({
+        json: json.result,
+        meta: json.meta,
+      });
+      return deserialized as any;
+    }
+  }
+  rpcFetch.abort = function abort() {
+    controller.abort();
+  };
+  return rpcFetch;
+}
+
+async function handleJsonrpcError({
+  status,
+  json,
+}: {
+  status: number;
+  json: JsonRpcResponse;
+}) {
+  if (status >= 400) {
+    const statusError = new Error('Unexpected HTTP status ' + status);
+
+    if (json?.error && typeof json.error?.message === 'string') {
+      let err = new Error(json.error.message);
+      Object.assign(err, json.error.data || {});
+      throw err;
+    }
+
+    throw statusError;
+  }
+}

package/src/build.ts

@@ -0,0 +1,303 @@
+import { red } from 'picocolors';
+import fsx from 'fs-extra';
+
+import {
+  ConsoleMessageId,
+  Extractor,
+  ExtractorConfig,
+  ExtractorLogLevel,
+} from '@microsoft/api-extractor';
+import chokidar from 'chokidar';
+import { getPackages } from '@manypkg/get-packages';
+
+import { transform } from '@babel/core';
+import { exec } from 'child_process';
+import { globSync } from 'fast-glob';
+import fs from 'fs';
+import path from 'path';
+import { plugins } from '.';
+import { directive, serverEntryName } from './utils';
+
+export async function buildOnce({ rootDir, url }) {
+  console.log();
+  console.log('building functions');
+  if (url && !url.endsWith('/')) {
+    // make sure that new URL uses the last portion of the path too
+    url += '/';
+  }
+  try {
+    new URL(url);
+  } catch (e) {
+    throw new Error(`Invalid url ${url}`);
+  }
+
+  let libOutDir = path.resolve('dist');
+  let serverOutDir = path.resolve('server');
+  await fs.promises.rm(libOutDir, { recursive: true }).catch(() => null);
+
+  const cwd = process.cwd();
+  const serverEntrypoint = path.resolve(rootDir, serverEntryName + '.ts');
+
+  try {
+    const globBase = path.relative(cwd, rootDir);
+    const globs = [path.posix.join(globBase, '**/*.{ts,tsx,js,jsx}')];
+    // console.log({ globs });
+    const allPossibleFiles = globSync(globs, {
+      onlyFiles: true,
+      absolute: true,
+    });
+    const actionFilesRelativePaths = allPossibleFiles
+      .filter((file) => {
+        const content = fs.readFileSync(file, 'utf8');
+        return content.includes(directive);
+      })
+      .map((x) => {
+        return path.relative(rootDir, x);
+      });
+    const importsCode = actionFilesRelativePaths
+      .map((filePath) => {
+        filePath = removeExtension(filePath);
+        return `${JSON.stringify(
+          '/' + filePath,
+        )}: () => import('./${filePath}.js')`;
+      })
+      .join(',');
+    const serverExposeContent =
+      `// this file was generated\n` +
+      `import { internalEdgeHandler, internalNodeJsHandler } from 'spiceflow/dist/server.js';\n` +
+      `export const methodsMap = {${importsCode}}\n` +
+      `export const edgeHandler = internalEdgeHandler({ methodsMap });\n` +
+      `export const nodeJsHandler = internalNodeJsHandler({ methodsMap });\n`;
+
+    if (!actionFilesRelativePaths.length) {
+      throw new Error('No functions files found!');
+    }
+    fs.writeFileSync(serverEntrypoint, serverExposeContent, 'utf8');
+
+    await Promise.all([
+      runCommand(
+        `tsc --incremental --declaration --noEmit false --outDir ${libOutDir} `,
+      ),
+    ]).catch((error) => {
+      // console.error(error);
+      console.error('Error running tsc, continue anyway');
+    });
+
+    // copy tsc output to server dir
+    await fsx.copy(libOutDir, serverOutDir, { overwrite: true });
+    // rename serverEntryName to index
+    const tscOutFiles = fs.readdirSync(serverOutDir);
+    for (const file of tscOutFiles) {
+      if (file.startsWith(serverEntryName)) {
+        const remaining = file.slice(serverEntryName.length);
+        fs.renameSync(
+          path.resolve(serverOutDir, file),
+          path.resolve(serverOutDir, 'index' + remaining),
+        );
+      }
+    }
+
+    const imports = [] as string[];
+
+    for (let actionFile of actionFilesRelativePaths) {
+      const abs = path.resolve(rootDir, actionFile);
+      const content = fs.readFileSync(abs, 'utf8');
+
+      const actionName = path.basename(actionFile, path.extname(actionFile));
+
+      const res = transform(content || '', {
+        babelrc: false,
+        sourceType: 'module',
+        plugins: plugins({
+          isServer: false,
+          url,
+          rootDir,
+        }),
+        filename: abs,
+
+        sourceMaps: false,
+      });
+      if (!res || !res.code) {
+        console.error(
+          `Error transforming ${actionFile}, returned nothing, maybe not an action?`,
+        );
+        continue;
+      }
+
+      const importPath =
+        './' +
+        path.posix.join(path.posix.dirname(actionFile), actionName + '.js');
+      console.log(`processed ${importPath}`);
+      imports.push(importPath);
+      fs.mkdirSync(path.resolve(libOutDir, path.dirname(importPath)), {
+        recursive: true,
+      });
+      fs.writeFileSync(path.resolve(libOutDir, importPath), res.code, 'utf-8');
+    }
+
+    // const generator = createGenerator({
+    //   path: dtsOutputFilePath,
+    //   type: '*',
+    //   tsconfig: 'tsconfig.json',
+    //   skipTypeCheck: true,
+    //   functions: 'comment',
+    // });
+    // const schema = generator.createSchema();
+    // fs.writeFileSync(
+    //   path.resolve(outDir, 'schema.json'),
+    //   JSON.stringify(schema, null, 2),
+    // );
+
+    const bundledPackages = (await getPackages(process.cwd())).packages.map(
+      (x) => x.packageJson.name,
+    );
+    // TODO devDependencies should be bundled too, given these are not shipped with the SDK
+    if (!bundledPackages.length) {
+      console.log('no workspace packages found, skipping types bundling');
+      return;
+    }
+    for (const actionFile of actionFilesRelativePaths) {
+      const entryPointDts = path.resolve(
+        libOutDir,
+        // path.relative(process.cwd(), rootDir),
+        path.dirname(actionFile),
+        path.basename(actionFile, path.extname(actionFile)) + '.d.ts',
+      );
+      console.log(`bundling types for ${path.relative(cwd, entryPointDts)}`);
+
+      rollupDtsFile({
+        bundledPackages,
+        inputFilePath: entryPointDts,
+        outputFilePath: entryPointDts,
+        tsconfigFilePath: 'tsconfig.json',
+      });
+    }
+  } finally {
+    await fs.promises.unlink(serverEntrypoint).catch(() => null);
+  }
+}
+const logger = console;
+
+let isBuilding = { ref: false };
+let missedWatch = { ref: false };
+
+export async function build({ rootDir, url, watch = false }) {
+  await buildOnce({ rootDir, url });
+  if (!watch) {
+    return;
+  }
+  const watcher = chokidar.watch(rootDir, {
+    // ignored: /(^|[\/\\])\../, // ignore dotfiles
+    ignored: ['**/node_modules/**', '**/dist/**', `src/${serverEntryName}.ts`],
+    persistent: true,
+  });
+  console.log('watching for changes');
+  watcher.on('change', async (path, stats) => {
+    if (isBuilding.ref) {
+      missedWatch.ref = true;
+      return;
+    }
+    isBuilding.ref = true;
+    try {
+      logger.log(`detected change in ${path}`);
+      await buildOnce({ rootDir, url });
+      if (missedWatch.ref) {
+        // logger.log('missed a change, rebuilding');
+        await buildOnce({ rootDir, url });
+        missedWatch.ref = false;
+      }
+    } finally {
+      isBuilding.ref = false;
+    }
+  });
+}
+
+function rollupDtsFile({
+  inputFilePath,
+  outputFilePath,
+  tsconfigFilePath,
+  bundledPackages,
+}: {
+  inputFilePath: string;
+  outputFilePath: string;
+  tsconfigFilePath: string;
+  bundledPackages: string[];
+}) {
+  let cwd = process.cwd();
+  if (!fs.existsSync(tsconfigFilePath)) {
+    throw new Error(`tsconfig.json not found at ${tsconfigFilePath}`);
+  }
+
+  let packageJsonFullPath = path.join(cwd, 'package.json');
+
+  const extractorConfig = ExtractorConfig.prepare({
+    configObject: {
+      mainEntryPointFilePath: inputFilePath,
+      bundledPackages,
+      apiReport: {
+        enabled: false,
+
+        // `reportFileName` is not been used. It's just to fit the requirement of API Extractor.
+        reportFileName: 'report.html',
+      },
+      docModel: { apiJsonFilePath: 'api.json', enabled: false },
+      dtsRollup: {
+        enabled: true,
+        untrimmedFilePath: outputFilePath,
+      },
+      tsdocMetadata: { enabled: false, tsdocMetadataFilePath: 'another.json' },
+      compiler: {
+        tsconfigFilePath: tsconfigFilePath,
+      },
+
+      projectFolder: cwd,
+    },
+    configObjectFullPath: undefined,
+    packageJsonFullPath,
+  });
+
+  // Invoke API Extractor
+  const extractorResult = Extractor.invoke(extractorConfig, {
+    // Equivalent to the "--local" command-line parameter
+    localBuild: true,
+
+    messageCallback: (message) => {
+      switch (message.messageId) {
+        case ConsoleMessageId.ApiReportCreated:
+          message.logLevel = ExtractorLogLevel.None;
+          break;
+        case ConsoleMessageId.Preamble:
+          message.logLevel = ExtractorLogLevel.None;
+          break;
+      }
+    },
+    showDiagnostics: false,
+    // Equivalent to the "--verbose" command-line parameter
+    showVerboseMessages: false,
+  });
+
+  if (!extractorResult.succeeded) {
+    throw new Error(
+      `API Extractor completed with ${extractorResult.errorCount} errors and ${extractorResult.warningCount} warnings when processing ${inputFilePath}`,
+    );
+  }
+}
+
+function removeExtension(filePath: string) {
+  return filePath.replace(/\.[j|t]sx?$/, '');
+}
+
+function runCommand(command: string) {
+  return new Promise((resolve, reject) => {
+    exec(command, {}, (error, stdout, stderr) => {
+      if (error) {
+        console.log();
+        console.error(red(stdout));
+        console.error(red(stderr));
+        reject(error);
+      } else {
+        resolve(stdout);
+      }
+    });
+  });
+}

package/src/cli.ts

@@ -0,0 +1,118 @@
+#! /usr/bin/env node
+import os from 'os';
+import fsx from 'fs-extra';
+
+import fs from 'fs-extra';
+
+import { cac } from 'cac';
+import { build, buildOnce } from './build.js';
+import { findRootDir } from './index.js';
+import { exec, execSync, spawn } from 'child_process';
+import path from 'path';
+
+export const cli = cac();
+
+cli
+  .command('', 'Generate an SDK package for your functions')
+  .option('--watch', 'Watch for changes')
+  .option('--url <url>', 'URL of the package, including the base path', {
+    default: 'http://localhost:3333',
+  })
+  .action(async (options) => {
+    const { url, watch } = options;
+    const rootDir = await findRootDir(process.cwd());
+    await build({ rootDir, url, watch });
+  });
+cli
+  .command('init', 'Generate a new spiceflow project')
+  .option('--name <name>', 'Name of this project')
+  .action(async (options) => {
+    // copy contents of the template dir here
+    const { name } = options;
+    if (!name) {
+      throw new Error('--name is required');
+    }
+    fsx.copySync(path.resolve(__dirname, '../sdk-template'), name, {
+      filter(pathname) {
+        return (
+          !pathname.includes('node_modules') &&
+          !pathname.includes('dist') &&
+          !pathname.endsWith('.tsbuildinfo')
+        );
+      },
+    });
+    // replace the package.json name
+    const packageJsonPath = path.resolve(name, 'package.json');
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+    packageJson.name = name;
+    delete packageJson.private;
+    // replace workspace:* with * from the package.json
+    for (const key of Object.keys(packageJson.dependencies || {})) {
+      const value = packageJson.dependencies[key];
+      if (value && value.startsWith('workspace:')) {
+        packageJson.dependencies[key] = '*';
+      }
+    }
+
+    fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
+  });
+cli
+  .command('serve', 'Expose a server for your functions')
+  .option('--basePath', 'base path for the server', { default: '/' })
+  .option('--port <port>', 'Port to listen on', { default: '3333' })
+  .option('--watch', 'Watch for changes')
+  .option(
+    '--skip-build',
+    'Skip building the server and SDK, you must build it yourself first',
+  )
+  .action(async (options) => {
+    let { basePath, skipBuild, watch, port } = options;
+    const nodePath = process.execPath || 'node';
+    const rootDir = await findRootDir(process.cwd());
+    if (!skipBuild) {
+      await build({ rootDir, watch, url: `http://127.0.0.1:${port}` });
+    }
+
+    const tempFilePath = path.resolve('_main.mjs');
+
+    const code = `import { methodsMap } from './server/index.js'; import { exposeNodeServer } from 'spiceflow/dist/expose.js'; exposeNodeServer({ methodsMap, basePath: '${basePath}', port: ${port} });`;
+    fs.writeFileSync(tempFilePath, code);
+    process.on('SIGINT', () => {
+      try {
+        fs.unlinkSync(tempFilePath);
+      } catch {}
+      process.exit(0);
+    });
+    // only enable watch if it's supported by node version
+    const major = parseInt(process.version.replace('v', '').split('.')[0]);
+    if (major && major < 18) {
+      console.log(`node version ${process.version} does not support --watch`);
+      watch = false;
+    }
+    try {
+      await new Promise<void>((resolve, reject) => {
+        const p = spawn(
+          `${nodePath} --no-warnings ${watch ? '--watch' : ''} ${JSON.stringify(
+            tempFilePath,
+          )}`,
+          {
+            stdio: 'inherit',
+            shell: true,
+          },
+        );
+        p.on('close', (code) => {
+          if (code === 0) {
+            resolve();
+          } else {
+            reject();
+          }
+        });
+      });
+    } finally {
+      fs.unlinkSync(tempFilePath);
+    }
+  });
+
+cli.help();
+
+cli.parse();

package/src/context-internal.ts

@@ -0,0 +1,36 @@
+import { AsyncLocalStorage } from 'async_hooks';
+import type { IncomingMessage, ServerResponse } from 'http';
+
+interface CommonContext {
+  // cookies(): ReadonlyRequestCookies;
+  // headers(): ReadonlyHeaders;
+}
+interface NodejsContext extends CommonContext {
+  req?: IncomingMessage;
+  res?: ServerResponse;
+}
+interface EdgeContext extends CommonContext {
+  req?: Request;
+  res?: Response;
+}
+
+const DEFAULT_CONTEXT = {
+  // headers() {},
+  // cookies() {},
+};
+
+export const asyncLocalStorage = new AsyncLocalStorage<
+  NodejsContext | EdgeContext
+>();
+
+export function getNodejsContext(): NodejsContext {
+  return (asyncLocalStorage.getStore() as NodejsContext) || DEFAULT_CONTEXT;
+}
+
+export function getEdgeContext(): EdgeContext {
+  return (asyncLocalStorage.getStore() as EdgeContext) || DEFAULT_CONTEXT;
+}
+
+export function getContext(): CommonContext {
+  return asyncLocalStorage.getStore() || DEFAULT_CONTEXT;
+}

package/src/context.ts

@@ -0,0 +1 @@
+export { getNodejsContext, getEdgeContext, getContext } from './context-internal';

package/src/expose.ts

@@ -0,0 +1,34 @@
+import http from 'http';
+import { internalNodeJsHandler } from './server';
+
+export async function exposeNodeServer({ methodsMap, basePath, port }) {
+  const handler = internalNodeJsHandler({ methodsMap });
+
+  const server = http.createServer(async (req, res) => {
+    let ended = false;
+    res.on('close', () => {
+      ended = true;
+    });
+
+    try {
+      if (req.method === 'GET' && req.url === '/') {
+        res.statusCode = 200;
+        res.end('ok');
+        return;
+      }
+      console.log(`[spiceflow] ${req.url}`);
+      await handler({ req, res, basePath });
+    } catch (error) {
+      if (ended) {
+        return;
+      }
+      console.error(error);
+
+      res.statusCode = 500;
+      res.end('Internal server error');
+    }
+  });
+  server.listen(port, () => {
+    console.log(`server listening on http://127.0.0.1:${port}`);
+  });
+}

package/src/headers.ts

@@ -0,0 +1,19 @@
+// import { asyncLocalStorage } from './context-internal';
+
+// function myHeaders() {
+//   const res = asyncLocalStorage.getStore()?.headers();
+//   if (!res) {
+//     throw new Error('Called headers() outside of app dir or rpc actions');
+//   }
+//   return res;
+// }
+
+// function myCookies() {
+//   const res = asyncLocalStorage.getStore()?.cookies();
+//   if (!res) {
+//     throw new Error('Called cookies() outside of app dir or rpc actions');
+//   }
+//   return res;
+// }
+
+// export { myCookies as cookies, myHeaders as headers };

package/src/index.ts

@@ -0,0 +1,42 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { PluginOptions } from './babelTransformRpc';
+
+import { WrapMethod } from './server';
+
+export interface WithRpcConfig {}
+
+export { WrapMethod };
+
+export function plugins({
+  isServer,
+  rootDir: nextDir,
+  url,
+}: PluginOptions) {
+  const rpcPluginOptions: PluginOptions = {
+    isServer,
+    rootDir: nextDir,
+    url,
+  };
+
+  return [
+    require.resolve('@babel/plugin-syntax-jsx'),
+    [require.resolve('@babel/plugin-transform-typescript'), { isTSX: true }],
+    [require.resolve('../dist/babelTransformRpc'), rpcPluginOptions],
+    process.env.DEBUG_ACTIONS && [
+      require.resolve('../dist/babelDebugOutputs'),
+      rpcPluginOptions,
+    ],
+  ].filter(Boolean) as any[];
+}
+
+export function findRootDir(dir: string): string {
+  {
+    let curDir = path.resolve(dir, 'src');
+    if (fs.existsSync(curDir)) return path.resolve(curDir);
+  }
+
+  throw new Error(
+    "Couldn't find a src directory. Please create one under the project root",
+  );
+}