repl-sdk 0.0.0 → 1.0.0

package/src/resolve.js

@@ -0,0 +1,352 @@
+import { exports as resolveExports } from 'resolve.exports';
+import { resolve as resolveImports } from 'resolve.imports';
+
+/**
+ * @typedef {import('./request.js').Request} Request
+ */
+import { assert, fakeDomain } from './utils.js';
+
+/**
+ * If a package wanted, they could provide a special export condition
+ * targeting REPLs.
+ *
+ * This format should still be ESM.
+ * CJS is not supported in browsers, and I won't support CJS in this REPL.
+ */
+const CONDITIONS = ['repl', 'module', 'browser', 'import', 'default', 'development'];
+
+/**
+ * @type {Map<string, import('./types.ts').RequestAnswer>} specifier => filePath in the tgz
+ */
+const resolveCache = new Map();
+
+const AT = '___AT___';
+const fakeProtocol = 'repl://';
+
+/**
+ *
+ * @param {*} start packageName or packageName with file
+ * @param {*} target file to resolve within the packageName
+ * @returns
+ */
+export function resolvePath(start, target) {
+  /**
+   * How to make the whole package name look like one segment for URL
+   */
+  const base = start.replace(/^@([^/]+)\/([^/]+)/, `${AT}$1___$2`);
+
+  const url = new URL(target, fakeProtocol + fakeDomain + base);
+
+  /**
+   * href omits the protocol
+   * (which is what we want)
+   */
+  return url.href
+    .replace(fakeProtocol + fakeDomain, '')
+    .replace(AT, '@')
+    .replace('___', '/')
+    .replace(/^\//, './');
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+export function resolve(untarred, request) {
+  let answer = undefined;
+
+  const key = request.key;
+
+  if (resolveCache.has(key)) {
+    return resolveCache.get(key);
+  }
+
+  answer ||= fromImports(untarred, request, answer);
+  answer ||= fromInternalImport(untarred, request, answer);
+  answer ||= fromExportsString(untarred, request, answer);
+  answer ||= fromExports(untarred, request, answer);
+  answer ||= fromModule(untarred, request, answer);
+  answer ||= fromBrowser(untarred, request, answer);
+  answer ||= fromMain(untarred, request, answer);
+  answer ||= fromIndex(untarred, request, answer);
+  answer ||= fromFallback(untarred, request, answer);
+
+  if (answer) {
+    resolveCache.set(key, answer);
+  }
+
+  return answer;
+}
+
+/**
+ * These are likely all private imports
+ *
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+export function fromInternalImport(untarred, request, answer) {
+  if (answer) return answer;
+
+  if (!request.from) return answer;
+
+  const fromSpecifier = request.from;
+  const answerFrom = resolve(untarred, fromSpecifier);
+
+  if (!answerFrom) {
+    printError(untarred, fromSpecifier, answer);
+
+    return;
+  }
+
+  const inTarFile = resolvePath(
+    fromSpecifier.name + '/' + answerFrom.inTarFile,
+    request.to
+  ).replace(new RegExp(`^${fromSpecifier.name}/`), '');
+  const result = checkFile(untarred, inTarFile);
+
+  if (result) {
+    return createAnswer(result, request, 'internalImport');
+  }
+
+  // Internal imports should always exist, unless a package is just broken.
+  printError(untarred, request, answer);
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromExports(untarred, request, answer) {
+  if (answer) return answer;
+
+  const exports = untarred.manifest.exports;
+
+  if (!(typeof exports === 'object')) return answer;
+
+  const foundArray = resolveExports(untarred.manifest, request.to, {
+    conditions: CONDITIONS,
+  });
+
+  const found = foundArray?.map((f) => checkFile(untarred, f)).find(Boolean);
+
+  if (found) {
+    return createAnswer(found, request, 'exports');
+  }
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+export function fromImports(untarred, request, answer) {
+  if (answer) return answer;
+  if (!request.to.startsWith('#')) return answer;
+
+  const imports = untarred.manifest.imports;
+
+  if (!(typeof imports === 'object')) return answer;
+
+  const found = resolveImports({ content: untarred.manifest }, request.to, {
+    conditions: CONDITIONS,
+  });
+
+  if (found) {
+    return createAnswer(found.replace(/^\.\//, ''), request, 'imports');
+  }
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromExportsString(untarred, request, answer) {
+  if (answer) return answer;
+  if (!hasExports(untarred)) return answer;
+
+  // technically not legacy, but it's the same logic
+  return checkLegacyEntry(untarred, request, 'exports');
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromModule(untarred, request, answer) {
+  if (answer) return answer;
+  if (hasExports(untarred)) return answer;
+
+  return checkLegacyEntry(untarred, request, 'module');
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromBrowser(untarred, request, answer) {
+  if (answer) return answer;
+  if (hasExports(untarred)) return answer;
+
+  return checkLegacyEntry(untarred, request, 'browser');
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromMain(untarred, request, answer) {
+  if (answer) return answer;
+  if (hasExports(untarred)) return answer;
+
+  return checkLegacyEntry(untarred, request, 'main');
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {string} entryName
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function checkLegacyEntry(untarred, request, entryName) {
+  if (request.to !== '.') return;
+
+  const filePath = untarred.manifest[/** @type {keyof typeof untarred.manifest} */ (entryName)];
+
+  if (!filePath || typeof filePath !== 'string') return;
+
+  const result = checkFile(untarred, filePath);
+
+  if (result) {
+    return createAnswer(result, request, entryName);
+  }
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromIndex(untarred, request, answer) {
+  if (answer) return answer;
+  if (hasExports(untarred)) return answer;
+
+  if (request.to === '.') {
+    if (untarred.contents['index.js']) {
+      return {
+        inTarFile: 'index.js',
+        ext: 'js',
+        from: 'index',
+      };
+    }
+  }
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | import('./types.ts').RequestAnswer} the in-tar path
+ */
+function fromFallback(untarred, request, answer) {
+  if (answer) return answer;
+
+  const result = checkFile(untarred, request.to);
+
+  if (result) {
+    return createAnswer(result, request, 'fallback');
+  }
+}
+
+/**
+ *
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {string | undefined} filePath
+ * @returns {string | undefined} the variant
+ */
+function checkFile(untarred, filePath) {
+  if (!filePath) return;
+
+  for (const prefix of ['', 'pkg/']) {
+    const path = prefix + filePath;
+    const dotless = prefix + filePath.replace(/^\.\//, '');
+
+    if (untarred.contents[path]) {
+      return path;
+    }
+
+    if (untarred.contents[dotless]) {
+      return dotless;
+    }
+  }
+}
+
+/**
+ * @param {string} filePath
+ */
+function extName(filePath) {
+  return filePath.split('.').pop();
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ */
+function hasExports(untarred) {
+  return Boolean(untarred.manifest.exports);
+}
+
+/**
+ * @param {string} forFile
+ * @param {Request} request
+ * @param {string} fromMethod
+ */
+function createAnswer(forFile, request, fromMethod) {
+  const ext = extName(forFile);
+
+  assert(
+    `All files must have an extension. This file (in ${request.name}) did not have an extension: ${forFile}`,
+    ext
+  );
+
+  return {
+    inTarFile: forFile,
+    ext,
+    from: fromMethod,
+  };
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {Request} request
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @throws {Error}
+ */
+export function printError(untarred, request, answer) {
+  const { name, exports, main, module, browser } = untarred.manifest;
+
+  console.group(`${name} file info`);
+  console.info(`${name} has these files: `, Object.keys(untarred.contents));
+  console.info(`We searched for '${request.original}'`);
+  console.info(`from: `, { exports, main, module, browser });
+  console.info(`And found: `, answer);
+  console.info(`The request was: `, request);
+  console.groupEnd();
+
+  // eslint-disable-next-line no-debugger
+  debugger;
+  throw new Error(`Could not find file for ${request.original}`);
+}

package/src/resolve.resolvePath.test.ts

@@ -0,0 +1,24 @@
+import { expect as errorExpect, it } from 'vitest';
+
+const expect = errorExpect.soft;
+
+import { resolvePath } from './resolve.js';
+
+function test({ from, to, expected }: { from: string; to: string; expected: string }) {
+  it(`import ${to} from ${from}`, () => {
+    expect(resolvePath(from, to)).deep.equal(expected);
+  });
+}
+
+test({ from: '@scope/pkgName', to: 'index.js', expected: '@scope/pkgName/index.js' });
+test({ from: '@scope/pkgName/index.js', to: './foo.js', expected: '@scope/pkgName/foo.js' });
+test({
+  from: '@scope/pkgName/folder/index.js',
+  to: './foo.js',
+  expected: '@scope/pkgName/folder/foo.js',
+});
+test({
+  from: '@scope/pkgName/folder/index.js',
+  to: '../foo.js',
+  expected: '@scope/pkgName/foo.js',
+});

package/src/resolve.test.ts

@@ -0,0 +1,23 @@
+import { expect as errorExpect, it } from 'vitest';
+
+import { Request } from './request.js';
+import { resolve } from './resolve.js';
+
+import type { UntarredPackage } from './types.js';
+
+const expect = errorExpect.soft;
+
+it('resolves the entrypoint (rehype-raw)', () => {
+  const untarred = {
+    contents: {
+      'index.js': 'entry file',
+    },
+    manifest: {
+      exports: './index.js',
+    },
+  };
+  const request = Request.fromSpecifier('rehype-raw');
+  const answer = resolve(untarred as unknown as UntarredPackage, request);
+
+  expect(answer?.inTarFile).toBe('index.js');
+});

package/src/specifier.js

@@ -0,0 +1,71 @@
+/**
+ * Parses
+ *   @scope/pkgName
+ *   pkgName
+ *   @scope/pkgName@version
+ *   pkgName@version
+ *   @scope/pkgName@version/path
+ *   pkgName@version/path
+ *
+ * @param {string} specifier
+ * @returns {{ name: string, version: string | undefined, path: string }}
+ */
+export function parseSpecifier(specifier) {
+  let name = '';
+  let version = '';
+  let path = '.';
+
+  const chars = specifier.split('');
+
+  const hasScope = chars[0] === '@';
+
+  let isVersion = false;
+  let isPath = false;
+  let finishedName = false;
+  let slashCount = 0;
+
+  for (let i = 0; i < chars.length; i++) {
+    const char = chars[i];
+
+    // Enter version mode
+    // Goes until the end, or a slash
+    if (char === '@' && i > 0) {
+      finishedName = true;
+      isVersion = true;
+      continue;
+    } else if (char === '/') {
+      slashCount++;
+
+      if (hasScope) {
+        if (slashCount > 1) {
+          finishedName = true;
+        }
+      } else {
+        finishedName = true;
+      }
+
+      if (finishedName) {
+        isPath = true;
+      }
+    }
+
+    if (isVersion && char === '/') {
+      isVersion = false;
+      isPath = true;
+    }
+
+    if (isVersion) {
+      version += char;
+    } else if (isPath) {
+      path += char;
+    } else {
+      name += char;
+    }
+  }
+
+  return {
+    name,
+    version: version || undefined,
+    path,
+  };
+}

package/src/specifier.test.ts

@@ -0,0 +1,90 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+
+import { expect as errorExpect, it } from 'vitest';
+
+import { parseSpecifier } from './specifier.js';
+
+const expect = errorExpect.soft;
+
+function test(input: string, expected: { name: string; version?: string; path: string }) {
+  it(input, () => {
+    expect(parseSpecifier(input)).deep.equal(expected);
+  });
+}
+
+test('pkgName', {
+  name: 'pkgName',
+  version: undefined,
+  path: '.',
+});
+
+test('pkgName/foo', {
+  name: 'pkgName',
+  version: undefined,
+  path: './foo',
+});
+
+test('pkgName@beta', {
+  name: 'pkgName',
+  version: 'beta',
+  path: '.',
+});
+
+test('pkgName@beta/foo', {
+  name: 'pkgName',
+  version: 'beta',
+  path: './foo',
+});
+
+test('pkgName@5.4.3-beta.2', {
+  name: 'pkgName',
+  version: '5.4.3-beta.2',
+  path: '.',
+});
+
+test('pkgName@5.4.3', {
+  name: 'pkgName',
+  version: '5.4.3',
+  path: '.',
+});
+
+test('pkgName@5.4.3/foo', {
+  name: 'pkgName',
+  version: '5.4.3',
+  path: './foo',
+});
+
+test('pkgName@5.4.3/foo/bar', {
+  name: 'pkgName',
+  version: '5.4.3',
+  path: './foo/bar',
+});
+
+test('@scope/pkgName', {
+  name: '@scope/pkgName',
+  version: undefined,
+  path: '.',
+});
+
+test('@scope/pkgName/foo', {
+  name: '@scope/pkgName',
+  version: undefined,
+  path: './foo',
+});
+
+test('@scope/pkgName@1.0.0', {
+  name: '@scope/pkgName',
+  version: '1.0.0',
+  path: '.',
+});
+
+test('@scope/pkgName@1.0.0/foo', {
+  name: '@scope/pkgName',
+  version: '1.0.0',
+  path: './foo',
+});
+
+it('invalid', () => {
+  // @ts-expect-error
+  expect(() => parseSpecifier(undefined)).toThrow();
+});

package/src/tar-worker.js

@@ -0,0 +1,61 @@
+import { expose } from 'comlink';
+import { parseTar } from 'tarparser';
+
+import { cache } from './cache.js';
+import { getNPMInfo, getTarUrl } from './npm.js';
+
+const obj = { getTar };
+
+expose(obj);
+
+/**
+ * @param {string} name of the package
+ * @param {string} requestedVersion version or tag to fetch the package at
+ */
+async function getTar(name, requestedVersion) {
+  const key = `${name}@${requestedVersion}`;
+  const untarred = cache.tarballs.get(key);
+
+  if (untarred) {
+    return untarred;
+  }
+
+  const contents = await cache.cachedPromise(`getTar:${key}`, async () => {
+    const json = await getNPMInfo(name, requestedVersion);
+    const tgzUrl = await getTarUrl(json, requestedVersion);
+
+    const response = await fetch(tgzUrl, {
+      headers: {
+        ACCEPT: 'application/octet-stream',
+      },
+    });
+
+    return await untar(await response.arrayBuffer());
+  });
+
+  const manifest = JSON.parse(contents['package.json'].text);
+
+  const info = /** @type {import('./types.ts').UntarredPackage}*/ ({ manifest, contents });
+
+  cache.tarballs.set(key, info);
+
+  return info;
+}
+
+/**
+ * @param {ArrayBuffer} arrayBuffer
+ */
+async function untar(arrayBuffer) {
+  /**
+   * @type {{ [name: string]: import('tarparser').FileDescription }}
+   */
+  const contents = {};
+
+  for (const file of await parseTar(arrayBuffer)) {
+    if (file.type === 'file') {
+      contents[file.name.slice(8)] = file; // remove `package/` prefix
+    }
+  }
+
+  return contents;
+}

package/src/tar.js

@@ -0,0 +1,76 @@
+import { wrap } from 'comlink';
+
+import { cache } from './cache.js';
+import { Request } from './request.js';
+import { printError, resolve } from './resolve.js';
+import { assert, unzippedPrefix } from './utils.js';
+
+const worker = new Worker(new URL('./tar-worker.js', import.meta.url), {
+  name: 'Tar & NPM Downloader Worker',
+  type: 'module',
+});
+const com = wrap(worker);
+
+/**
+ * @param {string} url request URL
+ * @returns {Promise<undefined | { code: string, ext: string }>}
+ */
+export async function getFromTarball(url) {
+  const key = url.replace(unzippedPrefix + '/', '');
+  const request = cache.requestCache.get(key);
+
+  assert(`Missing request for ${url}`, request);
+
+  if (cache.fileCache.has(key)) {
+    return cache.fileCache.get(key);
+  }
+
+  const data = await cache.cachedPromise(key, async () => {
+    const untarred = await com.getTar(request.name, request.version);
+    const answer = resolve(untarred, request);
+
+    if (!answer) {
+      throw new Error(`Could not find file for ${request.original}`);
+    }
+
+    return { answer, name: request.name, version: request.version };
+  });
+
+  const untarred = await com.getTar(request.name, request.version);
+
+  const result = getFile(untarred, key, data.answer);
+
+  assert(`Missing file for ${url}`, result);
+
+  cache.fileCache.set(key, result);
+
+  return result;
+}
+
+/**
+ * @param {import('./types.ts').UntarredPackage} untarred
+ * @param {string} key
+ * @param {undefined | import('./types.ts').RequestAnswer} answer
+ * @returns {undefined | { code: string, ext: string }}
+ */
+function getFile(untarred, key, answer) {
+  const request = Request.fromRequestId(key);
+
+  if (!answer) {
+    printError(untarred, request, answer);
+
+    return;
+  }
+
+  const { inTarFile, ext } = answer;
+
+  const code = untarred.contents[inTarFile]?.text;
+
+  if (!code) {
+    printError(untarred, request, answer);
+
+    return;
+  }
+
+  return { code, ext };
+}