vzcode 0.28.0 → 0.30.0

package/src/client/useTypeScript/worker.ts

@@ -0,0 +1,242 @@
+import * as tsvfs from '@typescript/vfs';
+import ts from 'typescript';
+import { File, Files, VZCodeContent } from '../../types';
+import {
+  AutocompleteRequest,
+  AutocompleteResponse,
+  LinterRequest,
+  LinterResponse,
+} from './requestTypes';
+
+let isFileSystemInitialized = false;
+
+let env: tsvfs.VirtualTypeScriptEnvironment = null;
+
+const debug = false;
+
+// replace .js or .jsx with .ts or .tsx,
+// to support TypeScript completions on non-TS files.
+const getTSFileName = (fileName: string) => {
+  if (fileName.endsWith('.js')) {
+    return fileName.replace('.js', '.ts');
+  }
+  if (fileName.endsWith('.jsx')) {
+    return fileName.replace('.jsx', '.tsx');
+  }
+  return fileName;
+};
+
+// Returns true if the file name ends with `.ts` or `.tsx`.
+const isTS = (fileName: string) => {
+  return (
+    fileName.endsWith('.ts') || fileName.endsWith('.tsx')
+  );
+};
+
+// This is a place for things we only do _once_.
+const initializeFileSystem = async () => {
+  if (debug) {
+    console.log('initializeFileSystem');
+  }
+  const compilerOptions: ts.CompilerOptions = {};
+
+  // `true` breaks in a Web Worker because
+  // this uses `localStorage` under the hood,
+  // which is not available in a Web Worker.
+  const cache = false;
+
+  const fsMap = await tsvfs.createDefaultMapFromCDN(
+    compilerOptions,
+    ts.version,
+    cache,
+    ts,
+  );
+  const sys: ts.System = tsvfs.createSystem(fsMap);
+
+  // We'll add files to this later.
+  const rootFiles = [];
+
+  env = tsvfs.createVirtualTypeScriptEnvironment(
+    sys,
+    rootFiles,
+    ts,
+    compilerOptions,
+  );
+
+  if (debug) {
+    console.log('initializeFileSystem done');
+  }
+};
+
+const setFile = (tsFileName: string, text: string) => {
+  const existingFile = env.getSourceFile(tsFileName);
+  if (existingFile === undefined) {
+    env.createFile(tsFileName, text);
+  } else {
+    env.updateFile(tsFileName, text);
+  }
+};
+
+// Inspired by: https://stackblitz.com/edit/codemirror-6-typescript?file=client%2Findex.ts%3AL44-L44
+const convertToCodeMirrorDiagnostic = (
+  tsErrors: ts.Diagnostic[],
+) => {
+  return tsErrors.map((tsError: ts.Diagnostic) => ({
+    from: tsError.start,
+    to: tsError.start + tsError.length,
+    severity: 'error',
+    message:
+      typeof tsError.messageText === 'string'
+        ? tsError.messageText
+        : tsError.messageText.messageText,
+  }));
+};
+
+onmessage = async ({ data }) => {
+  if (debug) {
+    console.log('message received');
+  }
+  // Initialize the file system.
+  if (!isFileSystemInitialized) {
+    isFileSystemInitialized = true;
+    await initializeFileSystem();
+  }
+
+  // Sanity check.
+  if (env === null) {
+    throw new Error('File system not initialized');
+  }
+
+  // Handle the update-content event, which
+  // updates the files as they change.
+  // This handles the cases where:
+  //  * The file system is populated for the first time.
+  //  * Files are edited by remote users.
+  if (data.event === 'update-content') {
+    if (debug) {
+      console.log('update-content message received');
+    }
+    // Unpack the files
+    const content: VZCodeContent = data.details;
+    const files: Files = content.files;
+
+    // Iterate over the files
+    for (const fileId of Object.keys(files)) {
+      const file: File = files[fileId];
+      const { name, text } = file;
+
+      const tsFileName = getTSFileName(name);
+
+      if (!isTS(tsFileName)) {
+        continue;
+      }
+
+      setFile(tsFileName, text);
+      // TODO - Handle renaming files.
+      // TODO - Handle deleting files.
+      // TODO - Handle directories.
+    }
+  }
+
+  if (data.event === 'autocomplete-request') {
+    if (debug) {
+      console.log('autocomplete-request message received');
+    }
+    // Should not happen.
+    if (env === null) {
+      console.log('env is null');
+      return;
+    }
+
+    // Example of `data`:
+    // {
+    //   "event": "autocomplete-request",
+    //   "pos": 8,
+    //   "location": "index.js",
+    //   "requestId": "0.9090605799171392"
+    // }
+
+    const autocompleteRequest: AutocompleteRequest = data;
+    const { fileName, fileContent, position, requestId } =
+      autocompleteRequest;
+
+    const tsFileName = getTSFileName(fileName);
+
+    let completions = null;
+    if (isTS(tsFileName)) {
+      // Update the file in the file system to the
+      // absolute latest version. This is critical
+      // for correct completions.
+      setFile(tsFileName, fileContent);
+
+      completions =
+        env.languageService.getCompletionsAtPosition(
+          tsFileName,
+          position,
+          {},
+        );
+    }
+
+    const autocompleteResponse: AutocompleteResponse = {
+      event: 'post-completions',
+      completions,
+      requestId,
+    };
+
+    postMessage(autocompleteResponse);
+  }
+
+  if (data.event === 'lint-request') {
+    if (debug) {
+      console.log('Lint Request');
+    }
+    const linterRequest: LinterRequest = data;
+    const { fileName, fileContent, requestId } =
+      linterRequest;
+
+    const tsFileName = getTSFileName(fileName);
+    let tsErrors = null;
+    // Since we are also updating the server when we autocomplete we do not need to update
+    if (isTS(tsFileName)) {
+      setFile(tsFileName, fileContent);
+      // Creates an array of diagnostic objects containing
+      // both semantic and syntactic diagnostics.
+      tsErrors = env.languageService
+        .getSemanticDiagnostics(tsFileName)
+        .concat(
+          env.languageService.getSyntacticDiagnostics(
+            tsFileName,
+          ),
+        );
+
+      // Be less aggressive for non-TS files,
+      // e.g. files that end in .js or .jsx.
+      if (!isTS(fileName)) {
+        // This code is for errors like:
+        // "Binding element 'data' implicitly has an 'any' type."
+        const LINT_ERROR_CODE_ANY = 7031;
+
+        // This code is for errors like:
+        // "Cannot find module 'd3' or its corresponding type declarations."
+        const LINT_ERROR_CODE_IMPORT = 2307;
+
+        if (debug) {
+          console.log(tsErrors);
+        }
+        tsErrors = tsErrors.filter(
+          (error: { code: number }) =>
+            error.code !== LINT_ERROR_CODE_ANY &&
+            error.code !== LINT_ERROR_CODE_IMPORT,
+        );
+      }
+      tsErrors = convertToCodeMirrorDiagnostic(tsErrors);
+    }
+
+    const linterResponse: LinterResponse = {
+      event: 'post-error-linter',
+      tsErrors,
+      requestId,
+    };
+    postMessage(linterResponse);
+  }
+};

package/src/client/usernameLocalStorage.ts

@@ -1,5 +1,5 @@
 // This module deals with storing and accessing the
-
+// username in local storage.
 import { useEffect, useRef } from 'react';
 import { Username } from '../types';
 

package/src/client/vzReducer/createInitialState.ts

@@ -1,19 +1,18 @@
 import { VZState } from '.';
+import { Username } from '../../types';
 import { ThemeLabel } from '../themes';
 
 export const createInitialState = ({
   defaultTheme,
-  initialUsername,
+  initialUsername = 'Anonymous',
 }: {
   defaultTheme: ThemeLabel;
-  initialUsername?: string;
+  initialUsername?: Username;
 }): VZState => ({
   tabList: [],
   activeFileId: null,
   theme: defaultTheme,
   isSettingsOpen: false,
   editorWantsFocus: false,
-
-  // TODO unify this with App.tsx
-  username: initialUsername || 'Anonymous',
+  username: initialUsername,
 });

package/src/index.ts

@@ -13,13 +13,17 @@ export type {
   ThemeOption,
 } from './client/themes';
 
+export { VZCodeProvider } from './client/App/VZCodeContext';
 export { VZSidebar } from './client/VZSidebar';
 export { VZSettings } from './client/VZSettings';
+export { VZResizer } from './client/VZResizer';
+export { VZLeft } from './client/App/VZLeft';
+export { VZMiddle } from './client/App/VZMiddle';
+
 export {
   SplitPaneResizeProvider,
   SplitPaneResizeContext,
 } from './client/SplitPaneResizeContext';
-export { Resizer } from './client/Resizer';
 export { TabList } from './client/TabList';
 export { CodeEditor } from './client/CodeEditor';
 export { usePrettier } from './client/usePrettier';

package/src/server/handleAIAssist.js

@@ -1,45 +1,99 @@
 import OpenAI from 'openai';
-import { editOp } from 'ot-json1';
+import { editOp, type, replaceOp } from 'ot-json1';
 
-const openai = new OpenAI();
+// Feature flag to slow down AI for development/testing
+const slowdown = false;
+
+let openai;
+if (process.env.OPENAI_API_KEY !== undefined) {
+  openai = new OpenAI();
+}
 
 export async function generateAIResponse({
   inputText,
   insertionCursor,
   fileId,
+  streamId,
   shareDBDoc,
 }) {
-  const stream = await openai.chat.completions.create({
-    model: 'gpt-3.5-turbo',
+  function accomodateDocChanges(op, source) {
+    if (!opComesFromAIAssist(op, source)) {
+      if (op !== null) {
+        insertionCursor = type
+          .transformPosition(
+            ['files', fileId, 'text', insertionCursor],
+            op,
+          )
+          .slice(-1)[0];
+      }
+    }
+  }
+
+  shareDBDoc.on('op', accomodateDocChanges);
+
+  streams[streamId] = await openai.chat.completions.create({
+    // model: 'gpt-3.5-turbo',
+    model: 'gpt-4',
+
     messages: [
       {
         role: 'system',
         content:
-          'Write typescript or javascript code that would follow the prompt',
+          'Write typescript or javascript code to continue the current file, given the other files for context. Use // for comments.',
       },
       { role: 'user', content: inputText },
     ],
     stream: true,
   });
 
-  let currentCursor = insertionCursor;
-
-  for await (const part of stream) {
+  for await (const part of streams[streamId]) {
     const op = editOp(
       ['files', fileId, 'text'],
       'text-unicode',
       [
-        currentCursor,
+        insertionCursor,
         part.choices[0]?.delta?.content || '',
       ],
     );
 
-    shareDBDoc.submitOp(op);
-    currentCursor += (part.choices[0]?.delta?.content || '')
-      .length;
+    shareDBDoc.submitOp(op, { source: AISourceName });
+
+    // Wait for 500ms
+    if (slowdown) {
+      await new Promise((resolve) => {
+        setTimeout(resolve, 1000);
+      });
+    }
+
+    insertionCursor += (
+      part.choices[0]?.delta?.content || ''
+    ).length;
   }
+  shareDBDoc.off('op', accomodateDocChanges);
+
+  const confirmCompletedOperation = replaceOp(
+    [
+      'aiStreams',
+      streamId,
+      'AIStreamStatus',
+      'serverIsRunning',
+    ],
+    true,
+    false,
+  );
+  shareDBDoc.submitOp(confirmCompletedOperation, {
+    source: 'AIServer',
+  });
+}
+
+const AISourceName = 'AIAssist';
+
+function opComesFromAIAssist(ops, source) {
+  return source === AISourceName;
 }
 
+const streams = {};
+
 export const handleAIAssist =
   (shareDBDoc) => async (req, res) => {
     const {
@@ -67,3 +121,13 @@ export const handleAIAssist =
       });
     }
   };
+
+export function haltGeneration(streamId) {
+  const stream = streams[streamId];
+
+  // Stream can be undefined here if the user
+  // clicks start and stop very quickly.
+  if (stream) {
+    stream.controller.abort();
+  }
+}

package/src/server/index.js

@@ -13,7 +13,12 @@ import dotenv from 'dotenv';
 import { computeInitialDocument } from './computeInitialDocument.js';
 import { json1Presence } from '../ot.js';
 import bodyParser from 'body-parser';
-import { handleAIAssist } from './handleAIAssist.js';
+import {
+  generateAIResponse,
+  haltGeneration,
+  handleAIAssist,
+} from './handleAIAssist.js';
+import { replaceOp } from 'ot-json1';
 
 dotenv.config({ path: '../../.env' });
 
@@ -208,12 +213,60 @@ function debounceSave() {
 
 // Subscribe to listen for modifications
 shareDBDoc.subscribe(() => {
-  shareDBDoc.on('op', (op) => {
+  shareDBDoc.on('op', (op, source) => {
     if (shareDBDoc.data.isInteracting) {
       throttleSave();
     } else {
       debounceSave();
     }
+
+    if (
+      op !== null &&
+      op[0] == 'aiStreams' &&
+      source !== 'AIServer' &&
+      source !== 'AIAssist'
+    ) {
+      //This is an insert operation (a new request)
+      if (
+        op[op.length - 1]['i'] !== undefined &&
+        op[op.length - 1]['i']['AIStreamStatus'] !==
+          undefined
+      ) {
+        const input =
+          op[op.length - 1]['i']['AIStreamStatus'];
+
+        generateAIResponse({
+          inputText: input.text,
+          insertionCursor: input.insertionCursor,
+          shareDBDoc: shareDBDoc,
+          streamId: op[op.length - 2],
+          fileId: input.fileId,
+        });
+
+        const confirmStartOperation = replaceOp(
+          [
+            ...op.filter(
+              (value) => typeof value === 'string',
+            ),
+            'AIStreamStatus',
+            'serverIsRunning',
+          ],
+          true,
+          true,
+        );
+
+        shareDBDoc.submitOp(confirmStartOperation, {
+          source: 'AIServer',
+        });
+      }
+      if (
+        op[op.length - 1]['r'] !== null &&
+        op[op.length - 2] == 'clientWantsToStart'
+      ) {
+        //client wants to halt generation
+        haltGeneration(op[1]);
+      }
+    }
   });
 });
 

package/src/types.ts

@@ -72,7 +72,10 @@ export type ShareDBDoc<T> = {
     event: string,
     callback: (op: JSONOp, source: boolean) => void,
   ) => void;
-  off: (event: string, callback: () => void) => void;
+  off: (
+    event: string,
+    callback: (op: JSONOp, source: boolean) => void,
+  ) => void;
   submitOp: (
     op: JSONOp,
     options?: any,
@@ -80,6 +83,61 @@ export type ShareDBDoc<T> = {
   ) => void;
 };
 
+// A unique ID for an AI stream.
+// Generated by the client.
+export type AIStreamId = string;
+
+// TODO define this interface.
+export type AIStreamStatus = {
+  // This field is managed by the client.
+  //  * `true` if the client wants the server to start the generation.
+  //  * `false` if the client does not want the server to start the generation.
+  clientWantsToStart: boolean;
+
+  // This field is managed by the server.
+  //  * `true` if the server is running the generation.
+  //  * `false` if the server is not running the generation.
+  serverIsRunning: boolean;
+
+  text: string;
+
+  startingInsertionCursor: number;
+  fileId: FileId;
+};
+
+/// Alternative universe
+
+// // // If the generation started yet or not
+// // //  * If not, the server will start it.
+// // started: boolean;
+
+// // Required?
+// // aiStreamId: AIStreamId
+
+// // The generation status
+// //  * "running" if the generation is running
+// //  * "finished" if the generation is finished
+// //  * "error" if the generation errored
+// //  * "stopped" if the generation was stopped
+// status:
+//   | 'unstarted'
+//   | 'running'
+//   | 'finished'
+//   | 'error'
+//   | 'stopped';
+
+// // Possible states:
+// //  * { started: 'unstarted' } - client added this,
+// //    It should trigger the server to start the generation.
+// //  * { status: 'running' } - server added this,
+// //    It indicates that the generation is running.
+// //  * { status: 'finished' } - server added this,
+// //    It indicates that the generation is finished.
+// //  * { status: 'stopped' } - server added this,
+// //    It indicates that the generation was stopped.
+// //
+// // After `status` transitions to `finished` or `stopped`,
+
 // The ShareDB document type for VZCode.
 export type VZCodeContent = {
   // `files`
@@ -93,6 +151,25 @@ export type VZCodeContent = {
   //   * `false` or `undefined` when they are not (e.g. normal typing)
   //     * Hot reloading is debounced when this is `false`.
   isInteracting?: boolean;
+
+  // `aiStreams`
+  //  * The AI streams in the VZCode instance.
+  //  * Keys are AI stream IDs.
+  //  * Values are AI streams.
+  // What is this for?
+  //  * Synchronize the status of streams between the
+  //    client and server.
+  //  * This can power the transition of the icon from
+  //    lightning bolt to stop sign and back (when stream finishes)
+  aiStreams: {
+    // These need to be removed at some point?
+    [aiStreamId: AIStreamId]: {
+      AIStreamStatus: AIStreamStatus;
+    };
+  };
+
+  // TODO consider is there a way to replace the HTTP request to
+  // the AIAssist endpoint with a manipulation of the ShareDB document?
 };
 
 // An id used for presence.