vzcode 2.3.0 → 2.4.0

package/src/server/computeInitialDocument.ts

@@ -0,0 +1,220 @@
+import fs from 'fs';
+import path from 'path';
+import { randomId } from '../randomId.js';
+import {
+  enableDirectories,
+  debugDirectories,
+  debugIgnore,
+} from './featureFlags.js';
+import createIgnore from 'ignore';
+import { ignoreFilePattern, baseIgnore } from './config.js';
+import { isDirectory } from './isDirectory.js';
+
+// Import the image file utility
+const isImageFile = (fileName) => {
+  return (
+    fileName.match(
+      /\.(png|jpg|jpeg|gif|bmp|svg|webp)$/i,
+    ) !== null
+  );
+};
+
+/**
+ * @param {string} fullPath - absolut path of the workspace root
+ * @param {string} currentDirectoryPath - path where the ignore file is found, relative to fullPath
+ * @param {string} fileName - name of the ignore file
+ * @returns {string[]} parsed lines
+ */
+const parseIgnoreFile = (
+  fullPath,
+  currentDirectory,
+  fileName,
+) => {
+  const filePath = path.join(
+    fullPath,
+    currentDirectory,
+    fileName,
+  );
+  const content = fs.readFileSync(filePath, 'utf8');
+  const globs = content
+    .split(/[\n\r]+/)
+    .filter(
+      // remove blank line and comments
+      (line) => line.length > 0 && !line.startsWith('#'),
+    )
+    .map((line) => {
+      const { bang, slash, glob } = line.match(
+        /^(?<bang>!?)(?<slash>\/?)(?<glob>.*)$/,
+      ).groups;
+      const hasSlash =
+        Boolean(slash) || /\/.*\S/.test(glob);
+      const relativeGlob = path.posix.join(
+        currentDirectory.replace(
+          // escape characters with special meaning in glob expressions
+          /[*?!# \[\]\\]/g,
+          (char) => '\\' + char,
+        ),
+        // a pattern that doesn't include a slash (not counting a trailing one) matches files in any descendant directory od the current one
+        hasSlash ? '' : '**',
+        glob,
+      );
+      // preserve leading `!` and `/` characters
+      return bang + slash + relativeGlob;
+    });
+  if (debugIgnore) {
+    console.debug(
+      'at',
+      currentDirectory,
+      'parsing',
+      fileName,
+      'obtained globs',
+      globs,
+    );
+  }
+  return globs;
+};
+
+// Lists files from the file system,
+// converts them into the VZCode internal
+// ShareDB-compatible data structure.
+export const computeInitialDocument = ({ fullPath }) => {
+  // Isolate files, not directories.
+  // Inspired by https://stackoverflow.com/questions/41472161/fs-readdir-ignore-directories
+
+  // Initialize the document using our data structure for representing files.
+  const initialDocument = {
+    // `files`
+    //  * Keys are file ids, which are random numbers.
+    //  * Values are objects with properties:
+    //    * text - the text content of the file
+    //    * name - the file name
+    files: {},
+
+    // `isInteracting`
+    //  * Whether the user is currently interacting with the document
+    //    using an interactive code widget such as number dragger.
+    //  * When true, the auto-save to the file system is changed to be
+    //    more frequent (throttled not debounced).
+    // isInteracting: false,
+    // This is `undefined` initially.
+  };
+
+  /**
+   * Stack for recursively traversing directories.
+   * @type {string[]}
+   */
+  let files = [];
+
+  const ignoreFileMatcher = createIgnore().add(
+    ignoreFilePattern,
+  );
+  const isIgnoreFile = (fileName) =>
+    ignoreFileMatcher.ignores(fileName);
+
+  const unsearchedDirectories = [
+    {
+      currentDirectory: '.',
+      ignore: createIgnore().add(baseIgnore),
+    },
+  ];
+
+  while (unsearchedDirectories.length !== 0) {
+    const { currentDirectory, ignore: parentIgnore } =
+      unsearchedDirectories.pop();
+    const currentDirectoryPath = path.join(
+      fullPath,
+      currentDirectory,
+    );
+    const dirEntries = fs
+      .readdirSync(currentDirectoryPath, {
+        withFileTypes: true,
+      })
+      .filter((dirent) =>
+        enableDirectories ? true : dirent.isFile(),
+      );
+    // find .ignore or .gitignore files in the current directory
+    const ignoreFiles = dirEntries
+      .filter(
+        (dirent) =>
+          dirent.isFile() && isIgnoreFile(dirent.name),
+      )
+      .map((file) => file.name);
+    let ignore = parentIgnore;
+    if (ignoreFiles.length > 0) {
+      const globs = ignoreFiles.flatMap((fileName) =>
+        parseIgnoreFile(
+          fullPath,
+          currentDirectory,
+          fileName,
+        ),
+      );
+      ignore = createIgnore().add(parentIgnore).add(globs);
+    }
+    const newFiles = dirEntries
+      .filter((dirent) => {
+        const relativePath =
+          path.posix.join(currentDirectory, dirent.name) +
+          (dirent.isDirectory() ? '/' : '');
+        const keep = !ignore.ignores(relativePath);
+        if (debugIgnore && !keep) {
+          console.debug(
+            'at',
+            currentDirectory,
+            'ignoring',
+            relativePath,
+          );
+        }
+        return keep;
+      })
+      // Add a trailing slash for directories
+      .map((dirent) => {
+        const relativePath = path.posix.join(
+          currentDirectory,
+          dirent.name,
+        );
+
+        if (!dirent.isDirectory()) {
+          return relativePath;
+        }
+        unsearchedDirectories.push({
+          currentDirectory: relativePath,
+          ignore,
+        });
+        return relativePath + '/';
+      });
+    // console.log(currentDirectory);
+
+    files.push(...newFiles);
+  }
+
+  files.forEach((file) => {
+    const id = randomId();
+    let text = null;
+
+    if (!isDirectory(file)) {
+      const filePath = path.join(fullPath, file);
+
+      if (isImageFile(file)) {
+        // Read image files as binary and convert to base64
+        const buffer = fs.readFileSync(filePath);
+        text = buffer.toString('base64');
+      } else {
+        // Read non-image files as UTF-8 text
+        text = fs.readFileSync(filePath, 'utf-8');
+      }
+    }
+
+    initialDocument.files[id] = {
+      text,
+      name: file,
+    };
+  });
+
+  if (debugDirectories) {
+    console.log('files:');
+    console.log(files);
+    console.log('initialDocument:');
+    console.log(JSON.stringify(initialDocument, null, 2));
+  }
+  return initialDocument;
+};

package/src/server/config.ts

@@ -0,0 +1,24 @@
+/**
+ *patterns describing which files should e comsidered as ignore files when creating the file tree.
+ *
+ * Format: one pattern per line, in gitignore format as defimed by https://git-scm.com/docs/gitignore#_pattern_format
+ */
+export const ignoreFilePattern =
+  process.env.IGNORE_FILE_PATTERN ??
+  `
+.vzignore
+.ignore
+.gitignore
+`;
+
+/**
+ * the base ignore patterns, that should be applied before any ignore file found when creating the file tree.
+ *
+ * These patterns apply even in the absence of an ignore file.
+ */
+export const baseIgnore =
+  process.env.BASE_IGNORE ??
+  `
+.git/
+node_modules/
+`;

package/src/server/featureFlags.ts

@@ -0,0 +1,8 @@
+// Feature flag for directories, disabled until it's fully working.
+export const enableDirectories = true;
+
+export const debugDirectories = false;
+
+export const debugIgnore = false;
+
+export const enableESLint = true;

package/src/server/generateAIResponse.ts

@@ -0,0 +1,188 @@
+import OpenAI from 'openai';
+import { json1Presence } from '../ot.js';
+
+// This module implements the generation of the AI response,
+// using the OpenAI client.
+
+// These environment variables are used to configure the OpenAI client.
+// See `.env.sample` for the expected values.
+const { VZCODE_AI_API_KEY, VZCODE_AI_BASE_URL } =
+  process.env;
+
+// If neither of these are not set, the OpenAI client will not be initialized,
+// e.g. for local development where no AI is needed.
+const isAIEnabled =
+  VZCODE_AI_API_KEY !== undefined &&
+  VZCODE_AI_BASE_URL !== undefined;
+
+const { editOp, type } = json1Presence;
+
+// Debug flag to log more information during development.
+const debug = false;
+
+// Feature flag to slow down AI for development/testing.
+// Particularly relevant for debugging ShareDB and CodeMirror sync issues.
+// This allows you to test the case of editing the document at the
+// same time as the AI generation is happening.
+const slowdown = false;
+
+// The options passed into the OpenAI client
+// new OpenAI(openAIOptions)
+const openAIOptions: { apiKey?: string; baseURL?: string } =
+  {};
+
+// Support specifying the API key via an environment variable
+// If VZCODE_AI_API_KEY is not set, note that the OpenAI client
+// will look for the OPENAI_API_KEY environment variable instead.
+if (process.env.VZCODE_AI_API_KEY !== undefined) {
+  openAIOptions.apiKey = process.env.VZCODE_AI_API_KEY;
+}
+
+// Support for local AI server
+if (process.env.VZCODE_AI_BASE_URL !== undefined) {
+  // The OpenAI client errors if the API key is not set,
+  // so in the case where we don't need an API key,
+  // e.g. for testing with a local AI server like LM Studio or LocalAI,
+  // we populate the option with a fake API key, so it doesn't error.
+  if (!openAIOptions.apiKey) {
+    openAIOptions.apiKey = 'Fake API Key';
+  }
+
+  openAIOptions.baseURL = process.env.VZCODE_AI_BASE_URL;
+}
+
+debug &&
+  console.log(
+    'openAIOptions: ' +
+      JSON.stringify(openAIOptions, null, 2),
+  );
+let openai;
+
+if (isAIEnabled) {
+  openai = new OpenAI(openAIOptions);
+}
+
+// The name of the source that the AI responses
+// will be attributed to in ShareDB operations.
+const AIShareDBSourceName = 'AIAssist';
+
+// Returns trie if the operation comes from AI.
+const opComesFromAIAssist = (ops, source) =>
+  source === AIShareDBSourceName;
+
+// Keeps track of the currently ongoing AI streams.
+// There could be many streams at the same time.
+export const streams: Record<string, any> = {};
+
+export const generateAIResponse = async ({
+  inputText,
+  insertionCursor,
+  fileId,
+  streamId,
+  shareDBDoc,
+}) => {
+  if (!isAIEnabled) {
+    console.log(
+      '[generateAIResponse] AI is not enabled. Skipping AI generation.',
+    );
+    console.log(
+      '[generateAIResponse] To enable AI, see .env.sample for the required environment variables.',
+    );
+    return;
+  }
+
+  if (debug) {
+    console.log(
+      '[generateAIResponse] inputText:',
+      inputText,
+    );
+    console.log(
+      '[generateAIResponse] insertionCursor:',
+      insertionCursor,
+    );
+    console.log('[generateAIResponse] fileId:', fileId);
+    console.log('[generateAIResponse] streamId:', streamId);
+    console.log(
+      '[generateAIResponse] shareDBDoc:',
+      shareDBDoc,
+    );
+  }
+
+  // Handle the case that a user edits the text in the document
+  // that comes becofore the insertion cursor.
+  const accomodateDocChanges = (op, source) => {
+    if (!opComesFromAIAssist(op, source)) {
+      if (op !== null) {
+        insertionCursor = type
+          .transformPosition(
+            ['files', fileId, 'text', insertionCursor],
+            op,
+          )
+          .slice(-1)[0];
+      }
+    }
+  };
+  shareDBDoc.on('op', accomodateDocChanges);
+
+  // The prompt!
+  const messages = [
+    {
+      role: 'system',
+      content: [
+        'You are an expert programmer.',
+        'Your task is to output ONLY the code that replaces <FILL_ME> correctly.',
+        'Do not add any markdown around.',
+        'Do not duplicate the code before or after <FILL_ME>.',
+        'Do not make any changes outside of <FILL_ME>.',
+        'Do not enclose the output with backticks.',
+        'If any additional instructions are required, they will be provided in comments.',
+      ].join(' '),
+    },
+    { role: 'user', content: inputText },
+  ];
+
+  if (debug) {
+    console.log('[generateAIResponse] messages:');
+    console.log(JSON.stringify(messages, null, 2));
+  }
+
+  streams[streamId] = await openai.chat.completions.create({
+    // model: 'gpt-3.5-turbo',
+    model: 'gpt-4o',
+    messages,
+    reasoning: {
+      effort: 'medium',
+      exclude: false,
+    } as any, // Type assertion for OpenRouter-specific reasoning parameter
+    provider: {
+      sort: 'throughput',
+    } as any, // Type assertion for OpenRouter-specific provider parameter
+    stream: true,
+  });
+
+  for await (const part of streams[streamId]) {
+    const op = editOp(
+      ['files', fileId, 'text'],
+      'text-unicode',
+      [
+        insertionCursor,
+        part.choices[0]?.delta?.content || '',
+      ],
+    );
+
+    shareDBDoc.submitOp(op, {
+      source: AIShareDBSourceName,
+    });
+
+    if (slowdown) {
+      await new Promise((resolve) => {
+        setTimeout(resolve, 2000);
+      });
+    }
+
+    insertionCursor += (
+      part.choices[0]?.delta?.content || ''
+    ).length;
+  }
+  shareDBDoc.off('op', accomodateDocChanges);
+};

package/src/server/handleAIAssist.ts

@@ -0,0 +1,53 @@
+import {
+  generateAIResponse,
+  streams,
+} from './generateAIResponse.js';
+
+const debug = false;
+
+export const handleAIAssist =
+  (shareDBDoc: any) => async (req: any, res: any) => {
+    const { inputText, insertionCursor, fileId } = req.body;
+
+    if (debug) {
+      console.log('[handleAIAssist] inputText:', inputText);
+      console.log(
+        '[handleAIAssist] insertionCursor:',
+        insertionCursor,
+      );
+      console.log('[handleAIAssist] fileId:', fileId);
+    }
+
+    try {
+      // Generate a unique streamId for this request
+      const streamId = `stream_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+
+      await generateAIResponse({
+        inputText,
+        insertionCursor,
+        fileId,
+        streamId,
+        shareDBDoc,
+      });
+
+      res
+        .status(200)
+        .send({ message: 'Operation successful!' });
+    } catch (error: any) {
+      console.error('handleAIAssist error:', error);
+      res.status(500).send({
+        message: 'Internal Server Error',
+        error: error.message,
+      });
+    }
+  };
+
+export function haltGeneration(streamId: string) {
+  const stream = streams[streamId];
+
+  // Stream can be undefined here if the user
+  // clicks start and stop very quickly.
+  if (stream) {
+    stream.controller.abort();
+  }
+}

package/src/server/handleAIChatMessage.ts

@@ -0,0 +1,2 @@
+// Re-export the refactored handleAIChatMessage from the new modular structure
+export { handleAIChatMessage } from './aiChatHandler/index.js';

package/src/server/handleAICopilot.ts

@@ -0,0 +1,97 @@
+import { ChatOpenAI } from '@langchain/openai';
+import { StringOutputParser } from '@langchain/core/output_parsers';
+
+// This module implements the generation of the copilot response.
+
+// Debug flag to log more information during development.
+const debug = false;
+
+export const handleAICopilot = () => {
+  // These environment variables are used to configure the OpenAI client.
+  // See `.env.sample` for the expected values.
+  const {
+    VZCODE_AI_COPILOT_API_KEY,
+    VZCODE_AI_COPILOT_BASE_URL,
+    VZCODE_AI_COPILOT_MODEL,
+  } = process.env;
+
+  // If neither of these are not set, the OpenAI client will not be initialized,
+  // e.g. for local development where no AI is needed.
+  const isCopilotEnabled =
+    VZCODE_AI_COPILOT_API_KEY !== undefined &&
+    VZCODE_AI_COPILOT_BASE_URL !== undefined &&
+    VZCODE_AI_COPILOT_MODEL !== undefined;
+
+  debug &&
+    console.log('isCopilotEnabled: ' + isCopilotEnabled);
+
+  debug &&
+    console.log({
+      VZCODE_AI_COPILOT_API_KEY,
+      VZCODE_AI_COPILOT_BASE_URL,
+      VZCODE_AI_COPILOT_MODEL,
+    });
+
+  let chatModel;
+  if (isCopilotEnabled) {
+    const options = {
+      modelName: VZCODE_AI_COPILOT_MODEL,
+      configuration: {
+        apiKey: VZCODE_AI_COPILOT_API_KEY,
+        baseURL: VZCODE_AI_COPILOT_BASE_URL,
+      },
+      additionalParameters: {
+        provider: { sort: 'throughput' },
+      },
+      streaming: false,
+    };
+    debug && console.log('chatModel options:', options);
+    chatModel = new ChatOpenAI(options);
+  }
+
+  return async (req, res) => {
+    // Don't break if the AI is not enabled.
+    // This is useful for local development.
+    if (!isCopilotEnabled) {
+      res.status(200).send({ text: '' });
+      return;
+    }
+    const { prefix, suffix } = req.body;
+
+    const messages = [
+      {
+        role: 'user',
+        content: [
+          'Complete the missing part of the text below:\n\n',
+          '\`\`\`\n',
+          prefix,
+          '<MISSING_PART>',
+          suffix,
+          '\`\`\`\n\n',
+          'ONLY output the text that replaces <MISSING_PART>.\n\n',
+          'NEVER include \`\`\`javascript or similar code blocks.\n\n',
+        ].join(''),
+      },
+    ];
+    if (debug) {
+      console.log(
+        '[generateAIResponse] prompt:\n```\n' +
+          messages[0].content +
+          '\n```\n',
+      );
+    }
+    try {
+      const result = await chatModel.invoke(messages);
+      const parser = new StringOutputParser();
+      const text = await parser.invoke(result);
+
+      res.status(200).send({ text });
+    } catch (error) {
+      console.error('handleAICopilot error:', error);
+      res.status(500).send({
+        message: 'Internal Server Error',
+        error: error.message,
+      });
+    }
+  };
+};