vzcode 0.44.0 → 0.46.0

package/src/server/computeInitialDocument.js

@@ -4,8 +4,10 @@ import { randomId } from '../randomId.js';
 import {
   enableDirectories,
   debugDirectories,
+  debugIgnore,
 } from './featureFlags.js';
 import createIgnore from 'ignore';
+import { ignoreFilePattern, baseIgnore } from './config.js';
 
 const isDirectory = (file) => file.endsWith('/');
 
@@ -27,7 +29,7 @@ const parseIgnoreFile = (
   );
   const content = fs.readFileSync(filePath, 'utf8');
   const globs = content
-    .split(/(?:[\n\r])+/)
+    .split(/[\n\r]+/)
     .filter(
       // remove blank line and comments
       (line) => line.length > 0 && !line.startsWith('#'),
@@ -40,7 +42,7 @@ const parseIgnoreFile = (
         Boolean(slash) || /\/.*\S/.test(glob);
       const relativeGlob = path.posix.join(
         currentDirectory.replace(
-          // escape characters ith special meaning in glob expressions
+          // escape characters with special meaning in glob expressions
           /[*?!# \[\]\\]/g,
           (char) => '\\' + char,
         ),
@@ -48,10 +50,10 @@ const parseIgnoreFile = (
         hasSlash ? '' : '**',
         glob,
       );
-      // preserve leading ! and / characters
+      // preserve leading `!` and `/` characters
       return bang + slash + relativeGlob;
     });
-  if (debugDirectories) {
+  if (debugIgnore) {
     console.debug(
       'at',
       currentDirectory,
@@ -94,18 +96,21 @@ export const computeInitialDocument = ({ fullPath }) => {
    */
   let files = [];
 
-  // default paths to ignore
-  const globs = ['.git'];
-  const ignoreStack = [
-    { ignore: createIgnore().add('globs'), globs },
-  ];
+  const ignoreFileMatcher = createIgnore().add(
+    ignoreFilePattern,
+  );
+  const isIgnoreFile = (fileName) =>
+    ignoreFileMatcher.ignores(fileName);
 
   const unsearchedDirectories = [
-    { currentDirectory: '.', ignoreStack },
+    {
+      currentDirectory: '.',
+      ignore: createIgnore().add(baseIgnore),
+    },
   ];
 
   while (unsearchedDirectories.length !== 0) {
-    const { currentDirectory, ignoreStack } =
+    const { currentDirectory, ignore: parentIgnore } =
       unsearchedDirectories.pop();
     const currentDirectoryPath = path.join(
       fullPath,
@@ -122,13 +127,10 @@ export const computeInitialDocument = ({ fullPath }) => {
     const ignoreFiles = dirEntries
       .filter(
         (dirent) =>
-          dirent.isFile() &&
-          /^\.(?:git)?ignore$/.test(dirent.name),
+          dirent.isFile() && isIgnoreFile(dirent.name),
       )
       .map((file) => file.name);
-    const ignoreEntry = ignoreStack.at(-1);
-    let { ignore } = ignoreEntry;
-    let newIgnoreStack = ignoreStack;
+    let ignore = parentIgnore;
     if (ignoreFiles.length > 0) {
       const globs = ignoreFiles.flatMap((fileName) =>
         parseIgnoreFile(
@@ -137,33 +139,20 @@ export const computeInitialDocument = ({ fullPath }) => {
           fileName,
         ),
       );
-      const newEntry = { ignore, globs };
-      newIgnoreStack = ignoreStack.slice();
-      newIgnoreStack.push(newEntry);
-      ignore = createIgnore().add(
-        newIgnoreStack.flatMap((entry) => entry.globs),
-      );
-      newEntry.ignore = ignore;
-      if (debugDirectories) {
-        console.debug(
-          'at',
-          currentDirectory,
-          'ignoring globs',
-          newIgnoreStack.map((item) => item.globs),
-        );
-      }
+      ignore = createIgnore().add(parentIgnore).add(globs);
     }
     const newFiles = dirEntries
       .filter((dirent) => {
-        let keep = !ignore.ignores(
-          path.posix.join(currentDirectory, dirent.name),
-        );
-        if (debugDirectories && !keep) {
+        const relativePath =
+          path.posix.join(currentDirectory, dirent.name) +
+          (dirent.isDirectory() ? '/' : '');
+        const keep = !ignore.ignores(relativePath);
+        if (debugIgnore && !keep) {
           console.debug(
             'at',
             currentDirectory,
             'ignoring',
-            dirent.name,
+            relativePath,
           );
         }
         return keep;
@@ -180,7 +169,7 @@ export const computeInitialDocument = ({ fullPath }) => {
         }
         unsearchedDirectories.push({
           currentDirectory: relativePath,
-          ignoreStack: newIgnoreStack,
+          ignore,
         });
         return relativePath + '/';
       });

package/src/server/config.js

@@ -0,0 +1,24 @@
+/**
+ *patterns describing which files should e comsidered as ignore files when creating the file tree.
+ *
+ * Format: one pattern per line, in gitignore format as defimed by https://git-scm.com/docs/gitignore#_pattern_format
+ */
+export const ignoreFilePattern =
+  process.env.IGNORE_FILE_PATTERN ??
+  `
+.vzignore
+.ignore
+.gitignore
+`;
+
+/**
+ * the base ignore patterns, that should be applied before any ignore file found when creating the file tree.
+ *
+ * These patterns apply even in the absence of an ignore file.
+ */
+export const baseIgnore =
+  process.env.BASE_IGNORE ??
+  `
+.git/
+node_modules/
+`;

package/src/server/featureFlags.js

@@ -1,4 +1,10 @@
 // Feature flag for directories, disabled until it's fully working.
 export const enableDirectories = true;
 
-export const debugDirectories = false;
+export const debugDirectories = Boolean(
+  process.env.DEBUG_FILE_TREE,
+);
+
+export const debugIgnore = Boolean(
+  process.env.DEBUG_IGNORE,
+);

package/src/server/generateAIResponse.js

@@ -0,0 +1,101 @@
+import OpenAI from 'openai';
+import { json1Presence } from '../ot.js';
+
+const { editOp, type } = json1Presence;
+
+const debug = false;
+
+// Feature flag to slow down AI for development/testing
+const slowdown = false;
+
+let openai;
+if (process.env.OPENAI_API_KEY !== undefined) {
+  openai = new OpenAI();
+}
+
+const AISourceName = 'AIAssist';
+
+function opComesFromAIAssist(ops, source) {
+  return source === AISourceName;
+}
+
+const streams = {};
+
+export const generateAIResponse = async ({
+  inputText,
+  insertionCursor,
+  fileId,
+  streamId,
+  shareDBDoc,
+}) => {
+  if (debug) {
+    console.log(
+      '[generateAIResponse] inputText:',
+      inputText,
+    );
+    console.log(
+      '[generateAIResponse] insertionCursor:',
+      insertionCursor,
+    );
+    console.log('[generateAIResponse] fileId:', fileId);
+    console.log('[generateAIResponse] streamId:', streamId);
+    console.log(
+      '[generateAIResponse] shareDBDoc:',
+      shareDBDoc,
+    );
+  }
+  function accomodateDocChanges(op, source) {
+    if (!opComesFromAIAssist(op, source)) {
+      if (op !== null) {
+        insertionCursor = type
+          .transformPosition(
+            ['files', fileId, 'text', insertionCursor],
+            op,
+          )
+          .slice(-1)[0];
+      }
+    }
+  }
+
+  shareDBDoc.on('op', accomodateDocChanges);
+
+  streams[streamId] = await openai.chat.completions.create({
+    // model: 'gpt-3.5-turbo',
+    model: 'gpt-4',
+
+    messages: [
+      {
+        role: 'system',
+        content:
+          'Write typescript or javascript code to continue the current file, given the other files for context. Use // for comments.',
+      },
+      { role: 'user', content: inputText },
+    ],
+    stream: true,
+  });
+
+  for await (const part of streams[streamId]) {
+    const op = editOp(
+      ['files', fileId, 'text'],
+      'text-unicode',
+      [
+        insertionCursor,
+        part.choices[0]?.delta?.content || '',
+      ],
+    );
+
+    shareDBDoc.submitOp(op, { source: AISourceName });
+
+    // Wait for 500ms
+    if (slowdown) {
+      await new Promise((resolve) => {
+        setTimeout(resolve, 1000);
+      });
+    }
+
+    insertionCursor += (
+      part.choices[0]?.delta?.content || ''
+    ).length;
+  }
+  shareDBDoc.off('op', accomodateDocChanges);
+};

package/src/server/handleAIAssist.js

@@ -1,109 +1,20 @@
-import OpenAI from 'openai';
-import { json1Presence } from '../ot.js';
+import { generateAIResponse } from './generateAIResponse.js';
 
-const { editOp, type, replaceOp } = json1Presence;
+const debug = false;
 
-// Feature flag to slow down AI for development/testing
-const slowdown = false;
-
-let openai;
-if (process.env.OPENAI_API_KEY !== undefined) {
-  openai = new OpenAI();
-}
-
-export async function generateAIResponse({
-  inputText,
-  insertionCursor,
-  fileId,
-  streamId,
-  shareDBDoc,
-}) {
-  function accomodateDocChanges(op, source) {
-    if (!opComesFromAIAssist(op, source)) {
-      if (op !== null) {
-        insertionCursor = type
-          .transformPosition(
-            ['files', fileId, 'text', insertionCursor],
-            op,
-          )
-          .slice(-1)[0];
-      }
-    }
-  }
-
-  shareDBDoc.on('op', accomodateDocChanges);
-
-  streams[streamId] = await openai.chat.completions.create({
-    // model: 'gpt-3.5-turbo',
-    model: 'gpt-4',
-
-    messages: [
-      {
-        role: 'system',
-        content:
-          'Write typescript or javascript code to continue the current file, given the other files for context. Use // for comments.',
-      },
-      { role: 'user', content: inputText },
-    ],
-    stream: true,
-  });
+export const handleAIAssist =
+  (shareDBDoc) => async (req, res) => {
+    const { inputText, insertionCursor, fileId } = req.body;
 
-  for await (const part of streams[streamId]) {
-    const op = editOp(
-      ['files', fileId, 'text'],
-      'text-unicode',
-      [
+    if (debug) {
+      console.log('[handleAIAssist] inputText:', inputText);
+      console.log(
+        '[handleAIAssist] insertionCursor:',
         insertionCursor,
-        part.choices[0]?.delta?.content || '',
-      ],
-    );
-
-    shareDBDoc.submitOp(op, { source: AISourceName });
-
-    // Wait for 500ms
-    if (slowdown) {
-      await new Promise((resolve) => {
-        setTimeout(resolve, 1000);
-      });
+      );
+      console.log('[handleAIAssist] fileId:', fileId);
     }
 
-    insertionCursor += (
-      part.choices[0]?.delta?.content || ''
-    ).length;
-  }
-  shareDBDoc.off('op', accomodateDocChanges);
-
-  const confirmCompletedOperation = replaceOp(
-    [
-      'aiStreams',
-      streamId,
-      'AIStreamStatus',
-      'serverIsRunning',
-    ],
-    true,
-    false,
-  );
-  shareDBDoc.submitOp(confirmCompletedOperation, {
-    source: 'AIServer',
-  });
-}
-
-const AISourceName = 'AIAssist';
-
-function opComesFromAIAssist(ops, source) {
-  return source === AISourceName;
-}
-
-const streams = {};
-
-export const handleAIAssist =
-  (shareDBDoc) => async (req, res) => {
-    const {
-      text: inputText,
-      cursorLocation: insertionCursor,
-      fileId,
-    } = req.body;
-
     try {
       await generateAIResponse({
         inputText,

package/src/server/index.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import './setupEnv.js';
 import http from 'http';
 import express from 'express';
 import ShareDB from 'sharedb';
@@ -9,19 +10,10 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import open from 'open';
 import ngrok from 'ngrok';
-import dotenv from 'dotenv';
 import bodyParser from 'body-parser';
 import { json1Presence } from '../ot.js';
 import { computeInitialDocument } from './computeInitialDocument.js';
-import {
-  generateAIResponse,
-  haltGeneration,
-  handleAIAssist,
-} from './handleAIAssist.js';
-
-const { replaceOp } = json1Presence;
-
-dotenv.config({ path: '../../.env' });
+import { handleAIAssist } from './handleAIAssist.js';
 
 // The time in milliseconds by which auto-saving is debounced.
 const autoSaveDebounceTimeMS = 800;
@@ -105,7 +97,7 @@ shareDBDoc.create(initialDocument, json1Presence.type.uri);
 
 // Handle AI Assist requests.
 app.post(
-  '/AIAssist',
+  '/ai-assist',
   bodyParser.json(),
   handleAIAssist(shareDBDoc),
 );
@@ -221,53 +213,53 @@ shareDBDoc.subscribe(() => {
       debounceSave();
     }
 
-    if (
-      op !== null &&
-      op[0] == 'aiStreams' &&
-      source !== 'AIServer' &&
-      source !== 'AIAssist'
-    ) {
-      //This is an insert operation (a new request)
-      if (
-        op[op.length - 1]['i'] !== undefined &&
-        op[op.length - 1]['i']['AIStreamStatus'] !==
-          undefined
-      ) {
-        const input =
-          op[op.length - 1]['i']['AIStreamStatus'];
-
-        generateAIResponse({
-          inputText: input.text,
-          insertionCursor: input.insertionCursor,
-          shareDBDoc: shareDBDoc,
-          streamId: op[op.length - 2],
-          fileId: input.fileId,
-        });
-
-        const confirmStartOperation = replaceOp(
-          [
-            ...op.filter(
-              (value) => typeof value === 'string',
-            ),
-            'AIStreamStatus',
-            'serverIsRunning',
-          ],
-          true,
-          true,
-        );
-
-        shareDBDoc.submitOp(confirmStartOperation, {
-          source: 'AIServer',
-        });
-      }
-      if (
-        op[op.length - 1]['r'] !== null &&
-        op[op.length - 2] == 'clientWantsToStart'
-      ) {
-        //client wants to halt generation
-        haltGeneration(op[1]);
-      }
-    }
+    // if (
+    //   op !== null &&
+    //   op[0] == 'aiStreams' &&
+    //   source !== 'AIServer' &&
+    //   source !== 'AIAssist'
+    // ) {
+    //   //This is an insert operation (a new request)
+    //   if (
+    //     op[op.length - 1]['i'] !== undefined &&
+    //     op[op.length - 1]['i']['AIStreamStatus'] !==
+    //       undefined
+    //   ) {
+    //     const input =
+    //       op[op.length - 1]['i']['AIStreamStatus'];
+
+    //     generateAIResponse({
+    //       inputText: input.text,
+    //       insertionCursor: input.insertionCursor,
+    //       shareDBDoc: shareDBDoc,
+    //       streamId: op[op.length - 2],
+    //       fileId: input.fileId,
+    //     });
+
+    //     const confirmStartOperation = replaceOp(
+    //       [
+    //         ...op.filter(
+    //           (value) => typeof value === 'string',
+    //         ),
+    //         'AIStreamStatus',
+    //         'serverIsRunning',
+    //       ],
+    //       true,
+    //       true,
+    //     );
+
+    //     shareDBDoc.submitOp(confirmStartOperation, {
+    //       source: 'AIServer',
+    //     });
+    //   }
+    //   if (
+    //     op[op.length - 1]['r'] !== null &&
+    //     op[op.length - 2] == 'clientWantsToStart'
+    //   ) {
+    //     //client wants to halt generation
+    //     haltGeneration(op[1]);
+    //   }
+    // }
   });
 });
 

package/src/server/setupEnv.js

@@ -0,0 +1,18 @@
+import dotenv from 'dotenv';
+import { fileURLToPath } from 'url';
+import { join } from 'node:path';
+
+const dir = fileURLToPath(import.meta.url);
+dotenv.config({
+  path: join(dir, '../../../.env'),
+});
+if (process.env.DEBUG_SETUP_ENV) {
+  console.log(
+    'environment variables set up',
+    Object.fromEntries(
+      Object.entries(process.env).filter(
+        (entry) => !entry[0].endsWith('PATH'),
+      ),
+    ),
+  );
+}

package/src/types.ts

@@ -1,8 +1,6 @@
 // FileId
 //   * A unique ID for a file.
 
-import { JSONOp } from 'ot-json1';
-
 //   * This is a random string.
 export type FileId = string;
 
@@ -60,6 +58,8 @@ export interface FileTreeFile {
   fileId: FileId;
 }
 
+export type JSONOp = any;
+
 // `ShareDBDoc`
 // A ShareDB Document.
 // TODO better types? Integrate with upstream how?
@@ -87,23 +87,23 @@ export type ShareDBDoc<T> = {
 // Generated by the client.
 export type AIStreamId = string;
 
-// TODO define this interface.
-export type AIStreamStatus = {
-  // This field is managed by the client.
-  //  * `true` if the client wants the server to start the generation.
-  //  * `false` if the client does not want the server to start the generation.
-  clientWantsToStart: boolean;
+// // TODO define this interface.
+// export type AIStreamStatus = {
+//   // This field is managed by the client.
+//   //  * `true` if the client wants the server to start the generation.
+//   //  * `false` if the client does not want the server to start the generation.
+//   clientWantsToStart: boolean;
 
-  // This field is managed by the server.
-  //  * `true` if the server is running the generation.
-  //  * `false` if the server is not running the generation.
-  serverIsRunning: boolean;
+//   // This field is managed by the server.
+//   //  * `true` if the server is running the generation.
+//   //  * `false` if the server is not running the generation.
+//   serverIsRunning: boolean;
 
-  text: string;
+//   text: string;
 
-  startingInsertionCursor: number;
-  fileId: FileId;
-};
+//   startingInsertionCursor: number;
+//   fileId: FileId;
+// };
 
 /// Alternative universe
 
@@ -142,7 +142,7 @@ export type AIStreamStatus = {
 export type VZCodeContent = {
   // `files`
   //   * The files in the VZCode instance.
-  files: Files;
+  files?: Files;
 
   // `isInteracting`
   //   * `true` when the user is interacting
@@ -161,12 +161,12 @@ export type VZCodeContent = {
   //    client and server.
   //  * This can power the transition of the icon from
   //    lightning bolt to stop sign and back (when stream finishes)
-  aiStreams?: {
-    // These need to be removed at some point?
-    [aiStreamId: AIStreamId]: {
-      AIStreamStatus: AIStreamStatus;
-    };
-  };
+  // aiStreams?: {
+  //   // These need to be removed at some point?
+  //   [aiStreamId: AIStreamId]: {
+  //     AIStreamStatus: AIStreamStatus;
+  //   };
+  // };
 
   // TODO consider is there a way to replace the HTTP request to
   // the AIAssist endpoint with a manipulation of the ShareDB document?