semantic-grep 0.0.1

package/src/core/embedder.ts

@@ -0,0 +1,89 @@
+import * as AI from "ai";
+import { createVoyage } from "voyage-ai-provider";
+import { Effect, Schedule } from "effect";
+import { GlobalStore } from "../storage/global-store";
+import { ApiKeyMissingError, EmbeddingError } from "../types";
+
+export namespace Embedder {
+  const getVoyager = GlobalStore.getApiKeyOrFail.pipe(
+    Effect.map((apiKey) => createVoyage({ apiKey }))
+  );
+
+  export const embedQuery = (query: string) =>
+    getVoyager.pipe(
+      Effect.flatMap((voyager) =>
+        Effect.tryPromise({
+          try: () =>
+            AI.embed({
+              model: voyager.textEmbeddingModel("voyage-code-3"),
+              value: query,
+            }),
+          catch: (error) =>
+            new EmbeddingError({
+              message: "Failed to embed query",
+              cause: error,
+            }),
+        })
+      ),
+      Effect.map((result) => result.embedding)
+    );
+
+  export const embedMany = (values: string[]) => {
+    if (values.length === 0) {
+      return Effect.succeed({ embeddings: [] as number[][], tokens: 0 });
+    }
+
+    return getVoyager.pipe(
+      Effect.flatMap((voyager) =>
+        Effect.tryPromise({
+          try: () =>
+            AI.embedMany({
+              model: voyager.textEmbeddingModel("voyage-code-3"),
+              values,
+            }),
+          catch: (error) =>
+            new EmbeddingError({
+              message: "Failed to embed batch",
+              cause: error,
+            }),
+        })
+      ),
+      Effect.map((result) => ({
+        embeddings: result.embeddings,
+        tokens: result.usage?.tokens ?? 0,
+      })),
+      Effect.retry({
+        schedule: Schedule.exponential("1 second", 2),
+        times: 3,
+      }),
+      Effect.catchTag("EmbeddingError", (error) => {
+        console.error(`Embedding failed after retries: ${error.message}`);
+        return Effect.succeed({
+          embeddings: values.map(() => [] as number[]),
+          tokens: 0,
+        });
+      })
+    );
+  };
+
+  export const cosineSimilarity = (a: number[], b: number[]): number => {
+    if (a.length === 0 || b.length === 0) return 0;
+
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+      const ai = a[i] ?? 0;
+      const bi = b[i] ?? 0;
+      dotProduct += ai * bi;
+      normA += ai * ai;
+      normB += bi * bi;
+    }
+
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    if (denominator === 0) return 0;
+
+    return dotProduct / denominator;
+  };
+}

package/src/core/indexer.ts

@@ -0,0 +1,343 @@
+import path from "path";
+import * as crypto from "crypto";
+import { chunkBatchStreamEffect } from "code-chunk";
+import { Effect, Ref, Schedule, Fiber, Console } from "effect";
+import * as Chunk from "effect/Chunk";
+import * as Stream from "effect/Stream";
+
+import {
+  type ChunkWithMeta,
+  type ChunkWithEmbedding,
+  type ChunkReference,
+  type ProjectManifest,
+  FileReadError,
+} from "../types";
+import { Embedder } from "./embedder";
+import { EmbeddingCache } from "../storage/embedding-cache";
+import { GlobalStore } from "../storage/global-store";
+import { GitTracker } from "../watcher/git-tracker";
+
+interface IndexStats {
+  filesRead: Ref.Ref<number>;
+  chunksCreated: Ref.Ref<number>;
+  cacheHits: Ref.Ref<number>;
+  embedded: Ref.Ref<number>;
+  processed: Ref.Ref<number>;
+  skippedFiles: Ref.Ref<string[]>;
+}
+
+export namespace Indexer {
+  export const filterFilePath = GitTracker.filterFilePath;
+
+  export const hashChunk = (text: string): string => {
+    return crypto.createHash("sha256").update(text).digest("hex");
+  };
+
+  export const toChunkReference = (
+    chunk: ChunkWithEmbedding
+  ): ChunkReference => {
+    return {
+      chunkHash: chunk.hash,
+      filepath: chunk.filepath,
+      lineRange: chunk.lineRange,
+      text: chunk.text,
+    };
+  };
+
+  export const getFilePaths = (pwd: string) => GitTracker.getTrackedFiles(pwd);
+
+  export const readFile = (pwd: string, filepath: string) =>
+    Effect.tryPromise({
+      try: () => Bun.file(`${pwd}/${filepath}`).text(),
+      catch: (error) =>
+        new FileReadError({
+          path: `${pwd}/${filepath}`,
+          cause: error,
+        }),
+    });
+
+  const logProgress = (stats: IndexStats) =>
+    Effect.gen(function* () {
+      const filesRead = yield* Ref.get(stats.filesRead);
+      const chunksCreated = yield* Ref.get(stats.chunksCreated);
+      const cacheHits = yield* Ref.get(stats.cacheHits);
+      const embedded = yield* Ref.get(stats.embedded);
+      const processed = yield* Ref.get(stats.processed);
+      const skippedFiles = yield* Ref.get(stats.skippedFiles);
+
+      yield* Console.log(
+        `[Progress] Files: ${filesRead} | Skipped: ${skippedFiles.length} | Chunks: ${chunksCreated} | Cache hits: ${cacheHits} | Embedded: ${embedded} | Processed: ${processed}`
+      );
+    });
+
+  const processBatch = (chunks: readonly ChunkWithMeta[], stats: IndexStats) =>
+    Effect.gen(function* () {
+      const chunksWithCache = yield* Effect.all(
+        chunks.map((chunk) =>
+          EmbeddingCache.get(chunk.hash).pipe(
+            Effect.map((cached) => ({ ...chunk, cached }))
+          )
+        )
+      );
+
+      const cachedChunks = chunksWithCache.filter(
+        (c): c is typeof c & { cached: number[] } => c.cached !== null
+      );
+      const uncachedChunks = chunksWithCache.filter((c) => c.cached === null);
+
+      if (cachedChunks.length > 0) {
+        yield* Ref.update(stats.cacheHits, (n) => n + cachedChunks.length);
+      }
+
+      let embeddedChunks: ChunkWithEmbedding[] = [];
+      if (uncachedChunks.length > 0) {
+        const result = yield* Embedder.embedMany(
+          uncachedChunks.map((c) => c.text)
+        );
+
+        embeddedChunks = uncachedChunks.map((chunk, index) => ({
+          filepath: chunk.filepath,
+          hash: chunk.hash,
+          text: chunk.text,
+          lineRange: chunk.lineRange,
+          embedding: result.embeddings[index] ?? [],
+        }));
+
+        yield* Ref.update(stats.embedded, (n) => n + embeddedChunks.length);
+
+        yield* Effect.all(
+          embeddedChunks.map((chunk) =>
+            EmbeddingCache.save(chunk.hash, chunk.embedding)
+          )
+        );
+      }
+
+      const finalChunks: ChunkWithEmbedding[] = [
+        ...cachedChunks.map((c) => ({
+          filepath: c.filepath,
+          hash: c.hash,
+          text: c.text,
+          lineRange: c.lineRange,
+          embedding: c.cached,
+        })),
+        ...embeddedChunks,
+      ];
+
+      return finalChunks;
+    });
+
+  export const indexProject = (pwd: string) =>
+    Effect.gen(function* () {
+      yield* GlobalStore.ensureDirectories;
+
+      const stats: IndexStats = {
+        filesRead: yield* Ref.make(0),
+        chunksCreated: yield* Ref.make(0),
+        cacheHits: yield* Ref.make(0),
+        embedded: yield* Ref.make(0),
+        processed: yield* Ref.make(0),
+        skippedFiles: yield* Ref.make<string[]>([]),
+      };
+
+      const logFiber = yield* logProgress(stats).pipe(
+        Effect.repeat(Schedule.fixed("1 second")),
+        Effect.forkDaemon
+      );
+
+      const filePaths = yield* getFilePaths(pwd);
+      const fileResults = yield* Effect.all(
+        filePaths.map((filepath) =>
+          readFile(pwd, filepath).pipe(
+            Effect.tap(() => Ref.update(stats.filesRead, (n) => n + 1)),
+            Effect.map((code) => ({ filepath, code, skipped: false as const })),
+            Effect.catchAll(() =>
+              Ref.update(stats.skippedFiles, (arr) => [...arr, filepath]).pipe(
+                Effect.map(() => ({
+                  filepath,
+                  code: "",
+                  skipped: true as const,
+                }))
+              )
+            )
+          )
+        ),
+        { concurrency: "unbounded" }
+      );
+
+      const files = fileResults.filter((f) => !f.skipped);
+
+      yield* Console.log(
+        `[Indexing] Read ${files.length} files, starting chunking...`
+      );
+
+      const chunksStream = yield* chunkBatchStreamEffect(files, {
+        maxChunkSize: 4000,
+        concurrency: Infinity,
+      }).pipe(
+        Stream.flatMap((result) => {
+          if (result.error !== null || result.chunks === null) {
+            return Stream.empty;
+          }
+          return Stream.fromIterable(
+            result.chunks.map((chunk) => ({
+              filepath: result.filepath,
+              hash: hashChunk(chunk.contextualizedText),
+              text: chunk.contextualizedText,
+              lineRange: chunk.lineRange,
+            }))
+          );
+        }),
+        Stream.tap(() => Ref.update(stats.chunksCreated, (n) => n + 1)),
+        Stream.grouped(5),
+        Stream.mapEffect(
+          (batch) => processBatch(Chunk.toReadonlyArray(batch), stats),
+          { concurrency: 50 }
+        ),
+        Stream.flatMap((chunks) => Stream.fromIterable(chunks)),
+        Stream.tap(() => Ref.update(stats.processed, (n) => n + 1)),
+        Stream.runCollect,
+        Effect.map(Chunk.toReadonlyArray)
+      );
+
+      yield* Fiber.interrupt(logFiber);
+      yield* logProgress(stats);
+
+      const skippedFiles = yield* Ref.get(stats.skippedFiles);
+      if (skippedFiles.length > 0) {
+        yield* Console.log(
+          `[Skipped Files] Could not read ${skippedFiles.length} files`
+        );
+      }
+
+      yield* Console.log(
+        `[Complete] Indexed ${chunksStream.length} chunks total`
+      );
+
+      return chunksStream;
+    });
+
+  export const indexFiles = (pwd: string, filepaths: string[]) =>
+    Effect.gen(function* () {
+      yield* GlobalStore.ensureDirectories;
+
+      const filtered = filepaths.filter(filterFilePath);
+      if (filtered.length === 0) {
+        return [];
+      }
+
+      yield* Console.log(`[Indexing] Processing ${filtered.length} files...`);
+
+      const fileResults = yield* Effect.all(
+        filtered.map((filepath) =>
+          readFile(pwd, filepath).pipe(
+            Effect.map((code) => ({ filepath, code, skipped: false as const })),
+            Effect.catchAll(() =>
+              Effect.succeed({ filepath, code: "", skipped: true as const })
+            )
+          )
+        ),
+        { concurrency: "unbounded" }
+      );
+
+      const files = fileResults.filter((f) => !f.skipped);
+
+      const stats: IndexStats = {
+        filesRead: yield* Ref.make(files.length),
+        chunksCreated: yield* Ref.make(0),
+        cacheHits: yield* Ref.make(0),
+        embedded: yield* Ref.make(0),
+        processed: yield* Ref.make(0),
+        skippedFiles: yield* Ref.make<string[]>([]),
+      };
+
+      const chunksStream = yield* chunkBatchStreamEffect(files, {
+        maxChunkSize: 4000,
+        concurrency: Infinity,
+      }).pipe(
+        Stream.flatMap((result) => {
+          if (result.error !== null || result.chunks === null) {
+            return Stream.empty;
+          }
+          return Stream.fromIterable(
+            result.chunks.map((chunk) => ({
+              filepath: result.filepath,
+              hash: hashChunk(chunk.contextualizedText),
+              text: chunk.contextualizedText,
+              lineRange: chunk.lineRange,
+            }))
+          );
+        }),
+        Stream.tap(() => Ref.update(stats.chunksCreated, (n) => n + 1)),
+        Stream.grouped(5),
+        Stream.mapEffect(
+          (batch) => processBatch(Chunk.toReadonlyArray(batch), stats),
+          { concurrency: 50 }
+        ),
+        Stream.flatMap((chunks) => Stream.fromIterable(chunks)),
+        Stream.runCollect,
+        Effect.map(Chunk.toReadonlyArray)
+      );
+
+      yield* Console.log(
+        `[Complete] Indexed ${chunksStream.length} chunks from ${files.length} files`
+      );
+
+      return chunksStream;
+    });
+
+  export const saveIndexedChunks = (
+    pwd: string,
+    chunks: ChunkWithEmbedding[],
+    gitCommitHash?: string
+  ) => {
+    const projectId = GlobalStore.getProjectId(pwd);
+
+    const manifest: ProjectManifest = {
+      projectId,
+      projectPath: pwd,
+      lastIndexedAt: new Date().toISOString(),
+      gitCommitHash,
+      chunks: chunks.map(toChunkReference),
+    };
+
+    return GlobalStore.saveProjectManifest(manifest);
+  };
+
+  export const updateManifestWithChanges = (
+    pwd: string,
+    addedOrModifiedChunks: ChunkWithEmbedding[],
+    deletedFilepaths: string[],
+    gitCommitHash?: string
+  ) =>
+    Effect.gen(function* () {
+      const projectId = GlobalStore.getProjectId(pwd);
+      const existingManifest = yield* GlobalStore.loadProjectManifest(
+        projectId
+      );
+
+      let existingChunks: ChunkReference[] = [];
+      if (existingManifest) {
+        existingChunks = existingManifest.chunks.filter(
+          (c) => !deletedFilepaths.includes(c.filepath)
+        );
+        const modifiedFilepaths = new Set(
+          addedOrModifiedChunks.map((c) => c.filepath)
+        );
+        existingChunks = existingChunks.filter(
+          (c) => !modifiedFilepaths.has(c.filepath)
+        );
+      }
+
+      const newChunks = addedOrModifiedChunks.map(toChunkReference);
+
+      const manifest: ProjectManifest = {
+        projectId,
+        projectPath: pwd,
+        lastIndexedAt: new Date().toISOString(),
+        gitCommitHash,
+        chunks: [...existingChunks, ...newChunks],
+      };
+
+      yield* GlobalStore.saveProjectManifest(manifest);
+    });
+}

package/src/storage/embedding-cache.ts

@@ -0,0 +1,71 @@
+import { Effect } from "effect";
+import { EMBEDDINGS_DIR, FileReadError } from "../types";
+import { GlobalStore } from "./global-store";
+
+export namespace EmbeddingCache {
+  export const get = (hash: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const embeddingPath = `${EMBEDDINGS_DIR}/${hash}.json`;
+        const file = Bun.file(embeddingPath);
+        if (await file.exists()) {
+          return (await file.json()) as number[];
+        }
+        return null;
+      },
+      catch: (error) =>
+        new FileReadError({
+          path: `${EMBEDDINGS_DIR}/${hash}.json`,
+          cause: error,
+        }),
+    });
+
+  export const save = (hash: string, embedding: number[]) =>
+    GlobalStore.ensureDirectories.pipe(
+      Effect.flatMap(() =>
+        Effect.tryPromise({
+          try: () => {
+            const embeddingPath = `${EMBEDDINGS_DIR}/${hash}.json`;
+            return Bun.write(embeddingPath, JSON.stringify(embedding));
+          },
+          catch: (error) =>
+            new FileReadError({
+              path: `${EMBEDDINGS_DIR}/${hash}.json`,
+              cause: error,
+            }),
+        })
+      )
+    );
+
+  export const getMany = (hashes: string[]) =>
+    Effect.all(
+      hashes.map((hash) =>
+        get(hash).pipe(Effect.map((embedding) => [hash, embedding] as const))
+      ),
+      { concurrency: 1000 }
+    ).pipe(
+      Effect.map((entries) => new Map(entries) as Map<string, number[] | null>)
+    );
+
+  export const saveMany = (embeddings: Map<string, number[]>) =>
+    GlobalStore.ensureDirectories.pipe(
+      Effect.flatMap(() =>
+        Effect.all(
+          Array.from(embeddings.entries()).map(([hash, embedding]) =>
+            save(hash, embedding)
+          ),
+          { concurrency: 1000 }
+        )
+      )
+    );
+
+  export const exists = (hash: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const embeddingPath = `${EMBEDDINGS_DIR}/${hash}.json`;
+        const file = Bun.file(embeddingPath);
+        return file.exists();
+      },
+      catch: () => false,
+    }).pipe(Effect.catchAll(() => Effect.succeed(false)));
+}

package/src/storage/global-store.ts

@@ -0,0 +1,208 @@
+import * as crypto from "crypto";
+import { Effect } from "effect";
+import {
+  SEMANTIC_HOME,
+  EMBEDDINGS_DIR,
+  PROJECTS_DIR,
+  CONFIG_PATH,
+  type GlobalConfig,
+  type ProjectManifest,
+  type GitTreeCache,
+  ConfigError,
+  ApiKeyMissingError,
+} from "../types";
+
+const DEFAULT_CONFIG: GlobalConfig = {
+  version: "1.0.0",
+};
+
+export namespace GlobalStore {
+  export const getProjectId = (projectPath: string): string => {
+    const hash = crypto.createHash("sha256").update(projectPath).digest("hex");
+  };
+
+  export const getProjectDir = (projectId: string): string => {
+    return `${PROJECTS_DIR}/${projectId}`;
+  };
+
+  export const clearProjectCache = (projectId: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const projectDir = getProjectDir(projectId);
+        const manifestPath = `${projectDir}/manifest.json`;
+        const gitTreePath = `${projectDir}/git-tree.json`;
+
+        const manifestExists = await Bun.file(manifestPath).exists();
+        const gitTreeExists = await Bun.file(gitTreePath).exists();
+
+        if (manifestExists) {
+          await Bun.$`rm -f ${manifestPath}`;
+        }
+        if (gitTreeExists) {
+          await Bun.$`rm -f ${gitTreePath}`;
+        }
+
+        return {
+          manifestCleared: manifestExists,
+          gitTreeCleared: gitTreeExists,
+        };
+      },
+      catch: (error) =>
+        new ConfigError({
+          message: `Failed to clear repo cache for project ${projectId}`,
+          cause: error,
+        }),
+    });
+
+  export const ensureDirectories = Effect.tryPromise({
+    try: async () => {
+      await Bun.$`mkdir -p ${SEMANTIC_HOME}`;
+      await Bun.$`mkdir -p ${EMBEDDINGS_DIR}`;
+      await Bun.$`mkdir -p ${PROJECTS_DIR}`;
+    },
+    catch: (error) =>
+      new ConfigError({
+        message: "Failed to create directories",
+        cause: error,
+      }),
+  });
+
+  export const ensureProjectDir = (projectId: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const projectDir = getProjectDir(projectId);
+        await Bun.$`mkdir -p ${projectDir}`;
+      },
+      catch: (error) =>
+        new ConfigError({
+          message: `Failed to create project directory for ${projectId}`,
+          cause: error,
+        }),
+    });
+
+  export const loadConfig = Effect.tryPromise({
+    try: async () => {
+      const file = Bun.file(CONFIG_PATH);
+      if (await file.exists()) {
+        return (await file.json()) as GlobalConfig;
+      }
+      return DEFAULT_CONFIG;
+    },
+    catch: (error) =>
+      new ConfigError({
+        message: "Failed to load config",
+        cause: error,
+      }),
+  });
+
+  export const saveConfig = (config: GlobalConfig) =>
+    ensureDirectories.pipe(
+      Effect.flatMap(() =>
+        Effect.tryPromise({
+          try: () => Bun.write(CONFIG_PATH, JSON.stringify(config, null, 2)),
+          catch: (error) =>
+            new ConfigError({
+              message: "Failed to save config",
+              cause: error,
+            }),
+        })
+      )
+    );
+
+  export const setApiKey = (apiKey: string) =>
+    loadConfig.pipe(
+      Effect.map((config) => ({ ...config, voyageApiKey: apiKey })),
+      Effect.flatMap(saveConfig)
+    );
+
+  export const getApiKey = loadConfig.pipe(
+    Effect.map((config) => config.voyageApiKey)
+  );
+
+  export const getApiKeyOrFail = loadConfig.pipe(
+    Effect.flatMap((config) =>
+      config.voyageApiKey
+        ? Effect.succeed(config.voyageApiKey)
+        : Effect.fail(
+            new ApiKeyMissingError({
+              message:
+                "No API key configured. Run: semantic config add-api-key <your_key>",
+            })
+          )
+    )
+  );
+
+  export const loadProjectManifest = (projectId: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const manifestPath = `${getProjectDir(projectId)}/manifest.json`;
+        const file = Bun.file(manifestPath);
+        if (await file.exists()) {
+          return (await file.json()) as ProjectManifest;
+        }
+        return null;
+      },
+      catch: (error) =>
+        new ConfigError({
+          message: `Failed to load manifest for project ${projectId}`,
+          cause: error,
+        }),
+    });
+
+  export const saveProjectManifest = (manifest: ProjectManifest) =>
+    ensureProjectDir(manifest.projectId).pipe(
+      Effect.flatMap(() =>
+        Effect.tryPromise({
+          try: () => {
+            const projectDir = getProjectDir(manifest.projectId);
+            return Bun.write(
+              `${projectDir}/manifest.json`,
+              JSON.stringify(manifest, null, 2)
+            );
+          },
+          catch: (error) =>
+            new ConfigError({
+              message: `Failed to save manifest for project ${manifest.projectId}`,
+              cause: error,
+            }),
+        })
+      )
+    );
+
+  export const loadGitTreeCache = (projectId: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const cachePath = `${getProjectDir(projectId)}/git-tree.json`;
+        const file = Bun.file(cachePath);
+        if (await file.exists()) {
+          return (await file.json()) as GitTreeCache;
+        }
+        return null;
+      },
+      catch: (error) =>
+        new ConfigError({
+          message: `Failed to load git tree cache for project ${projectId}`,
+          cause: error,
+        }),
+    });
+
+  export const saveGitTreeCache = (projectId: string, cache: GitTreeCache) =>
+    ensureProjectDir(projectId).pipe(
+      Effect.flatMap(() =>
+        Effect.tryPromise({
+          try: () => {
+            const projectDir = getProjectDir(projectId);
+            return Bun.write(
+              `${projectDir}/git-tree.json`,
+              JSON.stringify(cache, null, 2)
+            );
+          },
+          catch: (error) =>
+            new ConfigError({
+              message: `Failed to save git tree cache for project ${projectId}`,
+              cause: error,
+            }),
+        })
+      )
+    );
+}