semantic-grep 0.0.1

package/src/types.ts

@@ -0,0 +1,132 @@
+import { Data } from "effect";
+
+export class ApiKeyMissingError extends Data.TaggedError("ApiKeyMissingError")<{
+  message: string;
+}> {}
+
+export class EmbeddingError extends Data.TaggedError("EmbeddingError")<{
+  message: string;
+  cause?: unknown;
+}> {}
+
+export class FileNotFoundError extends Data.TaggedError("FileNotFoundError")<{
+  path: string;
+}> {}
+
+export class FileReadError extends Data.TaggedError("FileReadError")<{
+  path: string;
+  cause?: unknown;
+}> {}
+
+export class GitError extends Data.TaggedError("GitError")<{
+  command: string;
+  message: string;
+}> {}
+
+export class ManifestNotFoundError extends Data.TaggedError(
+  "ManifestNotFoundError"
+)<{
+  projectId: string;
+}> {}
+
+export class ConfigError extends Data.TaggedError("ConfigError")<{
+  message: string;
+  cause?: unknown;
+}> {}
+
+export class IndexingError extends Data.TaggedError("IndexingError")<{
+  message: string;
+  cause?: unknown;
+}> {}
+
+export interface GlobalConfig {
+  voyageApiKey?: string;
+  version: string;
+}
+
+export interface ProjectManifest {
+  projectId: string;
+  projectPath: string;
+  lastIndexedAt: string;
+  gitCommitHash?: string;
+  chunks: ChunkReference[];
+}
+
+export interface ChunkReference {
+  chunkHash: string;
+  filepath: string;
+  lineRange: { start: number; end: number };
+  text: string;
+}
+
+export interface GitTreeCache {
+  commitHash: string;
+  fileHashes: Record<string, string>;
+}
+
+export interface ChunkWithMeta {
+  filepath: string;
+  hash: string;
+  text: string;
+  lineRange: { start: number; end: number };
+}
+
+export interface ChunkWithEmbedding extends ChunkWithMeta {
+  embedding: number[];
+}
+
+export interface FileChange {
+  filepath: string;
+  changeType: "added" | "modified" | "deleted";
+}
+
+export const SEMANTIC_HOME = `${process.env.HOME}/.semantic`;
+export const EMBEDDINGS_DIR = `${SEMANTIC_HOME}/embeddings`;
+export const PROJECTS_DIR = `${SEMANTIC_HOME}/projects`;
+export const CONFIG_PATH = `${SEMANTIC_HOME}/config.json`;
+
+export const TEXT_EXTENSIONS = new Set([
+  ".md",
+  ".mdx",
+  ".txt",
+  ".rst",
+  ".adoc",
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".java",
+  ".kt",
+  ".go",
+  ".rs",
+  ".rb",
+  ".php",
+  ".cpp",
+  ".c",
+  ".h",
+  ".cs",
+  ".json",
+  ".yml",
+  ".yaml",
+  ".toml",
+  ".ini",
+  ".sql",
+]);
+
+export const IRRELEVANT_FILES = new Set([
+  ".gitignore",
+  ".git",
+  ".github",
+  ".vscode",
+  ".DS_Store",
+  ".env",
+  ".env.local",
+  "pnpm-lock.yaml",
+  "package-lock.json",
+  "yarn.lock",
+  "bun.lockb",
+  "bun.lock",
+]);

package/src/watcher/file-watcher.ts

@@ -0,0 +1,119 @@
+import { watch, type FSWatcher } from "fs";
+import path from "path";
+import { Effect } from "effect";
+import { GitTracker } from "./git-tracker";
+
+export interface WatcherOptions {
+  debounceMs?: number;
+  onChanges: (changedFiles: string[]) => Promise<void>;
+}
+
+export interface FileWatcher {
+  start: () => void;
+  stop: () => void;
+}
+
+export function createFileWatcher(
+  pwd: string,
+  options: WatcherOptions
+): FileWatcher {
+  const debounceMs = options.debounceMs ?? 500;
+
+  let watcher: FSWatcher | null = null;
+  let pendingChanges = new Set<string>();
+  let debounceTimer: Timer | null = null;
+  let isProcessing = false;
+
+  const flushChanges = async () => {
+    if (pendingChanges.size === 0 || isProcessing) {
+      return;
+    }
+
+    isProcessing = true;
+    const filesToProcess = Array.from(pendingChanges);
+    pendingChanges.clear();
+
+    const trackedFiles: string[] = [];
+    for (const filepath of filesToProcess) {
+      const isTracked = await Effect.runPromise(
+        GitTracker.isTracked(pwd, filepath)
+      );
+      if (isTracked && GitTracker.filterFilePath(filepath)) {
+        trackedFiles.push(filepath);
+      }
+    }
+
+    if (trackedFiles.length > 0) {
+      try {
+        await options.onChanges(trackedFiles);
+      } catch (error) {
+        console.error("[Watcher] Error processing changes:", error);
+      }
+    }
+
+    isProcessing = false;
+
+    if (pendingChanges.size > 0) {
+      scheduleFlush();
+    }
+  };
+
+  const scheduleFlush = () => {
+    if (debounceTimer) {
+      clearTimeout(debounceTimer);
+    }
+    debounceTimer = setTimeout(flushChanges, debounceMs);
+  };
+
+  const handleChange = (event: string, filename: string | null) => {
+    if (!filename) return;
+
+    const filepath = filename.startsWith(pwd)
+      ? path.relative(pwd, filename)
+      : filename;
+
+    if (filepath.startsWith(".") || filepath.includes("/.")) {
+      return;
+    }
+
+    if (filepath.includes("node_modules")) {
+      return;
+    }
+
+    pendingChanges.add(filepath);
+    scheduleFlush();
+  };
+
+  const start = () => {
+    if (watcher) {
+      return;
+    }
+
+    try {
+      watcher = watch(pwd, { recursive: true }, handleChange);
+
+      watcher.on("error", (error) => {
+        console.error("[Watcher] Error:", error);
+      });
+
+      console.log(`[Watcher] Watching ${pwd} for changes...`);
+    } catch (error) {
+      console.error("[Watcher] Failed to start:", error);
+    }
+  };
+
+  const stop = () => {
+    if (debounceTimer) {
+      clearTimeout(debounceTimer);
+      debounceTimer = null;
+    }
+
+    if (watcher) {
+      watcher.close();
+      watcher = null;
+      console.log("[Watcher] Stopped watching");
+    }
+  };
+
+  return { start, stop };
+}

package/src/watcher/git-tracker.ts

@@ -0,0 +1,177 @@
+import simpleGit from "simple-git";
+import { Effect } from "effect";
+import path from "path";
+import {
+  type FileChange,
+  type GitTreeCache,
+  GitError,
+  TEXT_EXTENSIONS,
+  IRRELEVANT_FILES,
+} from "../types";
+import { GlobalStore } from "../storage/global-store";
+
+export namespace GitTracker {
+  const getGit = (pwd: string) => simpleGit(pwd);
+
+  export const filterFilePath = (filepath: string): boolean => {
+    return (
+      TEXT_EXTENSIONS.has(path.extname(filepath)) &&
+      !IRRELEVANT_FILES.has(path.basename(filepath))
+    );
+  };
+
+  export const detectChanges = (
+    cached: Record<string, string>,
+    current: Record<string, string>
+  ): FileChange[] => {
+    const changes: FileChange[] = [];
+
+    for (const [filepath, hash] of Object.entries(cached)) {
+      const currentHash = current[filepath];
+      if (!currentHash) {
+        changes.push({ filepath, changeType: "deleted" });
+      } else if (currentHash !== hash) {
+        changes.push({ filepath, changeType: "modified" });
+      }
+    }
+
+    for (const filepath of Object.keys(current)) {
+      if (!(filepath in cached)) {
+        changes.push({ filepath, changeType: "added" });
+      }
+    }
+
+    return changes;
+  };
+
+  export const getCurrentCommitHash = (pwd: string) =>
+    Effect.tryPromise({
+      try: () => getGit(pwd).revparse(["HEAD"]),
+      catch: (error) =>
+        new GitError({
+          command: "git rev-parse HEAD",
+          message: String(error),
+        }),
+    }).pipe(
+      Effect.map((hash) => hash.trim()),
+      Effect.catchAll(() => Effect.succeed(null as string | null))
+    );
+
+  export const getTrackedFiles = (pwd: string) =>
+    Effect.tryPromise({
+      try: () => getGit(pwd).raw(["ls-files"]),
+      catch: (error) =>
+        new GitError({
+          command: "git ls-files",
+          message: String(error),
+        }),
+    }).pipe(
+      Effect.map((output) =>
+        output
+          .trim()
+          .split("\n")
+          .filter((f) => f.length > 0)
+          .filter(filterFilePath)
+      )
+    );
+
+  export const getGitTree = (pwd: string) =>
+    Effect.tryPromise({
+      try: () =>
+        getGit(pwd).raw([
+          "ls-tree",
+          "-r",
+          "--format=%(objectname)\t%(path)",
+          "HEAD",
+        ]),
+      catch: (error) =>
+        new GitError({
+          command: "git ls-tree",
+          message: String(error),
+        }),
+    }).pipe(
+      Effect.map((output) => parseGitTreeOutput(output)),
+      Effect.catchAll(() => getGitTreeFromLsFiles(pwd))
+    );
+
+  const parseGitTreeOutput = (output: string): Record<string, string> => {
+    const fileHashes: Record<string, string> = {};
+
+    for (const line of output.trim().split("\n")) {
+      if (!line) continue;
+      const [hash, filepath] = line.split("\t");
+      if (hash && filepath && filterFilePath(filepath)) {
+        fileHashes[filepath] = hash;
+      }
+    }
+
+    return fileHashes;
+  };
+
+  const getGitTreeFromLsFiles = (pwd: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const git = getGit(pwd);
+        const output = await git.raw(["ls-files"]);
+        const fileHashes: Record<string, string> = {};
+
+        for (const filepath of output.trim().split("\n")) {
+          if (filepath && filterFilePath(filepath)) {
+            try {
+              const stat = await Bun.file(`${pwd}/${filepath}`).stat();
+              fileHashes[filepath] = stat?.mtimeMs?.toString() ?? "unknown";
+            } catch {
+              fileHashes[filepath] = "unknown";
+            }
+          }
+        }
+
+        return fileHashes;
+      },
+      catch: (error) =>
+        new GitError({
+          command: "git ls-files (fallback)",
+          message: String(error),
+        }),
+    });
+
+  export const isTracked = (pwd: string, filepath: string) =>
+    Effect.tryPromise({
+      try: () => getGit(pwd).raw(["ls-files", filepath]),
+      catch: () => "",
+    }).pipe(
+      Effect.map((output) => output.trim().length > 0),
+      Effect.catchAll(() => Effect.succeed(false))
+    );
+
+  export const getFilesToIndex = (pwd: string) =>
+    Effect.gen(function* () {
+      const projectId = GlobalStore.getProjectId(pwd);
+      const commitHash = yield* getCurrentCommitHash(pwd);
+      const currentFileHashes = yield* getGitTree(pwd);
+
+      const currentTree: GitTreeCache = {
+        commitHash: commitHash ?? "no-commit",
+        fileHashes: currentFileHashes,
+      };
+
+      const cachedTree = yield* GlobalStore.loadGitTreeCache(projectId);
+
+      if (!cachedTree) {
+        const allFiles = Object.keys(currentFileHashes).map((filepath) => ({
+          filepath,
+          changeType: "added" as const,
+        }));
+        return { changes: allFiles, currentTree, isFullReindex: true };
+      }
+
+      const changes = detectChanges(cachedTree.fileHashes, currentFileHashes);
+
+      return { changes, currentTree, isFullReindex: false };
+    });
+
+  export const saveGitTree = (pwd: string, tree: GitTreeCache) => {
+    const projectId = GlobalStore.getProjectId(pwd);
+    return GlobalStore.saveGitTreeCache(projectId, tree);
+  };
+}