npm - semantic-grep - Versions diffs - 0.0.1 - Mend

semantic-grep 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (18) hide show

package/CLAUDE.md +111 -0
package/README.md +15 -0
package/bin/semantic.ts +7 -0
package/index.ts +8 -0
package/package.json +29 -0
package/search.ts +69 -0
package/src/cli/commands/cache.ts +33 -0
package/src/cli/commands/config.ts +34 -0
package/src/cli/commands/search.ts +109 -0
package/src/cli/commands/watch.ts +159 -0
package/src/cli/index.ts +63 -0
package/src/core/embedder.ts +89 -0
package/src/core/indexer.ts +343 -0
package/src/storage/embedding-cache.ts +71 -0
package/src/storage/global-store.ts +208 -0
package/src/types.ts +132 -0
package/src/watcher/file-watcher.ts +119 -0
package/src/watcher/git-tracker.ts +177 -0

package/CLAUDE.md ADDED Viewed

@@ -0,0 +1,111 @@
+---
+description: Use Bun instead of Node.js, npm, pnpm, or vite.
+globs: "*.ts, *.tsx, *.html, *.css, *.js, *.jsx, package.json"
+alwaysApply: false
+---
+Default to using Bun instead of Node.js.
+- Use `bun <file>` instead of `node <file>` or `ts-node <file>`
+- Use `bun test` instead of `jest` or `vitest`
+- Use `bun build <file.html|file.ts|file.css>` instead of `webpack` or `esbuild`
+- Use `bun install` instead of `npm install` or `yarn install` or `pnpm install`
+- Use `bun run <script>` instead of `npm run <script>` or `yarn run <script>` or `pnpm run <script>`
+- Use `bunx <package> <command>` instead of `npx <package> <command>`
+- Bun automatically loads .env, so don't use dotenv.
+## APIs
+- `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`.
+- `bun:sqlite` for SQLite. Don't use `better-sqlite3`.
+- `Bun.redis` for Redis. Don't use `ioredis`.
+- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`.
+- `WebSocket` is built-in. Don't use `ws`.
+- Prefer `Bun.file` over `node:fs`'s readFile/writeFile
+- Bun.$`ls` instead of execa.
+## Testing
+Use `bun test` to run tests.
+```ts#index.test.ts
+import { test, expect } from "bun:test";
+test("hello world", () => {
+  expect(1).toBe(1);
+});
+```
+## Frontend
+Use HTML imports with `Bun.serve()`. Don't use `vite`. HTML imports fully support React, CSS, Tailwind.
+Server:
+```ts#index.ts
+import index from "./index.html"
+Bun.serve({
+  routes: {
+    "/": index,
+    "/api/users/:id": {
+      GET: (req) => {
+        return new Response(JSON.stringify({ id: req.params.id }));
+      },
+    },
+  },
+  // optional websocket support
+  websocket: {
+    open: (ws) => {
+      ws.send("Hello, world!");
+    },
+    message: (ws, message) => {
+      ws.send(message);
+    },
+    close: (ws) => {
+      // handle close
+    }
+  },
+  development: {
+    hmr: true,
+    console: true,
+  }
+})
+```
+HTML files can import .tsx, .jsx or .js files directly and Bun's bundler will transpile & bundle automatically. `<link>` tags can point to stylesheets and Bun's CSS bundler will bundle.
+```html#index.html
+<html>
+  <body>
+    <h1>Hello, world!</h1>
+    <script type="module" src="./frontend.tsx"></script>
+  </body>
+</html>
+```
+With the following `frontend.tsx`:
+```tsx#frontend.tsx
+import React from "react";
+import { createRoot } from "react-dom/client";
+// import .css files directly and it works
+import './index.css';
+const root = createRoot(document.body);
+export default function Frontend() {
+  return <h1>Hello, world!</h1>;
+}
+root.render(<Frontend />);
+```
+Then, run index.ts
+```sh
+bun --hot ./index.ts
+```
+For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`.

package/README.md ADDED Viewed

@@ -0,0 +1,15 @@
+# semantic-search
+To install dependencies:
+```bash
+bun install
+```
+To run:
+```bash
+bun run index.ts
+```
+This project was created using `bun init` in bun v1.3.5. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.

package/bin/semantic.ts ADDED Viewed

@@ -0,0 +1,7 @@
+#!/usr/bin/env bun
+import { main } from "../src/cli/index";
+main().catch((error) => {
+  console.error("Fatal error:", error);
+  process.exit(1);
+});

package/index.ts ADDED Viewed

@@ -0,0 +1,8 @@
+export { GlobalStore } from "./src/storage/global-store";
+export { EmbeddingCache } from "./src/storage/embedding-cache";
+export { Embedder } from "./src/core/embedder";
+export { Indexer } from "./src/core/indexer";
+export { GitTracker } from "./src/watcher/git-tracker";
+export { createFileWatcher } from "./src/watcher/file-watcher";
+export * from "./src/types";

package/package.json ADDED Viewed

@@ -0,0 +1,29 @@
+{
+  "name": "semantic-grep",
+  "version": "0.0.1",
+  "module": "index.ts",
+  "type": "module",
+  "bin": {
+    "semantic": "./bin/semantic.ts"
+  },
+  "scripts": {
+    "start": "bun run bin/semantic.ts",
+    "semantic": "bun run bin/semantic.ts"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "tsx": "^4.21.0"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "ai": "^6.0.6",
+    "code-chunk": "/Users/nisarg/Documents/experiments/code-chunk/packages/code-chunk/code-chunk-0.1.12.tgz",
+    "commander": "^14.0.3",
+    "effect": "^3.19.14",
+    "simple-git": "^3.30.0",
+    "voyage-ai-provider": "^3.0.0",
+    "zod": "^4.3.6"
+  }
+}

package/search.ts ADDED Viewed

@@ -0,0 +1,69 @@
+import { Command } from "commander";
+import { Effect, Console } from "effect";
+import { Embedder } from "./src/core/embedder";
+import type { ChunkWithEmbedding } from "./src/types";
+const program = new Command();
+program
+  .name("search")
+  .description("Standalone search script using final.json")
+  .argument("<query>", "Search query")
+  .option("--topk <number>", "Number of results to return", "10")
+  .option("--compact", "Compact output format", false)
+  .action(
+    async (query: string, options: { topk: string; compact: boolean }) => {
+      const topk = Math.min(20, Math.max(1, parseInt(options.topk, 10) || 10));
+      await Effect.runPromise(runSearch(query, topk, options.compact));
+    }
+  );
+const runSearch = (query: string, topk: number, compact: boolean) =>
+  Effect.gen(function* () {
+    yield* Console.log(`\nSearching for: "${query}"`);
+    yield* Console.log(`Top ${topk} results\n`);
+    const chunksFile = Bun.file("final.json");
+    const chunks: ChunkWithEmbedding[] = yield* Effect.tryPromise({
+      try: () => chunksFile.json().then((chunks) => chunks.flat()),
+      catch: (error) => new Error(`Failed to load final.json: ${error}`),
+    });
+    const queryEmbedding = yield* Embedder.embedQuery(query);
+    const results = chunks
+      .filter((chunk) => chunk.embedding && chunk.embedding.length > 0)
+      .map((chunk) => ({
+        chunk,
+        similarity: Embedder.cosineSimilarity(queryEmbedding, chunk.embedding),
+      }))
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, topk);
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]!;
+      const { chunk, similarity } = result;
+      const { filepath, lineRange } = chunk;
+      if (compact) {
+        yield* Console.log(
+          `#${i + 1} | ${similarity.toFixed(4)} | ${filepath}:${
+            lineRange.start + 1
+          }-${lineRange.end + 1}`
+        );
+        continue;
+      }
+      yield* Console.log("─".repeat(80));
+      yield* Console.log(
+        `#${i + 1} | Score: ${similarity.toFixed(4)} | ${filepath}:${
+          lineRange.start + 1
+        }-${lineRange.end + 1}`
+      );
+      yield* Console.log("─".repeat(80));
+      yield* Console.log(chunk.text);
+      yield* Console.log("");
+    }
+  });
+program.parse(process.argv);

package/src/cli/commands/cache.ts ADDED Viewed

@@ -0,0 +1,33 @@
+import { Effect, Console } from "effect";
+import { GlobalStore } from "../../storage/global-store";
+export const clearRepoCacheCommand = () =>
+  Effect.gen(function* () {
+    const pwd = process.cwd();
+    const projectId = GlobalStore.getProjectId(pwd);
+    const result = yield* GlobalStore.clearProjectCache(projectId);
+    if (!result.manifestCleared && !result.gitTreeCleared) {
+      yield* Console.log("[Cache] Repo cache already empty for this project");
+      return;
+    }
+    const clearedParts = [
+      result.manifestCleared ? "manifest" : null,
+      result.gitTreeCleared ? "git tree" : null,
+    ].filter((part): part is string => part !== null);
+    yield* Console.log(
+      `[Cache] Cleared repo cache (${clearedParts.join(
+        ", "
+      )}). Embeddings preserved.`
+    );
+  }).pipe(
+    Effect.catchTags({
+      ConfigError: (e) =>
+        Console.error(`Config error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+    })
+  );

package/src/cli/commands/config.ts ADDED Viewed

@@ -0,0 +1,34 @@
+import { Effect, Console } from "effect";
+import { GlobalStore } from "../../storage/global-store";
+export const configCommand = (
+  subcommand: "add-api-key" | "show",
+  apiKey?: string
+) =>
+  Effect.gen(function* () {
+    switch (subcommand) {
+      case "add-api-key": {
+        yield* GlobalStore.setApiKey(apiKey!);
+        yield* Console.log("API key saved successfully");
+        break;
+      }
+      case "show": {
+        const key = yield* GlobalStore.getApiKey;
+        if (key) {
+          const masked = key.slice(0, 4) + "..." + key.slice(-4);
+          yield* Console.log(`Voyage API Key: ${masked}`);
+        } else {
+          yield* Console.log("No API key configured");
+        }
+        break;
+      }
+    }
+  }).pipe(
+    Effect.catchTags({
+      ConfigError: (e) =>
+        Console.error(`Config error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+    })
+  );

package/src/cli/commands/search.ts ADDED Viewed

@@ -0,0 +1,109 @@
+import { Effect, Console } from "effect";
+import type { ChunkReference } from "../../types";
+import { GlobalStore } from "../../storage/global-store";
+import { EmbeddingCache } from "../../storage/embedding-cache";
+import { Embedder } from "../../core/embedder";
+interface SearchResult {
+  chunk: ChunkReference;
+  similarity: number;
+}
+interface SearchOptions {
+  topk: number;
+  compact: boolean;
+}
+export const searchCommand = (query: string, options: SearchOptions) =>
+  Effect.gen(function* () {
+    const { topk, compact } = options;
+    const pwd = process.cwd();
+    const projectId = GlobalStore.getProjectId(pwd);
+    const manifest = yield* GlobalStore.loadProjectManifest(projectId);
+    if (!manifest) {
+      yield* Console.error("Error: No index found for current directory");
+      yield* Console.error("Run 'semantic watch' first to index this project");
+      return yield* Effect.sync(() => process.exit(1));
+    }
+    yield* Console.log(`\nSearching for: "${query}"`);
+    yield* Console.log(`Top ${topk} results\n`);
+    const queryEmbedding = yield* Embedder.embedQuery(query);
+    const results: SearchResult[] = [];
+    for (const chunk of manifest.chunks) {
+      const embedding = yield* EmbeddingCache.get(chunk.chunkHash);
+      if (embedding && embedding.length > 0) {
+        const similarity = Embedder.cosineSimilarity(queryEmbedding, embedding);
+        results.push({ chunk, similarity });
+      }
+    }
+    results.sort((a, b) => b.similarity - a.similarity);
+    const topResults = results.slice(0, topk);
+    for (let i = 0; i < topResults.length; i++) {
+      const result = topResults[i]!;
+      const { chunk, similarity } = result;
+      const { filepath, lineRange } = chunk;
+      if (compact) {
+        yield* Console.log(
+          `#${i + 1} | ${similarity.toFixed(4)} | ${filepath}:${
+            lineRange.start + 1
+          }-${lineRange.end + 1}`
+        );
+        continue;
+      }
+      yield* Console.log("─".repeat(80));
+      yield* Console.log(
+        `#${i + 1} | Score: ${similarity.toFixed(4)} | ${filepath}:${
+          lineRange.start + 1
+        }-${lineRange.end + 1}`
+      );
+      yield* Console.log("─".repeat(80));
+      const codeBlock = yield* Effect.tryPromise({
+        try: async () => {
+          const file = Bun.file(`${pwd}/${filepath}`);
+          const content = await file.text();
+          const lines = content.split("\n");
+          return lines.slice(lineRange.start, lineRange.end + 1).join("\n");
+        },
+        catch: () => chunk.text,
+      }).pipe(Effect.catchAll(() => Effect.succeed(chunk.text)));
+      yield* Console.log(codeBlock);
+      yield* Console.log("");
+    }
+    if (topResults.length === 0) {
+      yield* Console.log("No results found.");
+    }
+  }).pipe(
+    Effect.catchTags({
+      ApiKeyMissingError: (e) =>
+        Console.error(`Error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+      EmbeddingError: (e) =>
+        Console.error(`Error embedding query: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+      ConfigError: (e) =>
+        Console.error(`Config error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+      FileReadError: (e) =>
+        Console.error(`File read error: ${e.path}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+    })
+  );

package/src/cli/commands/watch.ts ADDED Viewed

@@ -0,0 +1,159 @@
+import { Effect, Console } from "effect";
+import { GlobalStore } from "../../storage/global-store";
+import { Indexer } from "../../core/indexer";
+import { GitTracker } from "../../watcher/git-tracker";
+import { createFileWatcher } from "../../watcher/file-watcher";
+const performInitialSync = (pwd: string) =>
+  Effect.gen(function* () {
+    yield* Console.log("[Initial Sync] Checking for changes...");
+    const { changes, currentTree, isFullReindex } =
+      yield* GitTracker.getFilesToIndex(pwd);
+    if (changes.length === 0) {
+      yield* Console.log(
+        "[Initial Sync] No changes detected, index is up to date"
+      );
+      return;
+    }
+    if (isFullReindex) {
+      yield* Console.log(
+        `[Initial Sync] Full reindex required (${changes.length} files)`
+      );
+      const chunks = yield* Indexer.indexProject(pwd);
+      yield* Indexer.saveIndexedChunks(
+        pwd,
+        [...chunks],
+        currentTree.commitHash
+      );
+      yield* GitTracker.saveGitTree(pwd, currentTree);
+      yield* Console.log("[Initial Sync] Complete");
+    } else {
+      yield* Console.log(
+        `[Initial Sync] Incremental update (${changes.length} files changed)`
+      );
+      const addedOrModified = changes.filter((c) => c.changeType !== "deleted");
+      const deleted = changes.filter((c) => c.changeType === "deleted");
+      if (addedOrModified.length > 0) {
+        const chunks = yield* Indexer.indexFiles(
+          pwd,
+          addedOrModified.map((c) => c.filepath)
+        );
+        yield* Indexer.updateManifestWithChanges(
+          pwd,
+          [...chunks],
+          deleted.map((c) => c.filepath),
+          currentTree.commitHash
+        );
+      } else if (deleted.length > 0) {
+        yield* Indexer.updateManifestWithChanges(
+          pwd,
+          [],
+          deleted.map((c) => c.filepath)
+        );
+      }
+      yield* GitTracker.saveGitTree(pwd, currentTree);
+      yield* Console.log("[Initial Sync] Complete");
+    }
+  });
+const handleFileChanges = (pwd: string, changedFiles: string[]) =>
+  Effect.gen(function* () {
+    yield* Console.log(
+      `\n[Watcher] Detected changes in ${changedFiles.length} file(s)`
+    );
+    for (const file of changedFiles) {
+      yield* Console.log(`  - ${file}`);
+    }
+    const chunks = yield* Indexer.indexFiles(pwd, changedFiles);
+    const commitHash = yield* GitTracker.getCurrentCommitHash(pwd);
+    yield* Indexer.updateManifestWithChanges(
+      pwd,
+      [...chunks],
+      [],
+      commitHash ?? undefined
+    );
+    const currentFileHashes = yield* GitTracker.getGitTree(pwd);
+    yield* GitTracker.saveGitTree(pwd, {
+      commitHash: commitHash ?? "no-commit",
+      fileHashes: currentFileHashes,
+    });
+    yield* Console.log(
+      `[Watcher] Indexed ${chunks.length} chunks from ${changedFiles.length} file(s)\n`
+    );
+  }).pipe(
+    Effect.catchAll((error) =>
+      Console.error(`[Watcher] Error indexing changes: ${error}`)
+    )
+  );
+export const watchCommand = () =>
+  Effect.gen(function* () {
+    const pwd = process.cwd();
+    yield* GlobalStore.getApiKeyOrFail;
+    yield* GlobalStore.ensureDirectories;
+    yield* Console.log(`[semantic] Starting watch for: ${pwd}`);
+    yield* Console.log("");
+    yield* performInitialSync(pwd);
+    const watcher = createFileWatcher(pwd, {
+      debounceMs: 500,
+      onChanges: async (changedFiles) => {
+        await Effect.runPromise(handleFileChanges(pwd, changedFiles));
+      },
+    });
+    watcher.start();
+    const shutdown = () => {
+      console.log("\n[semantic] Shutting down...");
+      watcher.stop();
+      process.exit(0);
+    };
+    process.on("SIGINT", shutdown);
+    process.on("SIGTERM", shutdown);
+    yield* Console.log("");
+    yield* Console.log(
+      "[semantic] Watching for changes. Press Ctrl+C to stop."
+    );
+    yield* Effect.never;
+  }).pipe(
+    Effect.catchTags({
+      ApiKeyMissingError: (e) =>
+        Console.error(`Error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+      ConfigError: (e) =>
+        Console.error(`Config error: ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+      GitError: (e) =>
+        Console.error(`Git error (${e.command}): ${e.message}`).pipe(
+          Effect.flatMap(() => Effect.sync(() => process.exit(1)))
+        ),
+    })
+  );

package/src/cli/index.ts ADDED Viewed

@@ -0,0 +1,63 @@
+import { Command } from "commander";
+import { Effect } from "effect";
+import { clearRepoCacheCommand } from "./commands/cache";
+import { configCommand } from "./commands/config";
+import { searchCommand } from "./commands/search";
+import { watchCommand } from "./commands/watch";
+const program = new Command();
+program
+  .name("semantic")
+  .description("Semantic code search CLI")
+  .version("1.0.0");
+program
+  .command("watch")
+  .description("Watch and index the current directory")
+  .action(async () => {
+    await Effect.runPromise(watchCommand());
+  });
+program
+  .command("search <query>")
+  .description("Search the indexed codebase")
+  .option("--topk <number>", "Number of results to return", "10")
+  .option("--compact", "Compact output format", false)
+  .action(
+    async (query: string, options: { topk: string; compact: boolean }) => {
+      const topk = Math.min(20, Math.max(1, parseInt(options.topk, 10) || 10));
+      await Effect.runPromise(
+        searchCommand(query, { topk, compact: options.compact })
+      );
+    }
+  );
+const cacheCmd = program.command("cache").description("Manage repo cache");
+cacheCmd
+  .command("clear")
+  .description("Clear repo cache for current directory (keeps embeddings)")
+  .action(async () => {
+    await Effect.runPromise(clearRepoCacheCommand());
+  });
+const configCmd = program.command("config").description("Manage configuration");
+configCmd
+  .command("add-api-key <key>")
+  .description("Store your Voyage AI API key")
+  .action(async (key: string) => {
+    await Effect.runPromise(configCommand("add-api-key", key));
+  });
+configCmd
+  .command("show")
+  .description("Show current configuration")
+  .action(async () => {
+    await Effect.runPromise(configCommand("show"));
+  });
+export async function main(): Promise<void> {
+  await program.parseAsync(process.argv);
+}