xindex 1.0.0

package/componets/logger.ts

@@ -0,0 +1,34 @@
+export type ILogger = (...args: any[]) => void;
+
+export function Logger(fn: (...args: any[]) => void): ILogger {
+    return function log(...args) {
+        fn(...args);
+    }
+}
+
+const DEFAULT_FLUSH_MS = 100;
+
+export function BufferedLoggerToStdOut(flushMs = DEFAULT_FLUSH_MS): ILogger {
+    return BufferedLogger(s => process.stdout.write(s), flushMs);
+}
+
+export function BufferedLoggerToStdErr(flushMs = DEFAULT_FLUSH_MS): ILogger {
+    return BufferedLogger(s => process.stderr.write(s), flushMs);
+}
+
+export function BufferedLogger(write: (text: string) => void, flushMs = DEFAULT_FLUSH_MS): ILogger {
+    let buffer: string[] = [];
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    function flush() {
+        if (buffer.length === 0) return;
+        write(buffer.join("\n") + "\n");
+        buffer = [];
+        timer = null;
+    }
+
+    return function log(...args) {
+        buffer.push(args.map(a => typeof a === "object" ? JSON.stringify(a) : String(a)).join(" "));
+        if (!timer) timer = setTimeout(flush, flushMs);
+    }
+}

package/componets/walkFiles.ts

@@ -0,0 +1,51 @@
+import {readdir, stat, readFile} from "fs/promises";
+import {join, relative} from "path";
+import ignore from "ignore";
+import {ILogger} from "./logger.js";
+
+export type IWalkFiles = (inputs: string[]) => AsyncIterable<string>;
+
+export function WalkFiles({cwd, log, ignoreFiles = []}: {cwd: string, log: ILogger, ignoreFiles?: string[]}): IWalkFiles {
+
+    async function tryReadGitignore(dir: string): Promise<string> {
+        try {
+            return await readFile(join(dir, ".gitignore"), "utf8");
+        } catch {
+            return "";
+        }
+    }
+
+    async function* walk(dir: string, parentRules: string[]): AsyncIterable<string> {
+        const localGitignore = await tryReadGitignore(dir);
+        const rules = localGitignore ? [...parentRules, localGitignore] : parentRules;
+        const ig = ignore();
+        for (const rule of rules) ig.add(rule);
+        for (const pattern of ignoreFiles) ig.add(pattern);
+
+        const entries = await readdir(dir, {withFileTypes: true});
+        for (const entry of entries) {
+            const abs = join(dir, entry.name);
+            const rel = relative(cwd, abs);
+
+            if (entry.isDirectory()) {
+                if (entry.name.startsWith(".")) continue;
+                if (ig.ignores(rel + "/")) continue;
+                yield* walk(abs, rules);
+            } else {
+                if (ig.ignores(rel)) continue;
+                yield rel;
+            }
+        }
+    }
+
+    return async function* walkFiles(inputs) {
+        for (const input of inputs) {
+            const s = await stat(input);
+            if (s.isDirectory()) {
+                yield* walk(input, []);
+            } else {
+                yield relative(cwd, input);
+            }
+        }
+    };
+}

package/componets/watchFiles.ts

@@ -0,0 +1,106 @@
+import {readFile, stat, watch} from "fs/promises";
+import {join, relative} from "path";
+import ignore from "ignore";
+import {ILogger} from "./logger.js";
+import {IType} from "./IType.js";
+
+export enum FileEventType {
+    index = 'index',
+    remove = 'remove',
+}
+
+export type IFileEvent = IType<{ type: FileEventType.index, path: string }> | IType<{
+    type: FileEventType.remove,
+    path: string
+}>;
+
+export type IWatchFilesResult = { events: AsyncIterable<IFileEvent>, stop: () => void };
+export type IWatchFiles = (inputs: string[]) => IWatchFilesResult;
+
+export function WatchFiles({cwd, log, ignoreFiles = []}: { cwd: string, log: ILogger, ignoreFiles?: string[] }): IWatchFiles {
+
+    async function loadGitignore(dir: string): Promise<ReturnType<typeof ignore>> {
+        const ig = ignore();
+        ig.add(".*");
+        try {
+            const content = await readFile(join(dir, ".gitignore"), "utf8");
+            ig.add(content);
+        } catch {
+        }
+        for (const pattern of ignoreFiles) ig.add(pattern);
+        return ig;
+    }
+
+    return function watchFiles(inputs) {
+        const pending = new Map<string, IFileEvent>();
+        let notify: (() => void) | null = null;
+        let stopped = false;
+        const abortControllers: AbortController[] = [];
+
+        async function startWatching(dir: string) {
+            const ig = await loadGitignore(dir);
+            const ac = new AbortController();
+            abortControllers.push(ac);
+
+            try {
+                for await (const event of watch(dir, {recursive: true, signal: ac.signal})) {
+                    if (stopped) break;
+                    if (!event.filename) continue;
+
+                    const rel = relative(cwd, join(dir, event.filename));
+                    if (ig.ignores(rel)) continue;
+                    if (rel.endsWith("~")) continue;
+
+                    try {
+                        await stat(join(dir, event.filename));
+                        pending.set(rel, {type: FileEventType.index, path: rel});
+                    } catch {
+                        pending.set(rel, {type: FileEventType.remove, path: rel});
+                    }
+
+                    notify?.();
+                }
+            } catch (err: any) {
+                if (err?.name !== "AbortError" && !stopped) {
+                    log(`watch error: ${err}`);
+                }
+            }
+        }
+
+        for (const input of inputs) {
+            startWatching(input);
+        }
+
+        async function* debounced(): AsyncIterable<IFileEvent> {
+            while (!stopped) {
+                // Wait for at least one event
+                if (pending.size === 0) {
+                    await new Promise<void>(r => {
+                        notify = r;
+                    });
+                    notify = null;
+                    if (stopped) break;
+                }
+
+                // Debounce: wait 150ms for more events to accumulate
+                await new Promise(r => setTimeout(r, 150));
+
+                // Flush all pending events
+                const batch = [...pending.values()];
+                pending.clear();
+                for (const event of batch) {
+                    yield event;
+                }
+            }
+        }
+
+        return {
+            events: debounced(),
+            stop: () => {
+                stopped = true;
+                for (const ac of abortControllers) ac.abort();
+                notify?.();
+            },
+        };
+    };
+}

package/features/indexContent.ts

@@ -0,0 +1,16 @@
+import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
+import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
+import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
+
+export type IIndexContent = (id: string, content: string) => Promise<void>;
+
+export function IndexContent({extractKeywords, cleanUpKeywords, indexApi}: {
+    extractKeywords: IExtractKeywords,
+    cleanUpKeywords: ICleanUpKeywords,
+    indexApi: IIndexApi,
+}): IIndexContent {
+    return async function indexContent(id, content) {
+        const keywords = cleanUpKeywords(extractKeywords(content)).join(", ");
+        await indexApi({type: IndexCommandType.index, id, content: keywords, keywords});
+    }
+}

package/features/removeContent.ts

@@ -0,0 +1,9 @@
+import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
+
+export type IRemoveContent = (id: string) => Promise<void>;
+
+export function RemoveContent({indexApi}: {indexApi: IIndexApi}): IRemoveContent {
+    return async function removeContent(id) {
+        await indexApi({type: IndexCommandType.delete, id});
+    }
+}

package/features/resetIndex.ts

@@ -0,0 +1,9 @@
+import {IndexCommandType, IIndexApi} from "../componets/index/indexApi.js";
+
+export type IResetIndex = () => Promise<void>;
+
+export function ResetIndex({indexApi}: {indexApi: IIndexApi}): IResetIndex {
+    return async function resetIndex() {
+        await indexApi({type: IndexCommandType.reset});
+    }
+}

package/features/searchIndex.ts

@@ -0,0 +1,33 @@
+import {LocalIndex} from "vectra";
+import {IEmbed} from "../componets/llm/embed.js";
+import {IExtractKeywords} from "../componets/keywords/extractKeywords.js";
+import {ICleanUpKeywords} from "../componets/keywords/cleanUpKeywords.js";
+
+export type IIndexRecord = { score: number; id: string; keywords: string };
+
+export type ISearchIndex = (query: string, limit: number) => Promise<IIndexRecord[]>;
+
+export function SearchIndex({extractKeywords, cleanUpKeywords, embed, index, scoreThreshold = 0.05}: {
+    extractKeywords: IExtractKeywords,
+    cleanUpKeywords: ICleanUpKeywords,
+    embed: IEmbed,
+    index: LocalIndex,
+    scoreThreshold?: number
+}): ISearchIndex {
+    return async function searchContentIndex(query, limit) {
+        const keywords = cleanUpKeywords(extractKeywords(query));
+        const searchText = keywords.length > 0 ? keywords.join(", ") : query;
+        const vector = await embed(searchText);
+
+        const results = await index.queryItems(vector, searchText, limit);
+
+        return results
+            .filter(r => r.score >= scoreThreshold)
+            .sort((a, b) => b.score - a.score)
+            .map(r => ({
+                score: r.score,
+                id: r.item.id,
+                keywords: typeof r.item.metadata?.keywords === "string" ? r.item.metadata.keywords : "",
+            }));
+    }
+}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "xindex",
+  "version": "1.0.0",
+  "description": "Local semantic code search — index codebase, search by meaning or keywords",
+  "type": "module",
+  "main": "xindex.ts",
+  "bin": {
+    "xindex-index": "bin/xindex-index",
+    "xindex-search": "bin/xindex-search",
+    "xindex-mcp": "bin/xindex-mcp",
+    "xindex-reset": "bin/xindex-reset",
+    "xindex-watch": "bin/xindex-watch"
+  },
+  "scripts": {
+    "index": "tsx apps/run.index.ts",
+    "search": "tsx apps/run.search.ts",
+    "reset": "tsx apps/run.reset.ts",
+    "mcp": "tsx apps/run.mcp.ts",
+    "watch": "tsx apps/run.watch.ts"
+  },
+  "private": false,
+  "dependencies": {
+    "@huggingface/transformers": "^4.0.1",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "compromise": "^14.15.0",
+    "ignore": "^7.0.5",
+    "keyword-extractor": "^0.0.28",
+    "tsx": "^4.21.0",
+    "vectra": "^0.14.0",
+    "zod": "^4.3.6"
+  }
+}

package/packages/fun/src/IType.ts

@@ -0,0 +1,5 @@
+export type IType<T extends { type: string } = { type: string }> = Readonly<T>;
+export type IUseType<
+  Union extends IType,
+  Selector extends IType["type"],
+> = Extract<Union, { type: Selector }>;

package/packages/fun/src/array-finder.ts

@@ -0,0 +1,55 @@
+import { sortObject } from './sort-object';
+
+export function ArrayFinder<Type, ID>(
+  items: Type[] | ReadonlyArray<Type>,
+  identity: (item: Type) => ID
+): (query: ID) => Type | undefined {
+  const map = new Map<string, Type>();
+
+  const serializeId = (id: ID): string => {
+    if (id instanceof Object) {
+      return JSON.stringify(sortObject(id));
+    } else {
+      return String(id);
+    }
+  };
+
+  let initialized = false;
+
+  const initialize = () => {
+    for (const item of items) {
+      const id = serializeId(identity(item));
+      map.set(id, item);
+    }
+  };
+
+  return (id: ID) => {
+    if (!initialized) {
+      initialize();
+      initialized = true;
+    }
+
+    return map.get(serializeId(id));
+  };
+}
+
+export function StrictArrayFinder<Type, ID>(
+  items: Type[] | ReadonlyArray<Type>,
+  identity: (item: Type) => ID,
+  error: (item: ID) => Error | string
+): (id: ID) => Type {
+  const finder = ArrayFinder(items, identity);
+  return (id: ID) => {
+    const found = finder(id);
+
+    if (!found) {
+      const _error = error(id);
+      if (typeof _error === 'string') {
+        throw new Error(_error);
+      }
+      throw _error;
+    }
+
+    return found;
+  };
+}

package/packages/fun/src/array-index.ts

@@ -0,0 +1,35 @@
+import { sortObject } from './sort-object';
+
+export function ArrayIndex<Type, ID>(
+  items: Type[] | ReadonlyArray<Type>,
+  identity: (item: Type) => ID
+): (query: ID) => Type[] {
+  const groups: Record<string, Type[]> = {};
+
+  const serializeId = (id: ID): string => {
+    if (id instanceof Object) {
+      return JSON.stringify(sortObject(id));
+    } else {
+      return String(id);
+    }
+  };
+
+  let initialized = false;
+
+  const initialize = () => {
+    for (const item of items) {
+      const id = serializeId(identity(item));
+      groups[id] = groups[id] || [];
+      groups[id].push(item);
+    }
+  };
+
+  return (id: ID) => {
+    if (!initialized) {
+      initialize();
+      initialized = true;
+    }
+
+    return groups[serializeId(id)] ?? [];
+  };
+}

package/packages/fun/src/array.ts

@@ -0,0 +1,112 @@
+export function toArray<T>(value: T | Array<T> | ReadonlyArray<T>): T[] {
+  return (Array.isArray(value) ? value : [value]) as T[];
+}
+
+export function toNonEmptyArray<T>(value: T | Array<T> | ReadonlyArray<T> | null): T[] {
+  return toArray(value).filter(x => !!x) as T[];
+}
+
+export function Exclude(toExclude: string[]) {
+  return function (payload: object) {
+    const out = { ...payload };
+
+    toExclude.forEach(column => {
+      if (column in out) {
+        // @ts-ignore
+        delete out[column];
+      }
+    });
+
+    return out;
+  };
+}
+
+const PickNullByDefault = (_column: string): any => null;
+
+export function Pick(
+  toPick: string[],
+  defaultMapper: null | typeof PickNullByDefault = PickNullByDefault
+) {
+  return function (payload: object) {
+    const out = {};
+
+    toPick.forEach(column => {
+      if (column in payload) {
+        // @ts-ignore
+        out[column] = payload[column];
+      } else if (defaultMapper instanceof Function) {
+        // @ts-ignore
+        out[column] = defaultMapper(column);
+      }
+    });
+
+    return out;
+  };
+}
+
+type IGroupByResult<T, KType extends keyof any> = Record<KType, T[]>;
+type IGroupByIdentifier<T, KType extends keyof any = any> = (v: T) => KType;
+
+export function groupBy<T, KType extends keyof any = any>(
+  values: ReadonlyArray<T>,
+  identify: IGroupByIdentifier<T, KType>
+): IGroupByResult<T, KType> {
+  return values.reduce(
+    (a, v) => {
+      const key = identify(v);
+      a[key] = a[key] || [];
+      a[key].push(v);
+      return a;
+    },
+    {} as IGroupByResult<T, KType>
+  );
+}
+
+type IIdentifyBy<T> = (item: T) => any;
+
+export function hashMapBy<T extends object>(
+  values: T[],
+  identify: (item: T) => any
+): Partial<{ [P in any]: T }> {
+  return values.reduce(
+    (hashMap, item) => {
+      hashMap[identify(item)] = item;
+      return hashMap;
+    },
+    {} as Partial<{ [P in any]: T }>
+  );
+}
+
+export function uniqBy<T extends object>(values: T[], identify: IIdentifyBy<T>): T[] {
+  return Object.values(hashMapBy<T>(values, identify) as Record<any, any>);
+}
+
+export function orderBy<TObject, TProperty>(
+  predicate: (item: TObject) => TProperty,
+  direction: 'asc' | 'desc' = 'asc'
+) {
+  const factor = direction === 'asc' ? 1 : -1;
+  return (x: TObject, y: TObject): number => {
+    const a = predicate(x);
+    const b = predicate(y);
+    if (a > b) {
+      return 1 * factor;
+    } else if (a < b) {
+      return -1 * factor;
+    } else {
+      return 0;
+    }
+  };
+}
+
+export function filterValues<T>(values: (T | undefined | null)[]): T[] {
+  const filteredValues: T[] = [];
+
+  for (const value of values) {
+    if (value !== null && value !== undefined) {
+      filteredValues.push(value);
+    }
+  }
+
+  return filteredValues;
+}

package/packages/fun/src/assert.ts

@@ -0,0 +1,5 @@
+export function assert(value: unknown, message?: string): asserts value {
+  if (!value) {
+    throw new Error(message ?? 'Assertion failed');
+  }
+}

package/packages/fun/src/asyncRequest.ts

@@ -0,0 +1,35 @@
+import { Defer, IDefer } from './defer';
+
+export type IAsyncRequest<Request = any, Response = void, Cancellation = void> = Omit<
+  IDefer<Response>,
+  'promise'
+> & {
+  request: Request;
+  response: Promise<Response>;
+  cancelled: boolean;
+  cancel: (_: Cancellation) => Promise<void> | void;
+};
+
+export function AsyncRequest<Request = void, Response = void, Cancellation = void>(
+  request: Request,
+  onCancel?: (cancellation: Cancellation) => Promise<void> | void
+): IAsyncRequest<Request, Response, Cancellation> {
+  const deferred = Defer<Response>();
+
+  const cancel = async (cancellation: Cancellation) => {
+    instance.cancelled = true;
+    if (onCancel) {
+      return await onCancel(cancellation);
+    }
+  };
+
+  const instance = {
+    ...deferred,
+    cancelled: false,
+    request,
+    response: deferred.promise,
+    cancel,
+  };
+
+  return instance;
+}

package/packages/fun/src/callsites.ts

@@ -0,0 +1,18 @@
+// Ripped off https://github.com/sindresorhus/callsites/
+
+export function callsites() {
+  const _prepareStackTrace = Error.prepareStackTrace;
+  try {
+    let result: NodeJS.CallSite[] = [];
+    Error.prepareStackTrace = (_, callSites) => {
+      const callSitesWithoutCurrent = callSites.slice(1);
+      result = callSitesWithoutCurrent;
+      return callSitesWithoutCurrent;
+    };
+
+    new Error().stack;
+    return result;
+  } finally {
+    Error.prepareStackTrace = _prepareStackTrace;
+  }
+}

package/packages/fun/src/case-never.ts

@@ -0,0 +1,9 @@
+export const caseNever = (_: never): never => {
+  let value = 'UNKNOWN';
+
+  try {
+    value = JSON.stringify(_);
+  } catch {}
+
+  throw new Error(`caseNever: ${value}`);
+};

package/packages/fun/src/casting.ts

@@ -0,0 +1,41 @@
+export const castAsNumerical = (value: any): number | null => {
+  try {
+    const result = Number(value);
+    if (Number.isNaN(result)) {
+      return null;
+    }
+
+    return result;
+  } catch (error) {
+    return null;
+  }
+};
+
+export const castAsBoolean = (value: any): boolean | null => {
+  const stringValue = String(value).toLowerCase();
+  if (stringValue === 'true') {
+    return true;
+  }
+
+  if (stringValue === 'false') {
+    return false;
+  }
+
+  return null;
+};
+
+export const castAsTimestamp = (value: any): Date | null => {
+  try {
+    return new Date(value);
+  } catch (error) {
+    return null;
+  }
+};
+
+export const safeJsonParse = (value: string): any | null => {
+  try {
+    return JSON.parse(value);
+  } catch (error) {
+    return null;
+  }
+};

package/packages/fun/src/collect.ts

@@ -0,0 +1,13 @@
+export type ICollect<T> = (...args: T[]) => T[];
+
+export function Collect<T>(): ICollect<T> {
+  const records: T[] = [];
+
+  return function (...args: T[]) {
+    if (args.length) {
+      records.push(args[0]!);
+    }
+
+    return records;
+  };
+}