viberag 0.1.0

package/dist/rag/manifest/index.d.ts

@@ -0,0 +1,50 @@
+export interface ManifestStats {
+    totalFiles: number;
+    totalChunks: number;
+}
+export interface Manifest {
+    version: number;
+    schemaVersion: number;
+    createdAt: string;
+    updatedAt: string;
+    tree: object | null;
+    stats: ManifestStats;
+}
+/**
+ * Create an empty manifest with current schema version.
+ */
+export declare function createEmptyManifest(): Manifest;
+/**
+ * Check if manifest schema version is current.
+ */
+export declare function isSchemaVersionCurrent(manifest: Manifest): boolean;
+/**
+ * Get schema version mismatch info for display.
+ */
+export declare function getSchemaVersionInfo(manifest: Manifest): {
+    current: number;
+    required: number;
+    needsReindex: boolean;
+};
+/**
+ * Load manifest from disk.
+ * Returns an empty manifest if no file exists.
+ */
+export declare function loadManifest(projectRoot: string): Promise<Manifest>;
+/**
+ * Save manifest to disk.
+ * Creates the .viberag directory if it doesn't exist.
+ */
+export declare function saveManifest(projectRoot: string, manifest: Manifest): Promise<void>;
+/**
+ * Check if a manifest file exists.
+ */
+export declare function manifestExists(projectRoot: string): Promise<boolean>;
+/**
+ * Update manifest stats.
+ */
+export declare function updateManifestStats(manifest: Manifest, stats: ManifestStats): Manifest;
+/**
+ * Update manifest tree.
+ */
+export declare function updateManifestTree(manifest: Manifest, tree: object | null): Manifest;

package/dist/rag/manifest/index.js

@@ -0,0 +1,97 @@
+import fs from 'node:fs/promises';
+import { getManifestPath, getViberagDir } from '../constants.js';
+import { SCHEMA_VERSION } from '../storage/schema.js';
+/**
+ * Create an empty manifest with current schema version.
+ */
+export function createEmptyManifest() {
+    const now = new Date().toISOString();
+    return {
+        version: 1,
+        schemaVersion: SCHEMA_VERSION,
+        createdAt: now,
+        updatedAt: now,
+        tree: null,
+        stats: {
+            totalFiles: 0,
+            totalChunks: 0,
+        },
+    };
+}
+/**
+ * Check if manifest schema version is current.
+ */
+export function isSchemaVersionCurrent(manifest) {
+    return manifest.schemaVersion === SCHEMA_VERSION;
+}
+/**
+ * Get schema version mismatch info for display.
+ */
+export function getSchemaVersionInfo(manifest) {
+    return {
+        current: manifest.schemaVersion ?? 1, // Default to 1 for old manifests
+        required: SCHEMA_VERSION,
+        needsReindex: (manifest.schemaVersion ?? 1) < SCHEMA_VERSION,
+    };
+}
+/**
+ * Load manifest from disk.
+ * Returns an empty manifest if no file exists.
+ */
+export async function loadManifest(projectRoot) {
+    const manifestPath = getManifestPath(projectRoot);
+    try {
+        const content = await fs.readFile(manifestPath, 'utf-8');
+        return JSON.parse(content);
+    }
+    catch {
+        return createEmptyManifest();
+    }
+}
+/**
+ * Save manifest to disk.
+ * Creates the .viberag directory if it doesn't exist.
+ */
+export async function saveManifest(projectRoot, manifest) {
+    const viberagDir = getViberagDir(projectRoot);
+    await fs.mkdir(viberagDir, { recursive: true });
+    const manifestPath = getManifestPath(projectRoot);
+    const updated = {
+        ...manifest,
+        updatedAt: new Date().toISOString(),
+    };
+    await fs.writeFile(manifestPath, JSON.stringify(updated, null, '\t') + '\n');
+}
+/**
+ * Check if a manifest file exists.
+ */
+export async function manifestExists(projectRoot) {
+    const manifestPath = getManifestPath(projectRoot);
+    try {
+        await fs.access(manifestPath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Update manifest stats.
+ */
+export function updateManifestStats(manifest, stats) {
+    return {
+        ...manifest,
+        stats,
+        updatedAt: new Date().toISOString(),
+    };
+}
+/**
+ * Update manifest tree.
+ */
+export function updateManifestTree(manifest, tree) {
+    return {
+        ...manifest,
+        tree,
+        updatedAt: new Date().toISOString(),
+    };
+}

package/dist/rag/merkle/diff.d.ts

@@ -0,0 +1,26 @@
+import type { MerkleNode } from './node.js';
+/**
+ * Result of comparing two Merkle trees.
+ */
+export interface TreeDiff {
+    /** Paths of new files */
+    new: string[];
+    /** Paths of modified files */
+    modified: string[];
+    /** Paths of deleted files */
+    deleted: string[];
+    /** Whether there are any changes */
+    hasChanges: boolean;
+}
+/**
+ * Create an empty TreeDiff.
+ */
+export declare function createEmptyDiff(): TreeDiff;
+/**
+ * Compare two Merkle trees and return the differences.
+ *
+ * @param oldRoot - The previous tree's root node (or null if no previous tree)
+ * @param newRoot - The current tree's root node
+ * @returns TreeDiff with new, modified, and deleted file paths
+ */
+export declare function compareTrees(oldRoot: MerkleNode | null, newRoot: MerkleNode | null): TreeDiff;

package/dist/rag/merkle/diff.js

@@ -0,0 +1,95 @@
+/**
+ * Create an empty TreeDiff.
+ */
+export function createEmptyDiff() {
+    return {
+        new: [],
+        modified: [],
+        deleted: [],
+        hasChanges: false,
+    };
+}
+/**
+ * Collect all file paths from a node (recursively).
+ */
+function collectAllFiles(node, paths) {
+    if (node.type === 'file') {
+        paths.push(node.path);
+    }
+    else if (node.children) {
+        for (const child of node.children.values()) {
+            collectAllFiles(child, paths);
+        }
+    }
+}
+/**
+ * Compare two Merkle nodes and populate the diff.
+ */
+function compareNodes(oldNode, newNode, diff) {
+    // Quick check: if hashes match, entire subtree unchanged
+    if (oldNode.hash === newNode.hash) {
+        return;
+    }
+    // File modified
+    if (oldNode.type === 'file' && newNode.type === 'file') {
+        diff.modified.push(newNode.path);
+        return;
+    }
+    // Type changed (file→dir or dir→file)
+    if (oldNode.type !== newNode.type) {
+        collectAllFiles(oldNode, diff.deleted);
+        collectAllFiles(newNode, diff.new);
+        return;
+    }
+    // Both directories: compare children
+    const oldChildren = oldNode.children ?? new Map();
+    const newChildren = newNode.children ?? new Map();
+    // Find new entries (in new but not in old)
+    for (const [name, child] of newChildren) {
+        if (!oldChildren.has(name)) {
+            collectAllFiles(child, diff.new);
+        }
+    }
+    // Find deleted entries (in old but not in new)
+    for (const [name, child] of oldChildren) {
+        if (!newChildren.has(name)) {
+            collectAllFiles(child, diff.deleted);
+        }
+    }
+    // Recurse into shared entries
+    for (const [name, newChild] of newChildren) {
+        const oldChild = oldChildren.get(name);
+        if (oldChild) {
+            compareNodes(oldChild, newChild, diff);
+        }
+    }
+}
+/**
+ * Compare two Merkle trees and return the differences.
+ *
+ * @param oldRoot - The previous tree's root node (or null if no previous tree)
+ * @param newRoot - The current tree's root node
+ * @returns TreeDiff with new, modified, and deleted file paths
+ */
+export function compareTrees(oldRoot, newRoot) {
+    const diff = createEmptyDiff();
+    // No old tree - everything is new
+    if (!oldRoot) {
+        if (newRoot) {
+            collectAllFiles(newRoot, diff.new);
+        }
+        diff.hasChanges = diff.new.length > 0;
+        return diff;
+    }
+    // No new tree - everything is deleted
+    if (!newRoot) {
+        collectAllFiles(oldRoot, diff.deleted);
+        diff.hasChanges = diff.deleted.length > 0;
+        return diff;
+    }
+    // Both trees exist - compare them
+    compareNodes(oldRoot, newRoot, diff);
+    diff.hasChanges =
+        diff.new.length > 0 || diff.modified.length > 0 || diff.deleted.length > 0;
+    return diff;
+}

package/dist/rag/merkle/hash.d.ts

@@ -0,0 +1,34 @@
+import type { MerkleNode } from './node.js';
+/**
+ * Compute SHA256 hash of a file's content.
+ */
+export declare function computeFileHash(filepath: string): Promise<string>;
+/**
+ * Compute SHA256 hash of a string.
+ */
+export declare function computeStringHash(content: string): string;
+/**
+ * Compute SHA256 hash of a directory based on its children.
+ *
+ * Hash = SHA256(sorted child name+hash pairs)
+ * Format: "name1:hash1\nname2:hash2\n..."
+ */
+export declare function computeDirectoryHash(children: Map<string, MerkleNode>): string;
+/**
+ * Check if a file is likely binary based on extension.
+ * Falls back to checking for null bytes in the first chunk.
+ */
+export declare function isBinaryFile(filepath: string): Promise<boolean>;
+/**
+ * Check if a path should be excluded based on patterns.
+ *
+ * Supported pattern types:
+ * - "node_modules" - matches any path containing a "node_modules" segment
+ * - "*.pyc" - matches any file ending with .pyc
+ * - ".git" - matches any path containing a ".git" segment
+ */
+export declare function shouldExclude(relativePath: string, excludePatterns: string[]): boolean;
+/**
+ * Check if a file has a supported extension.
+ */
+export declare function hasValidExtension(filepath: string, extensions: string[]): boolean;

package/dist/rag/merkle/hash.js

@@ -0,0 +1,165 @@
+import { createHash } from 'node:crypto';
+import fs from 'node:fs/promises';
+/**
+ * Compute SHA256 hash of a file's content.
+ */
+export async function computeFileHash(filepath) {
+    const content = await fs.readFile(filepath);
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Compute SHA256 hash of a string.
+ */
+export function computeStringHash(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Compute SHA256 hash of a directory based on its children.
+ *
+ * Hash = SHA256(sorted child name+hash pairs)
+ * Format: "name1:hash1\nname2:hash2\n..."
+ */
+export function computeDirectoryHash(children) {
+    // Sort children by name for deterministic hashing
+    const sortedNames = [...children.keys()].sort();
+    // Build content string from name:hash pairs
+    const content = sortedNames
+        .map(name => {
+        const child = children.get(name);
+        return `${name}:${child.hash}`;
+    })
+        .join('\n');
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Known binary file extensions.
+ */
+const BINARY_EXTENSIONS = new Set([
+    // Images
+    '.png',
+    '.jpg',
+    '.jpeg',
+    '.gif',
+    '.bmp',
+    '.ico',
+    '.webp',
+    '.svg',
+    '.tiff',
+    '.tif',
+    // Audio/Video
+    '.mp3',
+    '.mp4',
+    '.wav',
+    '.avi',
+    '.mov',
+    '.webm',
+    '.flac',
+    '.ogg',
+    // Archives
+    '.zip',
+    '.tar',
+    '.gz',
+    '.bz2',
+    '.7z',
+    '.rar',
+    // Documents
+    '.pdf',
+    '.doc',
+    '.docx',
+    '.xls',
+    '.xlsx',
+    '.ppt',
+    '.pptx',
+    // Executables
+    '.exe',
+    '.dll',
+    '.so',
+    '.dylib',
+    '.bin',
+    // Fonts
+    '.ttf',
+    '.otf',
+    '.woff',
+    '.woff2',
+    '.eot',
+    // Other
+    '.wasm',
+    '.node',
+    '.pyc',
+    '.pyo',
+    '.class',
+    '.o',
+    '.a',
+]);
+/**
+ * Check if a file is likely binary based on extension.
+ * Falls back to checking for null bytes in the first chunk.
+ */
+export async function isBinaryFile(filepath) {
+    // Check extension first (fast path)
+    const ext = filepath.slice(filepath.lastIndexOf('.')).toLowerCase();
+    if (BINARY_EXTENSIONS.has(ext)) {
+        return true;
+    }
+    // Check for null bytes in first 8KB
+    try {
+        const handle = await fs.open(filepath, 'r');
+        try {
+            const buffer = Buffer.alloc(8192);
+            const { bytesRead } = await handle.read(buffer, 0, 8192, 0);
+            // Check for null bytes (common in binary files)
+            for (let i = 0; i < bytesRead; i++) {
+                if (buffer[i] === 0) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch {
+        // If we can't read the file, assume it's not binary
+        return false;
+    }
+}
+/**
+ * Check if a path should be excluded based on patterns.
+ *
+ * Supported pattern types:
+ * - "node_modules" - matches any path containing a "node_modules" segment
+ * - "*.pyc" - matches any file ending with .pyc
+ * - ".git" - matches any path containing a ".git" segment
+ */
+export function shouldExclude(relativePath, excludePatterns) {
+    // Split path into segments
+    const segments = relativePath.split('/');
+    const filename = segments[segments.length - 1] ?? '';
+    for (const pattern of excludePatterns) {
+        // Glob pattern: *.ext matches files with that extension
+        if (pattern.startsWith('*.')) {
+            const ext = pattern.slice(1); // ".pyc"
+            if (filename.endsWith(ext)) {
+                return true;
+            }
+            continue;
+        }
+        // Check if any segment matches the pattern exactly
+        if (segments.includes(pattern)) {
+            return true;
+        }
+        // Also check if the path starts with the pattern (for top-level exclusions)
+        if (relativePath.startsWith(pattern + '/') || relativePath === pattern) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Check if a file has a supported extension.
+ */
+export function hasValidExtension(filepath, extensions) {
+    const ext = filepath.slice(filepath.lastIndexOf('.'));
+    return extensions.includes(ext);
+}

package/dist/rag/merkle/index.d.ts

@@ -0,0 +1,68 @@
+import { type MerkleNode, type SerializedNode } from './node.js';
+import { type TreeDiff } from './diff.js';
+export * from './node.js';
+export * from './hash.js';
+export * from './diff.js';
+/**
+ * Statistics from building a Merkle tree.
+ */
+export interface BuildStats {
+    /** Total files scanned (before filtering) */
+    filesScanned: number;
+    /** Files indexed (after filtering) */
+    filesIndexed: number;
+    /** Hash cache hits (mtime optimization) */
+    cacheHits: number;
+    /** Hash cache misses (computed hash) */
+    cacheMisses: number;
+    /** Files skipped (binary, symlink, errors) */
+    filesSkipped: number;
+}
+/**
+ * A Merkle tree for efficient codebase change detection.
+ *
+ * The tree is content-addressed: if a file's content doesn't change,
+ * its hash stays the same. Directory hashes are computed from their
+ * children's hashes, so unchanged subtrees have unchanged hashes.
+ */
+export declare class MerkleTree {
+    /** Root node of the tree */
+    readonly root: MerkleNode | null;
+    /** Total number of files in the tree */
+    readonly fileCount: number;
+    /** Build statistics (populated after build) */
+    readonly buildStats: BuildStats;
+    private constructor();
+    /**
+     * Build a Merkle tree from the filesystem.
+     *
+     * Uses .gitignore for exclusions instead of hardcoded patterns.
+     * If extensions is provided, only files with those extensions are included.
+     * If extensions is empty/undefined, all text files are included.
+     *
+     * @param projectRoot - Absolute path to project root
+     * @param extensions - File extensions to include (e.g., [".py", ".ts"]), or empty for all
+     * @param _excludePatterns - DEPRECATED: Use .gitignore instead. This parameter is ignored.
+     * @param previousTree - Previous tree for mtime optimization
+     */
+    static build(projectRoot: string, extensions: string[], _excludePatterns: string[], previousTree?: MerkleTree): Promise<MerkleTree>;
+    /**
+     * Compare this tree with another tree.
+     *
+     * @param other - The other tree (usually the new/current tree)
+     * @returns TreeDiff with new, modified, and deleted files
+     */
+    compare(other: MerkleTree): TreeDiff;
+    /**
+     * Serialize the tree to a plain object for JSON storage.
+     */
+    toJSON(): SerializedNode | null;
+    /**
+     * Deserialize a tree from a plain object.
+     */
+    static fromJSON(data: SerializedNode | null): MerkleTree;
+    /**
+     * Create an empty tree.
+     */
+    static empty(): MerkleTree;
+}