rowpipe 1.1.0 → 1.2.0

package/dist/files/reader.js

@@ -0,0 +1,344 @@
+import { promises as fsPromises } from "node:fs";
+import * as path from "node:path";
+import { FileSystemError } from "../core/errors.js";
+import { createGlobMatcher } from "./glob.js";
+import { computeFileHash } from "./hash.js";
+import { inferMimeAndCategory } from "./mime.js";
+/**
+ * Normalizes a relative path to standard forward slashes.
+ */
+function normalizeRelativePath(relPath) {
+    let normalized = relPath.replace(/\\/g, "/");
+    while (normalized.startsWith("./")) {
+        normalized = normalized.slice(2);
+    }
+    return normalized || ".";
+}
+/**
+ * First-class streaming filesystem reader.
+ * Converts directories and files into streaming tabular DataBatches.
+ */
+export class FileSystemReader {
+    options;
+    constructor(optionsOrRoot) {
+        if (typeof optionsOrRoot === "string") {
+            this.options = { root: optionsOrRoot };
+        }
+        else {
+            this.options = { ...optionsOrRoot };
+        }
+    }
+    /**
+     * Reads the filesystem stream yielding DataBatches.
+     */
+    async *read(readOptions) {
+        const rootPath = path.resolve(this.options.root);
+        const recursive = this.options.recursive ?? true;
+        const maxDepth = this.options.maxDepth ?? Infinity;
+        const includeHidden = this.options.hidden ?? false;
+        const followSymlinks = this.options.followSymlinks ?? false;
+        const typeFilter = this.options.type ?? "file";
+        const hashAlgorithm = this.options.hash ?? false;
+        const enableMime = this.options.mime ?? false;
+        const concurrency = Math.max(1, this.options.concurrency ?? 8);
+        const onError = this.options.onError ?? "fail";
+        const deterministic = this.options.deterministic ?? false;
+        const batchSize = readOptions?.batchSize ?? this.options.batchSize ?? 1000;
+        const includeMatcher = this.options.include ? createGlobMatcher(this.options.include) : () => true;
+        const excludeMatcher = this.options.exclude
+            ? createGlobMatcher(this.options.exclude)
+            : () => false;
+        // Check if root exists
+        let rootStat;
+        try {
+            rootStat = await fsPromises.stat(rootPath);
+        }
+        catch (err) {
+            const fsErr = new FileSystemError(`Root path not found or unreadable: "${rootPath}"`, {
+                path: rootPath,
+                operation: "stat",
+                cause: err instanceof Error ? err : String(err),
+            });
+            if (onError === "fail")
+                throw fsErr;
+            return;
+        }
+        let rowsAccumulator = [];
+        let offset = 0;
+        // Helper to flush batch
+        const flushBatch = () => {
+            if (rowsAccumulator.length >= batchSize) {
+                const batch = {
+                    rows: rowsAccumulator,
+                    offset,
+                };
+                offset += rowsAccumulator.length;
+                rowsAccumulator = [];
+                return batch;
+            }
+            return null;
+        };
+        // Helper to enrich a single entry
+        const enrichFileEntry = async (fullPath, relPath, dirPath, name, stat, isSymlink) => {
+            try {
+                const dotIdx = name.lastIndexOf(".");
+                let basename = name;
+                let extension = "";
+                if (dotIdx > 0) {
+                    basename = name.slice(0, dotIdx);
+                    extension = name.slice(dotIdx + 1).toLowerCase();
+                }
+                let fileType = "file";
+                if (isSymlink) {
+                    fileType = "symlink";
+                }
+                else if (stat.isDirectory()) {
+                    fileType = "directory";
+                }
+                const normRelPath = normalizeRelativePath(relPath);
+                const record = {
+                    path: fullPath,
+                    relative_path: normRelPath,
+                    name,
+                    basename,
+                    extension,
+                    directory: dirPath,
+                    type: fileType,
+                    size: stat.isDirectory() ? 0 : stat.size,
+                    created_at: stat.birthtime ? stat.birthtime.toISOString() : stat.ctime.toISOString(),
+                    modified_at: stat.mtime.toISOString(),
+                    mode: stat.mode,
+                    is_file: stat.isFile(),
+                    is_directory: stat.isDirectory(),
+                    is_symlink: isSymlink,
+                };
+                if (!deterministic) {
+                    record.accessed_at = stat.atime.toISOString();
+                }
+                if (enableMime || this.options.category) {
+                    const { mime, category } = inferMimeAndCategory(extension || name);
+                    if (enableMime)
+                        record.mime = mime;
+                    record.category = category;
+                }
+                if (hashAlgorithm && stat.isFile()) {
+                    record.hash = await computeFileHash(fullPath, hashAlgorithm);
+                }
+                return record;
+            }
+            catch (err) {
+                const fsErr = new FileSystemError(`Failed to process file "${fullPath}"`, {
+                    path: fullPath,
+                    operation: "enrich",
+                    cause: err instanceof Error ? err : String(err),
+                });
+                if (onError === "fail")
+                    throw fsErr;
+                return null;
+            }
+        };
+        // Case 1: Root is a single file
+        if (!rootStat.isDirectory()) {
+            const name = path.basename(rootPath);
+            const dirPath = path.dirname(rootPath);
+            const record = await enrichFileEntry(rootPath, name, dirPath, name, rootStat, false);
+            if (record) {
+                yield {
+                    rows: [record],
+                    offset: 0,
+                };
+            }
+            return;
+        }
+        // Case 2: Root is a directory - walk with bounded memory and concurrency
+        const visitedRealPaths = new Set();
+        visitedRealPaths.add(rootPath);
+        const dirQueue = [{ dirPath: rootPath, depth: 0 }];
+        while (dirQueue.length > 0) {
+            const current = dirQueue.shift();
+            let dirHandle;
+            try {
+                dirHandle = await fsPromises.opendir(current.dirPath);
+            }
+            catch (err) {
+                const fsErr = new FileSystemError(`Cannot open directory "${current.dirPath}"`, {
+                    path: current.dirPath,
+                    operation: "opendir",
+                    cause: err instanceof Error ? err : String(err),
+                });
+                if (onError === "fail")
+                    throw fsErr;
+                continue;
+            }
+            const pendingWork = [];
+            try {
+                for await (const dirent of dirHandle) {
+                    const entryName = dirent.name;
+                    // Hidden file filtering
+                    if (!includeHidden && entryName.startsWith(".")) {
+                        continue;
+                    }
+                    // Default safe exclusions when not hidden
+                    if (!includeHidden && (entryName === ".git" || entryName === "node_modules")) {
+                        continue;
+                    }
+                    const fullPath = path.join(current.dirPath, entryName);
+                    const relPath = normalizeRelativePath(path.relative(rootPath, fullPath));
+                    // Check excludes
+                    if (excludeMatcher(relPath)) {
+                        continue;
+                    }
+                    if (dirent.isDirectory()) {
+                        if (recursive && current.depth + 1 <= maxDepth) {
+                            dirQueue.push({ dirPath: fullPath, depth: current.depth + 1 });
+                        }
+                        if (typeFilter === "directory" || typeFilter === "all") {
+                            if (includeMatcher(relPath)) {
+                                pendingWork.push({ fullPath, relPath, dirent });
+                            }
+                        }
+                    }
+                    else if (dirent.isSymbolicLink()) {
+                        if (followSymlinks) {
+                            try {
+                                const realPath = await fsPromises.realpath(fullPath);
+                                if (visitedRealPaths.has(realPath)) {
+                                    // Cycle detected, skip
+                                    continue;
+                                }
+                                visitedRealPaths.add(realPath);
+                                const targetStat = await fsPromises.stat(fullPath);
+                                if (targetStat.isDirectory()) {
+                                    if (recursive && current.depth + 1 <= maxDepth) {
+                                        dirQueue.push({ dirPath: fullPath, depth: current.depth + 1 });
+                                    }
+                                    if (typeFilter === "directory" || typeFilter === "all") {
+                                        if (includeMatcher(relPath)) {
+                                            pendingWork.push({ fullPath, relPath, dirent });
+                                        }
+                                    }
+                                }
+                                else if (targetStat.isFile()) {
+                                    if (typeFilter === "file" || typeFilter === "symlink" || typeFilter === "all") {
+                                        if (includeMatcher(relPath)) {
+                                            pendingWork.push({ fullPath, relPath, dirent });
+                                        }
+                                    }
+                                }
+                            }
+                            catch (err) {
+                                if (onError === "fail") {
+                                    throw new FileSystemError(`Broken symlink at "${fullPath}"`, {
+                                        path: fullPath,
+                                        operation: "realpath",
+                                        cause: err instanceof Error ? err : String(err),
+                                    });
+                                }
+                            }
+                        }
+                        else {
+                            if (typeFilter === "symlink" || typeFilter === "all") {
+                                if (includeMatcher(relPath)) {
+                                    pendingWork.push({ fullPath, relPath, dirent });
+                                }
+                            }
+                        }
+                    }
+                    else if (dirent.isFile()) {
+                        if (typeFilter === "file" || typeFilter === "all") {
+                            if (includeMatcher(relPath)) {
+                                pendingWork.push({ fullPath, relPath, dirent });
+                            }
+                        }
+                    }
+                }
+            }
+            finally {
+                // opendir automatically closes on iterator completion
+            }
+            // Process pending work in chunks using bounded concurrency pool
+            for (let i = 0; i < pendingWork.length; i += concurrency) {
+                const chunk = pendingWork.slice(i, i + concurrency);
+                const results = await Promise.all(chunk.map(async (item) => {
+                    try {
+                        const isSymlink = item.dirent.isSymbolicLink();
+                        const stat = isSymlink && !followSymlinks
+                            ? await fsPromises.lstat(item.fullPath)
+                            : await fsPromises.stat(item.fullPath);
+                        return await enrichFileEntry(item.fullPath, item.relPath, current.dirPath, item.dirent.name, stat, isSymlink);
+                    }
+                    catch (err) {
+                        const fsErr = new FileSystemError(`Cannot stat "${item.fullPath}"`, {
+                            path: item.fullPath,
+                            operation: "stat",
+                            cause: err instanceof Error ? err : String(err),
+                        });
+                        if (onError === "fail")
+                            throw fsErr;
+                        return null;
+                    }
+                }));
+                for (const record of results) {
+                    if (record) {
+                        rowsAccumulator.push(record);
+                        const batch = flushBatch();
+                        if (batch) {
+                            yield batch;
+                        }
+                    }
+                }
+            }
+        }
+        // Flush any remaining rows
+        if (rowsAccumulator.length > 0) {
+            yield {
+                rows: rowsAccumulator,
+                offset,
+            };
+            offset += rowsAccumulator.length;
+            rowsAccumulator = [];
+        }
+    }
+    /**
+     * Inspects filesystem directory metadata without full memory accumulation.
+     */
+    async inspect(_options) {
+        let totalFiles = 0;
+        let totalSizeBytes = 0;
+        const extensionCounts = new Map();
+        for await (const batch of this.read({ batchSize: 5000 })) {
+            for (const row of batch.rows) {
+                totalFiles += 1;
+                totalSizeBytes += Number(row["size"] ?? 0);
+                const ext = String(row["extension"] ?? "") || "<none>";
+                extensionCounts.set(ext, (extensionCounts.get(ext) ?? 0) + 1);
+            }
+        }
+        const columns = [
+            { name: "path", type: "string", nullPercentage: 0 },
+            { name: "relative_path", type: "string", nullPercentage: 0 },
+            { name: "name", type: "string", nullPercentage: 0 },
+            { name: "extension", type: "string", nullPercentage: 0 },
+            { name: "type", type: "string", nullPercentage: 0 },
+            { name: "size", type: "integer", nullPercentage: 0 },
+            { name: "created_at", type: "date", nullPercentage: 0 },
+            { name: "modified_at", type: "date", nullPercentage: 0 },
+            { name: "is_file", type: "boolean", nullPercentage: 0 },
+            { name: "is_directory", type: "boolean", nullPercentage: 0 },
+        ];
+        if (this.options.hash) {
+            columns.push({ name: "hash", type: "string", nullPercentage: 0 });
+        }
+        if (this.options.mime) {
+            columns.push({ name: "mime", type: "string", nullPercentage: 0 });
+        }
+        return {
+            format: "FILESYSTEM",
+            rowCount: totalFiles,
+            sizeBytes: totalSizeBytes,
+            columnCount: columns.length,
+            columns,
+        };
+    }
+}
+//# sourceMappingURL=reader.js.map

package/dist/files/reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../src/files/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,UAAU,EAAiB,MAAM,SAAS,CAAC;AAChE,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,eAAe,EAAwB,MAAM,mBAAmB,CAAC;AAE1E,OAAO,EAAE,iBAAiB,EAAqB,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAGjD;;GAEG;AACH,SAAS,qBAAqB,CAAC,OAAe;IAC5C,IAAI,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,UAAU,IAAI,GAAG,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACV,OAAO,CAA0B;IAElD,YAAY,aAA+C;QACzD,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,aAAa,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,IAAI,CAAC,WAA2B;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC;QACnD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;QACnD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC;QAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;QAC1D,MAAM,SAAS,GAAG,WAAW,EAAE,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;QAE3E,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QACnG,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;YACzC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;YACzC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC;QAEhB,uBAAuB;QACvB,IAAI,QAAe,CAAC;QACpB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,uCAAuC,QAAQ,GAAG,EAAE;gBACpF,IAAI,EAAE,QAAQ;gBACd,SAAS,EAAE,MAAM;gBACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aAChD,CAAC,CAAC;YACH,IAAI,OAAO,KAAK,MAAM;gBAAE,MAAM,KAAK,CAAC;YACpC,OAAO;QACT,CAAC;QAED,IAAI,eAAe,GAAiB,EAAE,CAAC;QACvC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,wBAAwB;QACxB,MAAM,UAAU,GAAG,GAAqB,EAAE;YACxC,IAAI,eAAe,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAc;oBACvB,IAAI,EAAE,eAAuD;oBAC7D,MAAM;iBACP,CAAC;gBACF,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC;gBACjC,eAAe,GAAG,EAAE,CAAC;gBACrB,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QAEF,kCAAkC;QAClC,MAAM,eAAe,GAAG,KAAK,EAC3B,QAAgB,EAChB,OAAe,EACf,OAAe,EACf,IAAY,EACZ,IAAW,EACX,SAAkB,EACU,EAAE;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,QAAQ,GAAG,IAAI,CAAC;gBACpB,IAAI,SAAS,GAAG,EAAE,CAAC;gBACnB,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBACjC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnD,CAAC;gBAED,IAAI,QAAQ,GAAa,MAAM,CAAC;gBAChC,IAAI,SAAS,EAAE,CAAC;oBACd,QAAQ,GAAG,SAAS,CAAC;gBACvB,CAAC;qBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC9B,QAAQ,GAAG,WAAW,CAAC;gBACzB,CAAC;gBAED,MAAM,WAAW,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBAEnD,MAAM,MAAM,GAAe;oBACzB,IAAI,EAAE,QAAQ;oBACd,aAAa,EAAE,WAAW;oBAC1B,IAAI;oBACJ,QAAQ;oBACR,SAAS;oBACT,SAAS,EAAE,OAAO;oBAClB,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI;oBACxC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;oBACpF,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;oBACtB,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE;oBAChC,UAAU,EAAE,SAAS;iBACtB,CAAC;gBAEF,IAAI,CAAC,aAAa,EAAE,CAAC;oBACnB,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBAChD,CAAC;gBAED,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;oBACxC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;oBACnE,IAAI,UAAU;wBAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;oBACnC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAC7B,CAAC;gBAED,IAAI,aAAa,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;oBACnC,MAAM,CAAC,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAC/D,CAAC;gBAED,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,2BAA2B,QAAQ,GAAG,EAAE;oBACxE,IAAI,EAAE,QAAQ;oBACd,SAAS,EAAE,QAAQ;oBACnB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAChD,CAAC,CAAC;gBACH,IAAI,OAAO,KAAK,MAAM;oBAAE,MAAM,KAAK,CAAC;gBACpC,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;QAEF,gCAAgC;QAChC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrF,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM;oBACJ,IAAI,EAAE,CAAC,MAA4C,CAAC;oBACpD,MAAM,EAAE,CAAC;iBACV,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,yEAAyE;QACzE,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC3C,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAQ/B,MAAM,QAAQ,GAA8C,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9F,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,EAAG,CAAC;YAClC,IAAI,SAAS,CAAC;YAEd,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,0BAA0B,OAAO,CAAC,OAAO,GAAG,EAAE;oBAC9E,IAAI,EAAE,OAAO,CAAC,OAAO;oBACrB,SAAS,EAAE,SAAS;oBACpB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAChD,CAAC,CAAC;gBACH,IAAI,OAAO,KAAK,MAAM;oBAAE,MAAM,KAAK,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAe,EAAE,CAAC;YAEnC,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;oBAE9B,wBAAwB;oBACxB,IAAI,CAAC,aAAa,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAChD,SAAS;oBACX,CAAC;oBAED,0CAA0C;oBAC1C,IAAI,CAAC,aAAa,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,SAAS,KAAK,cAAc,CAAC,EAAE,CAAC;wBAC7E,SAAS;oBACX,CAAC;oBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACvD,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAEzE,iBAAiB;oBACjB,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5B,SAAS;oBACX,CAAC;oBAED,IAAI,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;wBACzB,IAAI,SAAS,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;4BAC/C,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;wBACjE,CAAC;wBAED,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;4BACvD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;gCAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;4BAClD,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,IAAI,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC;wBACnC,IAAI,cAAc,EAAE,CAAC;4BACnB,IAAI,CAAC;gCACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gCACrD,IAAI,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oCACnC,uBAAuB;oCACvB,SAAS;gCACX,CAAC;gCACD,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gCAE/B,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gCACnD,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;oCAC7B,IAAI,SAAS,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;wCAC/C,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;oCACjE,CAAC;oCACD,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;wCACvD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;4CAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;wCAClD,CAAC;oCACH,CAAC;gCACH,CAAC;qCAAM,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;oCAC/B,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;wCAC9E,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;4CAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;wCAClD,CAAC;oCACH,CAAC;gCACH,CAAC;4BACH,CAAC;4BAAC,OAAO,GAAY,EAAE,CAAC;gCACtB,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;oCACvB,MAAM,IAAI,eAAe,CAAC,sBAAsB,QAAQ,GAAG,EAAE;wCAC3D,IAAI,EAAE,QAAQ;wCACd,SAAS,EAAE,UAAU;wCACrB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qCAChD,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;gCACrD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;oCAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;gCAClD,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;wBAC3B,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;4BAClD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;gCAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;4BAClD,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,sDAAsD;YACxD,CAAC;YAED,gEAAgE;YAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;gBACzD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBACvB,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;wBAC/C,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,cAAc;4BACvC,CAAC,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;4BACvC,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBACzC,OAAO,MAAM,eAAe,CAC1B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,OAAO,CAAC,OAAO,EACf,IAAI,CAAC,MAAM,CAAC,IAAI,EAChB,IAAI,EACJ,SAAS,CACV,CAAC;oBACJ,CAAC;oBAAC,OAAO,GAAY,EAAE,CAAC;wBACtB,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,gBAAgB,IAAI,CAAC,QAAQ,GAAG,EAAE;4BAClE,IAAI,EAAE,IAAI,CAAC,QAAQ;4BACnB,SAAS,EAAE,MAAM;4BACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;yBAChD,CAAC,CAAC;wBACH,IAAI,OAAO,KAAK,MAAM;4BAAE,MAAM,KAAK,CAAC;wBACpC,OAAO,IAAI,CAAC;oBACd,CAAC;gBACH,CAAC,CAAC,CACH,CAAC;gBAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,MAAM,EAAE,CAAC;wBACX,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC7B,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;wBAC3B,IAAI,KAAK,EAAE,CAAC;4BACV,MAAM,KAAK,CAAC;wBACd,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM;gBACJ,IAAI,EAAE,eAAuD;gBAC7D,MAAM;aACP,CAAC;YACF,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC;YACjC,eAAe,GAAG,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,OAAO,CAAC,QAAwB;QAC3C,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAElD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACzD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,UAAU,IAAI,CAAC,CAAC;gBAChB,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,QAAQ,CAAC;gBACvD,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAkC;YAC7C,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE;YACnD,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE;YAC5D,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE;YACnD,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE;YACxD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE;YACnD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,EAAE;YACpD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,EAAE;YACvD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,EAAE;YACxD,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,EAAE;YACvD,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,EAAE;SAC7D,CAAC;QAEF,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,OAAO;YACL,MAAM,EAAE,YAAY;YACpB,QAAQ,EAAE,UAAU;YACpB,SAAS,EAAE,cAAc;YACzB,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,OAAO;SACR,CAAC;IACJ,CAAC;CACF"}

package/dist/files/types.d.ts

@@ -0,0 +1,47 @@
+export type FileType = "file" | "directory" | "symlink";
+export type FileTypeFilter = "file" | "directory" | "symlink" | "all";
+export type HashAlgorithm = "sha256" | "sha1" | "md5" | "fast";
+export type FileCategory = "image" | "video" | "audio" | "archive" | "document" | "data" | "code" | "executable" | "font" | "other";
+export interface FileRecord {
+    path: string;
+    relative_path: string;
+    name: string;
+    basename: string;
+    extension: string;
+    directory: string;
+    type: FileType;
+    size: number;
+    created_at: string;
+    modified_at: string;
+    accessed_at?: string;
+    mode: number;
+    is_file: boolean;
+    is_directory: boolean;
+    is_symlink: boolean;
+    hash?: string;
+    mime?: string;
+    category?: FileCategory;
+    [key: string]: unknown;
+}
+export interface FileSystemReaderOptions {
+    root: string;
+    recursive?: boolean;
+    maxDepth?: number;
+    include?: string[];
+    exclude?: string[];
+    followSymlinks?: boolean;
+    hidden?: boolean;
+    type?: FileTypeFilter;
+    hash?: false | HashAlgorithm;
+    mime?: boolean;
+    category?: boolean;
+    concurrency?: number;
+    onError?: "fail" | "skip";
+    deterministic?: boolean;
+    batchSize?: number;
+}
+export interface FileMetadataEnricher {
+    name: string;
+    enrich(file: FileRecord): Promise<Record<string, unknown>>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/files/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/files/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;AAExD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,WAAW,GAAG,SAAS,GAAG,KAAK,CAAC;AAEtE,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GACpB,OAAO,GACP,OAAO,GACP,OAAO,GACP,SAAS,GACT,UAAU,GACV,MAAM,GACN,MAAM,GACN,YAAY,GACZ,MAAM,GACN,OAAO,CAAC;AAEZ,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,IAAI,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC;IAC7B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAC5D"}

package/dist/files/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/files/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/files/types.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -2,7 +2,12 @@ export * from "./core/types.js";
 export * from "./core/errors.js";
 export * from "./core/batch.js";
 export * from "./core/pipeline.js";
-export { CSVReader, type CSVReaderOptions, JSONReader, type JSONReaderOptions, JSONLReader, type JSONLReaderOptions, ParquetReader, type ParquetReaderOptions, XLSXReader, type XLSXReaderOptions, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
+export { CSVReader, type CSVReaderOptions, JSONReader, type JSONReaderOptions, JSONLReader, type JSONLReaderOptions, ParquetReader, type ParquetReaderOptions, XLSXReader, type XLSXReaderOptions, FileSystemReader, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
+export * from "./files/types.js";
+export * from "./files/glob.js";
+export * from "./files/mime.js";
+export * from "./files/hash.js";
+export * from "./files/reader.js";
 export { CSVWriter, type CSVWriterOptions, JSONWriter, JSONLWriter, MarkdownWriter, type MarkdownWriterOptions, ParquetWriter, type ParquetWriterOptions, XLSXWriter, type XLSXWriterOptions, createWriter, inferFormatFromPath as inferWriterFormatFromPath, } from "./writers/index.js";
 export * from "./transforms/select.js";
 export * from "./transforms/rename.js";
@@ -16,6 +21,16 @@ export * from "./analytics/reduce.js";
 export * from "./analytics/schema-inference.js";
 export * from "./analytics/semantic-types.js";
 export * from "./analytics/validator.js";
+export * from "./diff/types.js";
+export * from "./diff/key.js";
+export * from "./diff/comparator.js";
+export * from "./diff/hash.js";
+export * from "./diff/schema.js";
+export * from "./diff/engine.js";
+export * from "./diff/reporter.js";
+export * from "./diff/storage/memory-index.js";
+export * from "./diff/storage/disk-index.js";
+export * from "./diff/storage/spillable-index.js";
 export * from "./utils/compression.js";
 export * from "./utils/formatting.js";
 export * from "./utils/progress.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAGnC,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,KAAK,iBAAiB,EACtB,WAAW,EACX,KAAK,kBAAkB,EACvB,aAAa,EACb,KAAK,oBAAoB,EACzB,UAAU,EACV,KAAK,iBAAiB,EACtB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,WAAW,EACX,cAAc,EACd,KAAK,qBAAqB,EAC1B,aAAa,EACb,KAAK,oBAAoB,EACzB,UAAU,EACV,KAAK,iBAAiB,EACtB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAGzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAGnC,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,KAAK,iBAAiB,EACtB,WAAW,EACX,KAAK,kBAAkB,EACvB,aAAa,EACb,KAAK,oBAAoB,EACzB,UAAU,EACV,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAIlC,OAAO,EACL,SAAS,EACT,KAAK,gBAAgB,EACrB,UAAU,EACV,WAAW,EACX,cAAc,EACd,KAAK,qBAAqB,EAC1B,aAAa,EACb,KAAK,oBAAoB,EACzB,UAAU,EACV,KAAK,iBAAiB,EACtB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAG5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAGzC,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAGlD,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}

package/dist/index.js

@@ -4,7 +4,13 @@ export * from "./core/errors.js";
 export * from "./core/batch.js";
 export * from "./core/pipeline.js";
 // Readers
-export { CSVReader, JSONReader, JSONLReader, ParquetReader, XLSXReader, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
+export { CSVReader, JSONReader, JSONLReader, ParquetReader, XLSXReader, FileSystemReader, createReader, inferFormatFromPath as inferReaderFormatFromPath, } from "./readers/index.js";
+// Filesystem Subsystem
+export * from "./files/types.js";
+export * from "./files/glob.js";
+export * from "./files/mime.js";
+export * from "./files/hash.js";
+export * from "./files/reader.js";
 // Writers
 export { CSVWriter, JSONWriter, JSONLWriter, MarkdownWriter, ParquetWriter, XLSXWriter, createWriter, inferFormatFromPath as inferWriterFormatFromPath, } from "./writers/index.js";
 // Transforms
@@ -21,6 +27,17 @@ export * from "./analytics/reduce.js";
 export * from "./analytics/schema-inference.js";
 export * from "./analytics/semantic-types.js";
 export * from "./analytics/validator.js";
+// Diff Engine & Storage
+export * from "./diff/types.js";
+export * from "./diff/key.js";
+export * from "./diff/comparator.js";
+export * from "./diff/hash.js";
+export * from "./diff/schema.js";
+export * from "./diff/engine.js";
+export * from "./diff/reporter.js";
+export * from "./diff/storage/memory-index.js";
+export * from "./diff/storage/disk-index.js";
+export * from "./diff/storage/spillable-index.js";
 // Utilities
 export * from "./utils/compression.js";
 export * from "./utils/formatting.js";

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO;AACP,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAEnC,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EAEV,WAAW,EAEX,aAAa,EAEb,UAAU,EAEV,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EACV,WAAW,EACX,cAAc,EAEd,aAAa,EAEb,UAAU,EAEV,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,aAAa;AACb,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAEpC,YAAY;AACZ,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAEzC,YAAY;AACZ,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO;AACP,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAEnC,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EAEV,WAAW,EAEX,aAAa,EAEb,UAAU,EAEV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,uBAAuB;AACvB,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAGlC,UAAU;AACV,OAAO,EACL,SAAS,EAET,UAAU,EACV,WAAW,EACX,cAAc,EAEd,aAAa,EAEb,UAAU,EAEV,YAAY,EACZ,mBAAmB,IAAI,yBAAyB,GACjD,MAAM,oBAAoB,CAAC;AAE5B,aAAa;AACb,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAEpC,YAAY;AACZ,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AAEzC,wBAAwB;AACxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAElD,YAAY;AACZ,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}

package/dist/readers/csv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/readers/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAEV,UAAU,EACV,kBAAkB,EAClB,aAAa,EAEb,aAAa,EACd,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,qBAAa,SAAU,YAAW,aAAa;IAC7C,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,gBAAqB;IAQpE,OAAO,CAAC,cAAc;IAKtB;;OAEG;WACW,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IA2B9C,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU;IA+N1C,OAAO,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsBpE"}
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/readers/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAEV,UAAU,EACV,kBAAkB,EAClB,aAAa,EAEb,aAAa,EACd,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,qBAAa,SAAU,YAAW,aAAa;IAC7C,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,gBAAqB;IAQpE,OAAO,CAAC,cAAc;IAKtB;;OAEG;WACW,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IA2B9C,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU;IAqP1C,OAAO,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsBpE"}

package/dist/readers/csv.js

@@ -86,42 +86,51 @@ export class CSVReader {
             }
             const activeDelimiter = delimiter ?? ",";
             const delimLen = activeDelimiter.length;
+            const isSingleCharDelim = delimLen === 1;
+            const delimCode = activeDelimiter.charCodeAt(0);
+            let fieldStart = cursor;
             while (cursor < buffer.length) {
-                const char = buffer[cursor];
                 if (inQuotes) {
-                    if (char === '"') {
+                    const charCode = buffer.charCodeAt(cursor);
+                    if (charCode === 34 /* " */) {
                         if (cursor + 1 < buffer.length) {
-                            if (buffer[cursor + 1] === '"') {
-                                currentField += '"';
+                            if (buffer.charCodeAt(cursor + 1) === 34) {
+                                // Escaped double quote ("")
+                                currentField += buffer.slice(fieldStart, cursor) + '"';
                                 cursor += 2;
                                 byteOffset += 2;
+                                fieldStart = cursor;
                                 continue;
                             }
                             else {
+                                // Closing quote
+                                currentField += buffer.slice(fieldStart, cursor);
                                 inQuotes = false;
                                 cursor++;
                                 byteOffset++;
+                                fieldStart = cursor;
                                 continue;
                             }
                         }
                         else {
-                            // Reached end of current chunk buffer inside quotes; wait for next chunk
+                            // Reached end of current buffer inside quotes; wait for next chunk
                             break;
                         }
                     }
                     else {
-                        currentField += char;
                         cursor++;
                         byteOffset++;
                         continue;
                     }
                 }
                 else {
-                    if (char === '"') {
-                        if (currentField.length === 0) {
+                    const charCode = buffer.charCodeAt(cursor);
+                    if (charCode === 34 /* " */) {
+                        if (cursor === fieldStart && currentField.length === 0) {
                             inQuotes = true;
                             cursor++;
                             byteOffset++;
+                            fieldStart = cursor;
                             continue;
                         }
                         else {
@@ -133,27 +142,35 @@ export class CSVReader {
                             });
                         }
                     }
-                    // If remaining buffer is shorter than delimiter, wait for next chunk
-                    if (buffer.length - cursor < delimLen) {
-                        break;
-                    }
                     // Check for delimiter
-                    if (buffer.startsWith(activeDelimiter, cursor)) {
-                        currentRowFields.push(currentField);
+                    const isDelim = isSingleCharDelim
+                        ? charCode === delimCode
+                        : (buffer.length - cursor >= delimLen && buffer.startsWith(activeDelimiter, cursor));
+                    if (isDelim) {
+                        const fieldPart = buffer.slice(fieldStart, cursor);
+                        const val = currentField.length > 0 ? currentField + fieldPart : fieldPart;
+                        currentRowFields.push(val);
                         currentField = "";
                         cursor += delimLen;
                         byteOffset += delimLen;
+                        fieldStart = cursor;
                         continue;
                     }
+                    // If remaining buffer is shorter than multi-char delimiter, wait for next chunk
+                    if (!isSingleCharDelim && buffer.length - cursor < delimLen) {
+                        break;
+                    }
                     // Check for CRLF / LF newline
-                    if (char === "\r" || char === "\n") {
-                        if (char === "\r" && cursor + 1 >= buffer.length) {
+                    if (charCode === 13 /* \r */ || charCode === 10 /* \n */) {
+                        if (charCode === 13 && cursor + 1 >= buffer.length) {
                             // Wait for next chunk to check for \n
                             break;
                         }
-                        currentRowFields.push(currentField);
+                        const fieldPart = buffer.slice(fieldStart, cursor);
+                        const val = currentField.length > 0 ? currentField + fieldPart : fieldPart;
+                        currentRowFields.push(val);
                         currentField = "";
-                        if (char === "\r" && cursor + 1 < buffer.length && buffer[cursor + 1] === "\n") {
+                        if (charCode === 13 && cursor + 1 < buffer.length && buffer.charCodeAt(cursor + 1) === 10) {
                             cursor += 2;
                             byteOffset += 2;
                         }
@@ -161,6 +178,7 @@ export class CSVReader {
                             cursor++;
                             byteOffset++;
                         }
+                        fieldStart = cursor;
                         // Skip entirely empty row if it's trailing whitespace
                         if (currentRowFields.length === 1 &&
                             currentRowFields[0]?.trim() === "" &&
@@ -196,12 +214,14 @@ export class CSVReader {
                         }
                         continue;
                     }
-                    currentField += char;
                     cursor++;
                     byteOffset++;
                 }
             }
             // Compact processed buffer to keep memory bounded
+            if (fieldStart < cursor) {
+                currentField += buffer.slice(fieldStart, cursor);
+            }
             buffer = buffer.slice(cursor);
             cursor = 0;
         }