tar-vern 0.3.0 → 1.1.0

package/dist/index.js

@@ -1,24 +1,30 @@
 /*!
  * name: tar-vern
- * version: 0.3.0
+ * version: 1.1.0
  * description: Tape archiver library for Typescript
  * author: Kouji Matsui (@kekyo@mi.kekyo.net)
  * license: MIT
  * repository.url: https://github.com/kekyo/tar-vern.git
+ * git.commit.hash: 6d4ff13b538b16545ccc55b2e74f8e5f73999a34
  */
 import { Readable } from "stream";
-import { createGzip } from "zlib";
+import { createGzip, createGunzip } from "zlib";
 import { createReadStream, createWriteStream } from "fs";
-import { stat } from "fs/promises";
+import { stat, mkdir, writeFile, readdir } from "fs/promises";
+import { pipeline } from "stream/promises";
+import { join, dirname } from "path";
+const MAX_NAME = 100;
+const MAX_PREFIX = 155;
 const getUName = (candidateName, candidateId, reflectStat) => {
   return candidateName ?? (reflectStat === "all" ? candidateId.toString() : "root");
 };
 const getBuffer = (data) => {
   return Buffer.isBuffer(data) ? data : Buffer.from(data, "utf8");
 };
-const createDirectoryItem = async (path, reflectStat, options) => {
+const createDirectoryItem = async (path, reflectStat, options, signal) => {
   const rs = reflectStat ?? "none";
   if (rs !== "none" && options?.directoryPath) {
+    signal?.throwIfAborted();
     const stats = await stat(options.directoryPath);
     const mode = options?.mode ?? stats.mode;
     const uid = options?.uid ?? stats.uid;
@@ -55,7 +61,8 @@ const createDirectoryItem = async (path, reflectStat, options) => {
     };
   }
 };
-const createFileItem = async (path, content, options) => {
+const createFileItem = async (path, content, options, signal) => {
+  signal?.throwIfAborted();
   const mode = options?.mode ?? 420;
   const uid = options?.uid ?? 0;
   const gid = options?.gid ?? 0;
@@ -74,7 +81,7 @@ const createFileItem = async (path, content, options) => {
     content
   };
 };
-const createReadableFileItem = async (path, readable, options) => {
+const createReadableFileItem = async (path, readable, options, signal) => {
   const mode = options?.mode ?? 420;
   const uid = options?.uid ?? 0;
   const gid = options?.gid ?? 0;
@@ -86,6 +93,7 @@ const createReadableFileItem = async (path, readable, options) => {
     const chunks = [];
     length = 0;
     for await (const chunk of readable) {
+      signal?.throwIfAborted();
       const buffer = getBuffer(chunk);
       chunks.push(buffer);
       length += buffer.length;
@@ -102,7 +110,7 @@ const createReadableFileItem = async (path, readable, options) => {
       content: {
         kind: "readable",
         length,
-        readable: Readable.from(chunks)
+        readable: Readable.from(chunks, { signal })
       }
     };
   } else {
@@ -123,7 +131,7 @@ const createReadableFileItem = async (path, readable, options) => {
     };
   }
 };
-const createGeneratorFileItem = async (path, generator, options) => {
+const createGeneratorFileItem = async (path, generator, options, signal) => {
   const mode = options?.mode ?? 420;
   const uid = options?.uid ?? 0;
   const gid = options?.gid ?? 0;
@@ -135,6 +143,7 @@ const createGeneratorFileItem = async (path, generator, options) => {
     const chunks = [];
     length = 0;
     for await (const chunk of generator) {
+      signal?.throwIfAborted();
       const buffer = getBuffer(chunk);
       chunks.push(buffer);
       length += buffer.length;
@@ -151,7 +160,7 @@ const createGeneratorFileItem = async (path, generator, options) => {
       content: {
         kind: "readable",
         length,
-        readable: Readable.from(chunks)
+        readable: Readable.from(chunks, { signal })
       }
     };
   } else {
@@ -172,10 +181,11 @@ const createGeneratorFileItem = async (path, generator, options) => {
     };
   }
 };
-const createReadFileItem = async (path, filePath, reflectStat, options) => {
+const createReadFileItem = async (path, filePath, reflectStat, options, signal) => {
   const rs = reflectStat ?? "exceptName";
+  signal?.throwIfAborted();
   const stats = await stat(filePath);
-  const reader = createReadStream(filePath);
+  const reader = createReadStream(filePath, { signal });
   const mode = options?.mode ?? (rs !== "none" ? stats.mode : void 0);
   const uid = options?.uid ?? (rs !== "none" ? stats.uid : void 0);
   const gid = options?.gid ?? (rs !== "none" ? stats.gid : void 0);
@@ -190,16 +200,85 @@ const createReadFileItem = async (path, filePath, reflectStat, options) => {
     uid,
     gid,
     date
-  });
+  }, signal);
+};
+const storeReaderToFile = async (reader, path, signal) => {
+  const writer = createWriteStream(path, { signal });
+  await pipeline(reader, writer, { signal });
 };
-const storeReaderToFile = (reader, path) => {
-  const writer = createWriteStream(path);
-  reader.pipe(writer);
-  return new Promise((res, rej) => {
-    writer.on("finish", res);
-    writer.on("error", rej);
-    reader.on("error", rej);
-  });
+const getAllFilesInDirectory = async (baseDir, signal) => {
+  const collectFiles = async (currentDir, relativePath) => {
+    signal?.throwIfAborted();
+    try {
+      const entries = await readdir(currentDir, { withFileTypes: true });
+      const result = [];
+      const tasks = entries.map(async (entry) => {
+        signal?.throwIfAborted();
+        const entryRelativePath = join(relativePath, entry.name);
+        if (entry.isDirectory()) {
+          const entryFullPath = join(currentDir, entry.name);
+          const directoryContents = await collectFiles(entryFullPath, entryRelativePath);
+          return [entryRelativePath, ...directoryContents];
+        } else {
+          return [entryRelativePath];
+        }
+      });
+      const allResults = await Promise.all(tasks);
+      for (const entryResults of allResults) {
+        result.push(...entryResults);
+      }
+      return result;
+    } catch (error) {
+      console.warn(`Warning: Could not read directory ${currentDir}:`, error);
+      return [];
+    }
+  };
+  return await collectFiles(baseDir, "");
+};
+const createEntryItemGenerator = async function* (baseDir, relativePaths, reflectStat, signal) {
+  const rs = reflectStat ?? "exceptName";
+  const pathsToProcess = relativePaths ?? await getAllFilesInDirectory(baseDir, signal);
+  for (const relativePath of pathsToProcess) {
+    signal?.throwIfAborted();
+    const fsPath = join(baseDir, relativePath);
+    try {
+      signal?.throwIfAborted();
+      const stats = await stat(fsPath);
+      if (stats.isDirectory()) {
+        yield await createDirectoryItem(relativePath, rs, {
+          directoryPath: fsPath
+        }, signal);
+      } else if (stats.isFile()) {
+        yield await createReadFileItem(relativePath, fsPath, rs, void 0, signal);
+      }
+    } catch (error) {
+      console.warn(`Warning: Could not access ${fsPath}:`, error);
+      continue;
+    }
+  }
+};
+const extractTo = async (iterator, basePath, signal) => {
+  for await (const entry of iterator) {
+    signal?.throwIfAborted();
+    const targetPath = join(basePath, entry.path);
+    if (entry.kind === "directory") {
+      try {
+        signal?.throwIfAborted();
+        await mkdir(targetPath, { recursive: true, mode: entry.mode });
+      } catch (error) {
+        if (error.code !== "EEXIST") {
+          throw error;
+        }
+      }
+    } else if (entry.kind === "file") {
+      const parentDir = dirname(targetPath);
+      signal?.throwIfAborted();
+      await mkdir(parentDir, { recursive: true });
+      const fileEntry = entry;
+      const content = await fileEntry.getContent("buffer");
+      await writeFile(targetPath, content, { mode: entry.mode, signal });
+    }
+  }
 };
 const utf8ByteLength = (str) => {
   return Buffer.byteLength(str, "utf8");
@@ -217,8 +296,6 @@ const truncateUtf8Safe = (str, maxBytes) => {
   }
   return str.slice(0, i);
 };
-const MAX_NAME = 100;
-const MAX_PREFIX = 155;
 const splitPath = (path) => {
   if (utf8ByteLength(path) <= MAX_NAME) {
     return { prefix: "", name: path };
@@ -299,10 +376,11 @@ const createTarPacker = (entryItemGenerator, compressionType, signal) => {
             const totalPaddedContentBytes = getPaddedBytes(contentBytes);
             yield totalPaddedContentBytes;
           } else {
+            const content = entryItemContent;
             const tarHeaderBytes = createTarHeader(
               "file",
               entryItem.path,
-              entryItemContent.length,
+              content.length,
               entryItem.mode,
               entryItem.uname,
               entryItem.gname,
@@ -312,10 +390,10 @@ const createTarPacker = (entryItemGenerator, compressionType, signal) => {
             );
             yield tarHeaderBytes;
             let position = 0;
-            switch (entryItemContent.kind) {
+            switch (content.kind) {
               // Content is a generator
               case "generator": {
-                for await (const contentBytes of entryItemContent.generator) {
+                for await (const contentBytes of content.generator) {
                   signal?.throwIfAborted();
                   yield contentBytes;
                   position += contentBytes.length;
@@ -324,9 +402,9 @@ const createTarPacker = (entryItemGenerator, compressionType, signal) => {
               }
               // Content is a readable stream
               case "readable": {
-                for await (const content of entryItemContent.readable) {
+                for await (const chunk of content.readable) {
                   signal?.throwIfAborted();
-                  const contentBytes = getBuffer(content);
+                  const contentBytes = getBuffer(chunk);
                   yield contentBytes;
                   position += contentBytes.length;
                 }
@@ -364,25 +442,282 @@ const createTarPacker = (entryItemGenerator, compressionType, signal) => {
   switch (ct) {
     // No compression
     case "none": {
-      return Readable.from(entryItemIterator());
+      return Readable.from(entryItemIterator(), { signal });
     }
     // Gzip compression
     case "gzip": {
       const gzipStream = createGzip({ level: 9 });
-      const entryItemStream = Readable.from(entryItemIterator());
+      const entryItemStream = Readable.from(entryItemIterator(), { signal });
       entryItemStream.pipe(gzipStream);
       return gzipStream;
     }
   }
 };
+const parseOctalBytes = (buffer, offset, length) => {
+  const str = buffer.subarray(offset, offset + length).toString("ascii").replace(/\0/g, "").trim();
+  return str ? parseInt(str, 8) : 0;
+};
+const parseString = (buffer, offset, length) => {
+  return buffer.subarray(offset, offset + length).toString("utf8").replace(/\0/g, "").trim();
+};
+const readExactBytes = async (iterator, size, signal) => {
+  const chunks = [];
+  let totalRead = 0;
+  while (totalRead < size) {
+    signal?.throwIfAborted();
+    const { value, done } = await iterator.next();
+    if (done) {
+      if (totalRead === 0) {
+        return void 0;
+      } else {
+        throw new Error(`Unexpected end of stream: expected ${size} bytes, got ${totalRead} bytes`);
+      }
+    }
+    const chunk = getBuffer(value);
+    const needed = size - totalRead;
+    if (chunk.length <= needed) {
+      chunks.push(chunk);
+      totalRead += chunk.length;
+    } else {
+      chunks.push(chunk.subarray(0, needed));
+      await iterator.return?.(chunk.subarray(needed));
+      totalRead = size;
+    }
+  }
+  return Buffer.concat(chunks, size);
+};
+const skipExactBytes = async (iterator, size, signal) => {
+  let totalSkipped = 0;
+  while (totalSkipped < size) {
+    signal?.throwIfAborted();
+    const { value, done } = await iterator.next();
+    if (done) {
+      throw new Error(`Unexpected end of stream: expected to skip ${size} bytes, skipped ${totalSkipped} bytes`);
+    }
+    const chunk = getBuffer(value);
+    const needed = size - totalSkipped;
+    if (chunk.length <= needed) {
+      totalSkipped += chunk.length;
+    } else {
+      await iterator.return?.(chunk.subarray(needed));
+      totalSkipped = size;
+    }
+  }
+};
+const skipPaddingBytesTo512Boundary = async (iterator, contentSize, signal) => {
+  const padding = (512 - contentSize % 512) % 512;
+  if (padding > 0) {
+    await skipExactBytes(iterator, padding, signal);
+  }
+};
+const parseTarHeader = (buffer) => {
+  if (buffer.every((b) => b === 0)) {
+    return void 0;
+  }
+  const name = parseString(buffer, 0, 100);
+  const mode = parseOctalBytes(buffer, 100, 8);
+  const uid = parseOctalBytes(buffer, 108, 8);
+  const gid = parseOctalBytes(buffer, 116, 8);
+  const size = parseOctalBytes(buffer, 124, 12);
+  const mtime = new Date(parseOctalBytes(buffer, 136, 12) * 1e3);
+  const checksum = parseOctalBytes(buffer, 148, 8);
+  const typeflag = parseString(buffer, 156, 1);
+  const magic = parseString(buffer, 257, 6);
+  const uname = parseString(buffer, 265, 32);
+  const gname = parseString(buffer, 297, 32);
+  const prefix = parseString(buffer, 345, 155);
+  if (magic !== "ustar") {
+    throw new Error(`Invalid tar format: magic="${magic}"`);
+  }
+  let calculatedSum = 0;
+  for (let i = 0; i < 512; i++) {
+    if (i >= 148 && i < 156) {
+      calculatedSum += 32;
+    } else {
+      calculatedSum += buffer[i];
+    }
+  }
+  if (calculatedSum !== checksum) {
+    throw new Error(`Invalid checksum: expected ${checksum}, got ${calculatedSum}`);
+  }
+  let path = prefix ? `${prefix}/${name}` : name;
+  if (path.endsWith("/")) {
+    path = path.slice(0, -1);
+  }
+  const kind = typeflag === "5" ? "directory" : "file";
+  return {
+    kind,
+    path,
+    size,
+    mode,
+    uid,
+    gid,
+    mtime,
+    uname: uname || uid.toString(),
+    gname: gname || gid.toString(),
+    checksum,
+    consumed: false
+  };
+};
+const createBufferedAsyncIterator = (iterable, signal) => {
+  const buffer = [];
+  const iterator = iterable[Symbol.asyncIterator]();
+  return {
+    next: async () => {
+      signal?.throwIfAborted();
+      if (buffer.length > 0) {
+        return { value: buffer.shift(), done: false };
+      }
+      return iterator.next();
+    },
+    return: async (value) => {
+      if (value !== void 0) {
+        buffer.unshift(value);
+      }
+      return { value: void 0, done: false };
+    }
+  };
+};
+const createReadableFromIterator = (iterator, size, signal, consumedRef) => {
+  const generator = async function* () {
+    let remainingBytes = size;
+    while (remainingBytes > 0) {
+      signal?.throwIfAborted();
+      const { value, done } = await iterator.next();
+      if (done) {
+        throw new Error(`Unexpected end of stream: expected ${size} bytes, remaining ${remainingBytes} bytes`);
+      }
+      const chunk = getBuffer(value);
+      if (chunk.length <= remainingBytes) {
+        remainingBytes -= chunk.length;
+        yield chunk;
+      } else {
+        const needed = chunk.subarray(0, remainingBytes);
+        const excess = chunk.subarray(remainingBytes);
+        remainingBytes = 0;
+        await iterator.return?.(excess);
+        yield needed;
+        break;
+      }
+    }
+    await skipPaddingBytesTo512Boundary(iterator, size, signal);
+    consumedRef.consumed = true;
+  };
+  return Readable.from(generator(), { signal });
+};
+const createTarExtractor = async function* (readable, compressionType, signal) {
+  const ct = compressionType ?? "none";
+  let inputStream;
+  switch (ct) {
+    case "gzip":
+      const gunzip = createGunzip();
+      readable.pipe(gunzip);
+      inputStream = gunzip;
+      break;
+    case "none":
+    default:
+      inputStream = readable;
+      break;
+  }
+  const iterator = createBufferedAsyncIterator(inputStream, signal);
+  let header;
+  while (true) {
+    signal?.throwIfAborted();
+    if (header?.kind === "file" && !header.consumed) {
+      await skipExactBytes(iterator, header.size, signal);
+      await skipPaddingBytesTo512Boundary(iterator, header.size, signal);
+      header.consumed = true;
+    }
+    let headerBuffer;
+    try {
+      headerBuffer = await readExactBytes(iterator, 512, signal);
+    } catch (error) {
+      if (error instanceof Error && error.message.includes("Unexpected end of stream")) {
+        throw new Error("Invalid tar format: incomplete header");
+      }
+      throw error;
+    }
+    if (headerBuffer === void 0) {
+      break;
+    }
+    header = parseTarHeader(headerBuffer);
+    if (!header) {
+      const secondBlock = await readExactBytes(iterator, 512, signal);
+      if (secondBlock === void 0 || secondBlock.every((b) => b === 0)) {
+        break;
+      }
+      throw new Error("Invalid tar format: expected terminator block");
+    }
+    if (header.kind === "directory") {
+      yield {
+        kind: "directory",
+        path: header.path,
+        mode: header.mode,
+        uid: header.uid,
+        gid: header.gid,
+        uname: header.uname,
+        gname: header.gname,
+        date: header.mtime
+      };
+    } else {
+      const currentHeader = header;
+      yield {
+        kind: "file",
+        path: currentHeader.path,
+        mode: currentHeader.mode,
+        uid: currentHeader.uid,
+        gid: currentHeader.gid,
+        uname: currentHeader.uname,
+        gname: currentHeader.gname,
+        date: currentHeader.mtime,
+        getContent: async (type) => {
+          if (currentHeader.consumed) {
+            throw new Error("Content has already been consumed. Multiple calls to getContent are not supported.");
+          }
+          switch (type) {
+            // For string
+            case "string": {
+              const dataBuffer = await readExactBytes(iterator, currentHeader.size, signal);
+              if (dataBuffer === void 0) {
+                throw new Error(`Unexpected end of stream while reading file data for ${currentHeader.path}`);
+              }
+              await skipPaddingBytesTo512Boundary(iterator, currentHeader.size, signal);
+              currentHeader.consumed = true;
+              return dataBuffer.toString("utf8");
+            }
+            // For buffer
+            case "buffer": {
+              const dataBuffer = await readExactBytes(iterator, currentHeader.size, signal);
+              if (dataBuffer === void 0) {
+                throw new Error(`Unexpected end of stream while reading file data for ${currentHeader.path}`);
+              }
+              await skipPaddingBytesTo512Boundary(iterator, currentHeader.size, signal);
+              currentHeader.consumed = true;
+              return dataBuffer;
+            }
+            // For Readble stream
+            case "readable": {
+              const readable2 = createReadableFromIterator(iterator, currentHeader.size, signal, currentHeader);
+              return readable2;
+            }
+            default:
+              throw new Error(`Unsupported content type: ${type}`);
+          }
+        }
+      };
+    }
+  }
+};
 export {
   createDirectoryItem,
+  createEntryItemGenerator,
   createFileItem,
   createGeneratorFileItem,
   createReadFileItem,
   createReadableFileItem,
+  createTarExtractor,
   createTarPacker,
-  getBuffer,
+  extractTo,
   storeReaderToFile
 };
 //# sourceMappingURL=index.js.map