pushwork 1.0.0

package/src/utils/fs.ts

@@ -0,0 +1,289 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+import * as crypto from "crypto";
+import { glob } from "glob";
+import * as mimeTypes from "mime-types";
+import { FileSystemEntry, FileType } from "../types";
+import { isEnhancedTextFile } from "./mime-types";
+
+/**
+ * Check if a path exists
+ */
+export async function pathExists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Get file system entry metadata
+ */
+export async function getFileSystemEntry(
+  filePath: string
+): Promise<FileSystemEntry | null> {
+  try {
+    const stats = await fs.stat(filePath);
+    const type = stats.isDirectory()
+      ? FileType.DIRECTORY
+      : (await isEnhancedTextFile(filePath))
+      ? FileType.TEXT
+      : FileType.BINARY;
+
+    return {
+      path: filePath,
+      type,
+      size: stats.size,
+      mtime: stats.mtime,
+      permissions: stats.mode & parseInt("777", 8),
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Determine if a file is text or binary
+ */
+export async function isTextFile(filePath: string): Promise<boolean> {
+  try {
+    const mimeType = mimeTypes.lookup(filePath);
+    if (mimeType) {
+      return (
+        mimeType.startsWith("text/") ||
+        mimeType === "application/json" ||
+        mimeType === "application/xml" ||
+        mimeType.includes("javascript") ||
+        mimeType.includes("typescript")
+      );
+    }
+
+    // Sample first 8KB to detect binary content
+    const handle = await fs.open(filePath, "r");
+    const buffer = Buffer.alloc(Math.min(8192, (await handle.stat()).size));
+    await handle.read(buffer, 0, buffer.length, 0);
+    await handle.close();
+
+    // Check for null bytes which indicate binary content
+    return !buffer.includes(0);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Read file content as string or buffer
+ */
+export async function readFileContent(
+  filePath: string
+): Promise<string | Uint8Array> {
+  const isText = await isEnhancedTextFile(filePath);
+
+  if (isText) {
+    return await fs.readFile(filePath, "utf8");
+  } else {
+    const buffer = await fs.readFile(filePath);
+    return new Uint8Array(buffer);
+  }
+}
+
+/**
+ * Write file content from string or buffer
+ */
+export async function writeFileContent(
+  filePath: string,
+  content: string | Uint8Array
+): Promise<void> {
+  await ensureDirectoryExists(path.dirname(filePath));
+
+  if (typeof content === "string") {
+    await fs.writeFile(filePath, content, "utf8");
+  } else {
+    await fs.writeFile(filePath, content);
+  }
+}
+
+/**
+ * Ensure directory exists, creating it if necessary
+ */
+export async function ensureDirectoryExists(dirPath: string): Promise<void> {
+  try {
+    await fs.mkdir(dirPath, { recursive: true });
+  } catch (error: any) {
+    if (error.code !== "EEXIST") {
+      throw error;
+    }
+  }
+}
+
+/**
+ * Remove file or directory
+ */
+export async function removePath(filePath: string): Promise<void> {
+  try {
+    const stats = await fs.stat(filePath);
+    if (stats.isDirectory()) {
+      await fs.rmdir(filePath, { recursive: true });
+    } else {
+      await fs.unlink(filePath);
+    }
+  } catch (error: any) {
+    if (error.code !== "ENOENT") {
+      throw error;
+    }
+  }
+}
+
+/**
+ * Check if a path matches any of the exclude patterns
+ */
+function isExcluded(
+  filePath: string,
+  basePath: string,
+  excludePatterns: string[]
+): boolean {
+  const relativePath = path.relative(basePath, filePath);
+
+  for (const pattern of excludePatterns) {
+    // Handle different pattern types
+    if (pattern.startsWith(".") && !pattern.includes("*")) {
+      // Directory pattern like ".pushwork" or ".git"
+      if (
+        relativePath.startsWith(pattern) ||
+        relativePath.includes(`/${pattern}/`) ||
+        relativePath.includes(`\\${pattern}\\`)
+      ) {
+        return true;
+      }
+    } else if (pattern.includes("*")) {
+      // Glob pattern like "*.tmp"
+      const regex = new RegExp(
+        pattern.replace(/\*/g, ".*").replace(/\?/g, ".")
+      );
+      if (regex.test(relativePath)) {
+        return true;
+      }
+    } else {
+      // Exact directory name like "node_modules"
+      const parts = relativePath.split(/[/\\]/);
+      if (parts.includes(pattern)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+/**
+ * List directory contents with metadata
+ */
+export async function listDirectory(
+  dirPath: string,
+  recursive = false,
+  excludePatterns: string[] = []
+): Promise<FileSystemEntry[]> {
+  const entries: FileSystemEntry[] = [];
+
+  try {
+    const pattern = recursive
+      ? path.join(dirPath, "**/*")
+      : path.join(dirPath, "*");
+
+    // Convert exclude patterns to glob ignore patterns
+    const ignorePatterns = excludePatterns.map((pattern) => {
+      if (pattern.startsWith(".") && !pattern.includes("*")) {
+        // Directory patterns
+        return `${pattern}/**`;
+      }
+      return pattern;
+    });
+
+    const paths = await glob(pattern, {
+      dot: true,
+      ignore: ignorePatterns,
+    });
+
+    for (const filePath of paths) {
+      // Additional filtering for safety
+      if (!isExcluded(filePath, dirPath, excludePatterns)) {
+        const entry = await getFileSystemEntry(filePath);
+        if (entry) {
+          entries.push(entry);
+        }
+      }
+    }
+  } catch {
+    // Return empty array if directory doesn't exist or can't be read
+  }
+
+  return entries;
+}
+
+/**
+ * Copy file with metadata preservation
+ */
+export async function copyFile(
+  sourcePath: string,
+  destPath: string
+): Promise<void> {
+  await ensureDirectoryExists(path.dirname(destPath));
+  await fs.copyFile(sourcePath, destPath);
+
+  // Preserve file permissions
+  const stats = await fs.stat(sourcePath);
+  await fs.chmod(destPath, stats.mode);
+}
+
+/**
+ * Move/rename file or directory
+ */
+export async function movePath(
+  sourcePath: string,
+  destPath: string
+): Promise<void> {
+  await ensureDirectoryExists(path.dirname(destPath));
+  await fs.rename(sourcePath, destPath);
+}
+
+/**
+ * Calculate content hash for change detection
+ */
+export async function calculateContentHash(
+  content: string | Uint8Array
+): Promise<string> {
+  const hash = crypto.createHash("sha256");
+  hash.update(content);
+  return hash.digest("hex");
+}
+
+/**
+ * Get MIME type for file
+ */
+export function getMimeType(filePath: string): string {
+  return mimeTypes.lookup(filePath) || "application/octet-stream";
+}
+
+/**
+ * Get file extension
+ */
+export function getFileExtension(filePath: string): string {
+  const ext = path.extname(filePath);
+  return ext.startsWith(".") ? ext.slice(1) : ext;
+}
+
+/**
+ * Normalize path separators for cross-platform compatibility
+ */
+export function normalizePath(filePath: string): string {
+  return path.posix.normalize(filePath.replace(/\\/g, "/"));
+}
+
+/**
+ * Get relative path from base directory
+ */
+export function getRelativePath(basePath: string, filePath: string): string {
+  return normalizePath(path.relative(basePath, filePath));
+}

package/src/utils/index.ts

@@ -0,0 +1,8 @@
+// File system utilities
+export * from "./fs";
+
+// Content similarity utilities
+export * from "./content-similarity";
+
+// Enhanced MIME type detection
+export * from "./mime-types";

package/src/utils/mime-types.ts

@@ -0,0 +1,236 @@
+import * as mimeTypes from "mime-types";
+
+/**
+ * Custom MIME type definitions for developer files
+ * Based on patchwork-cli's approach
+ */
+const CUSTOM_MIME_TYPES: Record<string, string> = {
+  // TypeScript files - override the incorrect video/mp2t detection
+  ".ts": "text/typescript",
+  ".tsx": "text/tsx",
+
+  // Config file formats
+  ".json": "application/json",
+  ".yaml": "text/yaml",
+  ".yml": "text/yaml",
+  ".toml": "application/toml",
+  ".ini": "text/plain",
+  ".conf": "text/plain",
+  ".config": "text/plain",
+
+  // Vue.js single file components
+  ".vue": "text/vue",
+
+  // Modern CSS preprocessors
+  ".scss": "text/scss",
+  ".sass": "text/sass",
+  ".less": "text/less",
+  ".styl": "text/stylus",
+
+  // Modern JavaScript variants
+  ".mjs": "application/javascript",
+  ".cjs": "application/javascript",
+
+  // React JSX
+  ".jsx": "text/jsx",
+
+  // Svelte components
+  ".svelte": "text/svelte",
+
+  // Web assembly
+  ".wasm": "application/wasm",
+
+  // Other common dev files
+  ".d.ts": "text/typescript",
+  ".map": "application/json", // Source maps
+  ".env": "text/plain",
+  ".gitignore": "text/plain",
+  ".gitattributes": "text/plain",
+  ".editorconfig": "text/plain",
+  ".prettierrc": "application/json",
+  ".eslintrc": "application/json",
+  ".babelrc": "application/json",
+
+  // Documentation formats
+  ".mdx": "text/markdown",
+  ".rst": "text/x-rst",
+
+  // Docker files
+  Dockerfile: "text/plain",
+  ".dockerignore": "text/plain",
+
+  // Package manager files
+  "package.json": "application/json",
+  "package-lock.json": "application/json",
+  "yarn.lock": "text/plain",
+  "pnpm-lock.yaml": "text/yaml",
+  "composer.json": "application/json",
+  Pipfile: "text/plain",
+  "requirements.txt": "text/plain",
+
+  // Build tool configs
+  "webpack.config.js": "application/javascript",
+  "vite.config.js": "application/javascript",
+  "rollup.config.js": "application/javascript",
+  "tsconfig.json": "application/json",
+  "jsconfig.json": "application/json",
+};
+
+/**
+ * File extensions that should always be treated as text
+ * regardless of MIME type detection
+ */
+const FORCE_TEXT_EXTENSIONS = new Set([
+  ".ts",
+  ".tsx",
+  ".jsx",
+  ".vue",
+  ".svelte",
+  ".scss",
+  ".sass",
+  ".less",
+  ".styl",
+  ".env",
+  ".gitignore",
+  ".gitattributes",
+  ".editorconfig",
+  ".d.ts",
+  ".map",
+  ".mdx",
+  ".rst",
+  ".toml",
+  ".ini",
+  ".conf",
+  ".config",
+  ".lock",
+]);
+
+/**
+ * Get enhanced MIME type for file with custom dev file support
+ */
+export function getEnhancedMimeType(filePath: string): string {
+  const filename = filePath.split("/").pop() || "";
+  const extension = getFileExtension(filePath);
+
+  // Check custom definitions first (by extension)
+  if (extension && CUSTOM_MIME_TYPES[extension]) {
+    return CUSTOM_MIME_TYPES[extension];
+  }
+
+  // Check custom definitions by full filename
+  if (CUSTOM_MIME_TYPES[filename]) {
+    return CUSTOM_MIME_TYPES[filename];
+  }
+
+  // Fall back to standard mime-types library
+  const standardMime = mimeTypes.lookup(filePath);
+  if (standardMime) {
+    return standardMime;
+  }
+
+  // Final fallback
+  return "application/octet-stream";
+}
+
+/**
+ * Check if file extension should be forced to text type
+ */
+export function shouldForceAsText(filePath: string): boolean {
+  const extension = getFileExtension(filePath);
+  return extension ? FORCE_TEXT_EXTENSIONS.has(extension) : false;
+}
+
+/**
+ * Get file extension including the dot (internal helper)
+ */
+function getFileExtension(filePath: string): string {
+  const match = filePath.match(/\.[^.]*$/);
+  return match ? match[0] : "";
+}
+
+/**
+ * Enhanced text file detection with developer file support
+ */
+export async function isEnhancedTextFile(filePath: string): Promise<boolean> {
+  // Force certain extensions to be treated as text
+  if (shouldForceAsText(filePath)) {
+    return true;
+  }
+
+  // Check MIME type
+  const mimeType = getEnhancedMimeType(filePath);
+  if (isTextMimeType(mimeType)) {
+    return true;
+  }
+
+  // If it's a known binary type (but not the generic fallback), don't fall back to content detection
+  if (isBinaryMimeType(mimeType) && mimeType !== "application/octet-stream") {
+    return false;
+  }
+
+  // For generic octet-stream or unknown types, use content-based detection
+  return isTextByContent(filePath);
+}
+
+/**
+ * Check if MIME type indicates text content
+ */
+function isTextMimeType(mimeType: string): boolean {
+  return (
+    mimeType.startsWith("text/") ||
+    mimeType === "application/json" ||
+    mimeType === "application/xml" ||
+    mimeType === "application/javascript" ||
+    mimeType === "application/typescript" ||
+    mimeType === "application/toml" ||
+    mimeType.includes("javascript") ||
+    mimeType.includes("typescript") ||
+    mimeType.includes("json") ||
+    mimeType.includes("xml")
+  );
+}
+
+/**
+ * Check if MIME type indicates binary content
+ */
+function isBinaryMimeType(mimeType: string): boolean {
+  return (
+    mimeType.startsWith("image/") ||
+    mimeType.startsWith("video/") ||
+    mimeType.startsWith("audio/") ||
+    mimeType.startsWith("font/") ||
+    mimeType === "application/zip" ||
+    mimeType === "application/pdf" ||
+    mimeType === "application/octet-stream" ||
+    mimeType === "application/wasm" ||
+    mimeType.includes("binary")
+  );
+}
+
+/**
+ * Content-based text detection (fallback method)
+ */
+async function isTextByContent(filePath: string): Promise<boolean> {
+  try {
+    const fs = await import("fs/promises");
+
+    // Sample first 8KB to detect binary content
+    const handle = await fs.open(filePath, "r");
+    const stats = await handle.stat();
+    const sampleSize = Math.min(8192, stats.size);
+
+    if (sampleSize === 0) {
+      await handle.close();
+      return true; // Empty file is text
+    }
+
+    const buffer = Buffer.alloc(sampleSize);
+    await handle.read(buffer, 0, sampleSize, 0);
+    await handle.close();
+
+    // Check for null bytes which indicate binary content
+    return !buffer.includes(0);
+  } catch {
+    return false;
+  }
+}

package/src/utils/network-sync.ts

@@ -0,0 +1,153 @@
+import { DocHandle, StorageId } from "@automerge/automerge-repo";
+import * as A from "@automerge/automerge";
+
+/**
+ * Wait for documents to sync to the remote server
+ * Based on patchwork-cli implementation with timeout for debugging
+ */
+export async function waitForSync(
+  handlesToWaitOn: DocHandle<unknown>[],
+  syncServerStorageId?: StorageId,
+  timeoutMs: number = 60000 // 60 second timeout for debugging
+): Promise<void> {
+  if (!syncServerStorageId) {
+    console.warn(
+      "No sync server storage ID provided. Skipping network sync wait."
+    );
+    return;
+  }
+
+  if (handlesToWaitOn.length === 0) {
+    console.log("🔄 No documents to sync");
+    return;
+  }
+
+  // Debug logging only in verbose mode (can be controlled via env var later)
+  const verbose = false;
+
+  if (verbose) {
+    console.log(
+      `🔄 Waiting for ${handlesToWaitOn.length} documents to sync...`
+    );
+    console.log(`📡 Using sync server storage ID: ${syncServerStorageId}`);
+
+    handlesToWaitOn.forEach((handle, i) => {
+      const localHeads = handle.heads();
+      const syncInfo = handle.getSyncInfo(syncServerStorageId);
+      const remoteHeads = syncInfo?.lastHeads;
+      console.log(`  📄 Document ${i + 1}: ${handle.url}`);
+      console.log(`    🏠 Local heads: ${JSON.stringify(localHeads)}`);
+      console.log(`    🌐 Remote heads: ${JSON.stringify(remoteHeads)}`);
+      console.log(
+        `    ✅ Already synced: ${A.equals(localHeads, remoteHeads)}`
+      );
+    });
+  }
+
+  const promises = handlesToWaitOn.map(
+    (handle, index) =>
+      new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          const localHeads = handle.heads();
+          const syncInfo = handle.getSyncInfo(syncServerStorageId);
+          const remoteHeads = syncInfo?.lastHeads;
+          console.log(`⏰ TIMEOUT for document ${index + 1}: ${handle.url}`);
+          console.log(`  Final local heads: ${JSON.stringify(localHeads)}`);
+          console.log(`  Final remote heads: ${JSON.stringify(remoteHeads)}`);
+          reject(
+            new Error(
+              `Sync timeout after ${timeoutMs}ms for document ${handle.url}`
+            )
+          );
+        }, timeoutMs);
+
+        const checkSync = () => {
+          const newHeads = handle.heads();
+          const syncInfo = handle.getSyncInfo(syncServerStorageId);
+          const remoteHeads = syncInfo?.lastHeads;
+
+          if (verbose) {
+            console.log(`🔍 Checking sync for ${handle.url}:`);
+            console.log(`  Local heads: ${JSON.stringify(newHeads)}`);
+            console.log(`  Remote heads: ${JSON.stringify(remoteHeads)}`);
+            console.log(`  Heads equal: ${A.equals(newHeads, remoteHeads)}`);
+          }
+
+          // If the remote heads are already up to date, we can resolve immediately
+          if (A.equals(newHeads, remoteHeads)) {
+            if (verbose) {
+              console.log(`✅ Document ${index + 1} synced: ${handle.url}`);
+            }
+            clearTimeout(timeout);
+            resolve();
+            return true;
+          }
+          return false;
+        };
+
+        // Check if already synced
+        if (checkSync()) {
+          return;
+        }
+
+        // Otherwise, wait for remote-heads event
+        const onRemoteHeads = ({
+          storageId,
+          heads,
+        }: {
+          storageId: StorageId;
+          heads: any;
+        }) => {
+          if (verbose) {
+            console.log(`📡 Received remote heads event for ${handle.url}:`);
+            console.log(`  Event storage ID: ${storageId}`);
+            console.log(`  Expected storage ID: ${syncServerStorageId}`);
+            console.log(`  Event heads: ${JSON.stringify(heads)}`);
+            console.log(
+              `  Current local heads: ${JSON.stringify(handle.heads())}`
+            );
+          }
+
+          if (
+            storageId === syncServerStorageId &&
+            A.equals(handle.heads(), heads)
+          ) {
+            if (verbose) {
+              console.log(
+                `✅ Document ${index + 1} synced via event: ${handle.url}`
+              );
+            }
+            clearTimeout(timeout);
+            handle.off("remote-heads", onRemoteHeads);
+            resolve();
+          } else if (verbose) {
+            console.log(`❌ Heads/storage mismatch for ${handle.url}`);
+          }
+        };
+
+        if (verbose) {
+          console.log(`👂 Listening for remote-heads events on ${handle.url}`);
+        }
+        handle.on("remote-heads", onRemoteHeads);
+      })
+  );
+
+  try {
+    await Promise.all(promises);
+    if (verbose) {
+      console.log("✅ All documents synced to network");
+    }
+  } catch (error) {
+    console.error(`❌ Sync wait failed: ${error}`);
+    throw error;
+  }
+}
+
+/**
+ * Get the storage ID for the sync server
+ * Using the same ID as patchwork-cli for consistency
+ */
+export function getSyncServerStorageId(customStorageId?: string): StorageId {
+  return (customStorageId ||
+    "3760df37-a4c6-4f66-9ecd-732039a9385d") as StorageId;
+}