resulgit 1.0.21 → 1.0.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resulgit",
-  "version": "1.0.21",
+  "version": "1.0.22",
   "description": "A powerful CLI tool for version control system operations - clone, commit, push, pull, merge, branch management, and more",
   "main": "resulgit.js",
   "bin": {

package/resulgit.js

@@ -516,12 +516,28 @@ function shouldIgnore(p, patterns) {
   return false
 }
 
+// Stream-hash a file of any size without loading it into RAM.
+// Uses SHA-1 (same as VS Code extension and server).
+function streamHashFile(absPath) {
+  return new Promise((resolve, reject) => {
+    const h = crypto.createHash('sha1')
+    const s = require('fs').createReadStream(absPath, { highWaterMark: 4 * 1024 * 1024 })
+    s.on('data', chunk => h.update(chunk))
+    s.on('end', () => resolve(h.digest('hex')))
+    s.on('error', reject)
+  })
+}
+
+// collectLocal — stores ONLY the SHA-1 hash per file.
+// Content is never kept in memory so large repos (5GB+) don't cause OOM.
 async function collectLocal(dir) {
   const out = {}
   const base = path.resolve(dir)
   const patterns = loadIgnorePatterns(base)
+  const STREAM_THRESHOLD = 50 * 1024 * 1024 // 50 MB
   async function walk(cur, rel) {
-    const entries = await fs.promises.readdir(cur, { withFileTypes: true })
+    let entries
+    try { entries = await fs.promises.readdir(cur, { withFileTypes: true }) } catch { return }
     for (const e of entries) {
       const rp = rel ? rel + '/' + e.name : e.name
       if (shouldIgnore(rp, patterns)) continue
@@ -529,8 +545,17 @@ async function collectLocal(dir) {
       if (e.isDirectory()) {
         await walk(abs, rp)
       } else if (e.isFile()) {
-        const buf = await fs.promises.readFile(abs)
-        out[rp] = { content: buf.toString('utf8'), id: hashContent(buf) }
+        try {
+          const stat = await fs.promises.stat(abs)
+          let id
+          if (stat.size > STREAM_THRESHOLD) {
+            id = await streamHashFile(abs) // stream-hash large files
+          } else {
+            const buf = await fs.promises.readFile(abs)
+            id = hashContent(buf)
+          }
+          out[rp] = { id } // hash only — no content stored
+        } catch { /* skip unreadable */ }
       }
     }
   }
@@ -816,18 +841,13 @@ async function cmdCommit(opts) {
   const message = opts.message || ''
   if (!message) throw new Error('Missing --message')
 
-  // Check for unresolved conflicts
   const unresolvedConflicts = await checkForUnresolvedConflicts(dir)
   if (unresolvedConflicts.length > 0) {
     if (opts.json === 'true') {
       print({ error: 'Cannot commit with unresolved conflicts', conflicts: unresolvedConflicts }, true)
     } else {
       process.stderr.write(color('Error: Cannot commit with unresolved conflicts\n', 'red'))
-      process.stderr.write(color('Conflicts in files:\n', 'yellow'))
-      for (const p of unresolvedConflicts) {
-        process.stderr.write(color(`  ${p}\n`, 'red'))
-      }
-      process.stderr.write(color('\nResolve conflicts manually before committing.\n', 'yellow'))
+      for (const p of unresolvedConflicts) process.stderr.write(color(`  ${p}\n`, 'red'))
     }
     return
   }
@@ -835,18 +855,15 @@ async function cmdCommit(opts) {
   const metaDir = path.join(dir, '.vcs-next')
   const localPath = path.join(metaDir, 'local.json')
   let localMeta = { baseCommitId: '', baseFiles: {}, pendingCommit: null }
-  try {
-    const s = await fs.promises.readFile(localPath, 'utf8')
-    localMeta = JSON.parse(s)
-  } catch { }
+  try { const s = await fs.promises.readFile(localPath, 'utf8'); localMeta = JSON.parse(s) } catch { }
+
+  // Collect only hashes — never full content — so local.json stays tiny
   const local = await collectLocal(dir)
-  const files = {}
-  for (const [p, v] of Object.entries(local)) files[p] = v.content
-  localMeta.pendingCommit = { message, files, createdAt: Date.now() }
-  // Clear conflicts if they were resolved
-  if (localMeta.conflicts) {
-    delete localMeta.conflicts
-  }
+  const fileHashes = {}
+  for (const [p, v] of Object.entries(local)) fileHashes[p] = v.id
+
+  localMeta.pendingCommit = { message, files: fileHashes, createdAt: Date.now() }
+  if (localMeta.conflicts) delete localMeta.conflicts
   await fs.promises.mkdir(metaDir, { recursive: true })
   await fs.promises.writeFile(localPath, JSON.stringify(localMeta, null, 2))
   print({ pendingCommit: message }, opts.json === 'true')
@@ -1297,29 +1314,96 @@ async function checkForUnresolvedConflicts(dir) {
   return conflicts
 }
 
+// Upload a single large file (>50MB) via streaming multipart — no RAM usage.
+async function uploadLargeFile(server, repoId, relPath, absPath, token) {
+  const http = require('http')
+  const https = require('https')
+  const stat = await fs.promises.stat(absPath)
+  const fileSize = stat.size
+  const boundary = `----resulgit-${Date.now()}-${Math.random().toString(16).slice(2)}`
+  const eol = '\r\n'
+  const safeFilename = relPath.replace(/"/g, '\\"')
+  const headerBuf = Buffer.from(
+    `--${boundary}${eol}` +
+    `Content-Disposition: form-data; name="files"; filename="${safeFilename}"${eol}` +
+    `Content-Type: application/octet-stream${eol}${eol}`, 'utf8'
+  )
+  const footerBuf = Buffer.from(`${eol}--${boundary}--${eol}`, 'utf8')
+  const contentLength = headerBuf.length + fileSize + footerBuf.length
+  const timeoutMs = Math.max(300000, Math.min(contentLength * 2, 12 * 3600000))
+  return new Promise((resolve, reject) => {
+    let parsed; try { parsed = new URL(`${server}/api/repositories/${repoId}/blobs`) } catch (e) { reject(e); return }
+    const isHttps = parsed.protocol === 'https:'
+    const lib = isHttps ? https : http
+    const req = lib.request({
+      hostname: parsed.hostname,
+      port: parsed.port || (isHttps ? 443 : 80),
+      path: parsed.pathname,
+      method: 'POST',
+      timeout: timeoutMs,
+      headers: {
+        'Content-Type': `multipart/form-data; boundary=${boundary}`,
+        'Content-Length': String(contentLength),
+        ...(token ? { 'Authorization': `Bearer ${token}` } : {})
+      }
+    }, (res) => {
+      let data = ''
+      res.on('data', c => { data += c })
+      res.on('end', () => {
+        if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
+          reject(new Error(`Large-file upload HTTP ${res.statusCode} for ${relPath}: ${data.slice(0, 200)}`)); return
+        }
+        try {
+          const parsed = JSON.parse(data)
+          const blobId = (parsed.blobs || {})[relPath]
+          if (!blobId) { reject(new Error(`No blob ID returned for ${relPath}`)); return }
+          resolve(blobId)
+        } catch { reject(new Error('Invalid JSON from blob server')) }
+      })
+    })
+    req.on('error', e => reject(e))
+    req.on('timeout', () => { req.destroy(); reject(new Error(`Upload timed out for ${relPath}`)) })
+    req.write(headerBuf)
+    const stream = require('fs').createReadStream(absPath, { highWaterMark: 1024 * 1024 })
+    stream.on('data', chunk => req.write(chunk))
+    stream.on('end', () => { req.write(footerBuf); req.end() })
+    stream.on('error', e => { req.destroy(); reject(e) })
+  })
+}
+
+// Upload a batch of small files (<= 50MB each) as JSON.
+async function uploadSmallBatch(server, repoId, entries, token) {
+  // entries: Array<[relPath, absPath]>
+  const payload = {}
+  for (const [relPath, absPath] of entries) {
+    const buf = await fs.promises.readFile(absPath)
+    const text = buf.toString('utf8')
+    const isBinary = text.includes('\uFFFD') || buf.includes(0)
+    payload[relPath] = isBinary ? `__base64__${buf.toString('base64')}` : text
+  }
+  const url = new URL(`/api/repositories/${repoId}/blobs`, server).toString()
+  const headers = { 'Content-Type': 'application/json', ...(token ? { 'Authorization': `Bearer ${token}` } : {}) }
+  const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(payload) })
+  if (!res.ok) { const t = await res.text(); throw new Error(`Blob batch upload failed: ${res.status} ${t.slice(0, 200)}`) }
+  const data = await res.json()
+  return data.blobs || {}
+}
+
 async function cmdPush(opts) {
   const dir = path.resolve(opts.dir || '.')
   const remoteMeta = readRemoteMeta(dir)
   const metaDir = path.join(dir, '.vcs-next')
   const localPath = path.join(metaDir, 'local.json')
   let localMeta = { baseCommitId: '', baseFiles: {}, pendingCommit: null }
-  try {
-    const s = await fs.promises.readFile(localPath, 'utf8')
-    localMeta = JSON.parse(s)
-  } catch { }
+  try { const s = await fs.promises.readFile(localPath, 'utf8'); localMeta = JSON.parse(s) } catch { }
 
-  // Check for unresolved conflicts in files
   const unresolvedConflicts = await checkForUnresolvedConflicts(dir)
   if (unresolvedConflicts.length > 0) {
     if (opts.json === 'true') {
       print({ error: 'Unresolved conflicts', conflicts: unresolvedConflicts }, true)
     } else {
       process.stderr.write(color('Error: Cannot push with unresolved conflicts\n', 'red'))
-      process.stderr.write(color('Conflicts in files:\n', 'yellow'))
-      for (const p of unresolvedConflicts) {
-        process.stderr.write(color(`  ${p}\n`, 'red'))
-      }
-      process.stderr.write(color('\nResolve conflicts manually, then try pushing again.\n', 'yellow'))
+      for (const p of unresolvedConflicts) process.stderr.write(color(`  ${p}\n`, 'red'))
     }
     return
   }
@@ -1327,48 +1411,82 @@ async function cmdPush(opts) {
   const cfg = loadConfig()
   const server = getServer(opts, cfg) || remoteMeta.server
   const token = getToken(opts, cfg) || remoteMeta.token
-  const remote = await fetchRemoteSnapshot(server, remoteMeta.repoId, remoteMeta.branch, token)
-  const base = localMeta.baseFiles || {}
+
+  // Collect local files — hash-only (no content in RAM)
   const local = await collectLocal(dir)
-  const conflicts = []
-  const merged = {}
-  const paths = new Set([...Object.keys(base), ...Object.keys(remote.files), ...Object.keys(local).map(k => k)])
-  for (const p of paths) {
-    const b = p in base ? base[p] : null
-    const r = p in remote.files ? remote.files[p] : null
-    const l = p in local ? local[p].content : null
-    const changedLocal = String(l) !== String(b)
-    const changedRemote = String(r) !== String(b)
-    if (changedLocal && changedRemote && String(l) !== String(r)) {
-      const line = firstDiffLine(l || '', r || '')
-      conflicts.push({ path: p, line })
-    } else if (changedLocal && !changedRemote) {
-      if (l !== null) merged[p] = l
-    } else if (!changedLocal && changedRemote) {
-      if (r !== null) merged[p] = r
-    } else {
-      if (b !== null) merged[p] = b
-    }
-  }
-  if (conflicts.length > 0) {
-    if (opts.json === 'true') {
-      print({ conflicts }, true)
-    } else {
-      process.stderr.write(color('Error: Cannot push with conflicts\n', 'red'))
-      process.stderr.write(color('Conflicts detected:\n', 'yellow'))
-      for (const c of conflicts) {
-        process.stderr.write(color(`  ${c.path}:${c.line}\n`, 'red'))
-      }
-    }
-    return
-  }
-  const body = { message: localMeta.pendingCommit?.message || (opts.message || 'Push'), files: merged, branchName: remoteMeta.branch }
-  const url = new URL(`/api/repositories/${remoteMeta.repoId}/commits`, server).toString()
-  const data = await request('POST', url, body, token)
-  localMeta.baseCommitId = data.id || remote.commitId || ''
-  localMeta.baseFiles = merged
+  const base = localMeta.baseFiles || {}
+
+  // Determine which files changed vs base (use hash comparison)
+  const upserts = [] // [relPath] files to upload
+  const deletions = [] // [relPath] files deleted locally
+  const LARGE = 50 * 1024 * 1024
+
+  const allPaths = new Set([...Object.keys(base), ...Object.keys(local)])
+  for (const p of allPaths) {
+    const inLocal = p in local
+    const inBase = p in base
+    if (!inLocal) {
+      deletions.push(p)
+    } else if (!inBase || local[p].id !== base[p]) {
+      upserts.push(p)
+    }
+  }
+
+  // Upload blobs (stream large, batch small)
+  const finalFiles = {} // relPath -> blobId
+  const BATCH = 200
+  const largeFiles = []
+  const smallFiles = []
+  for (const relPath of upserts) {
+    const absPath = path.join(dir, relPath)
+    let size = 0
+    try { size = (await fs.promises.stat(absPath)).size } catch { continue }
+    if (size > LARGE) largeFiles.push(relPath)
+    else smallFiles.push(relPath)
+  }
+
+  // Stream large files one at a time
+  for (let i = 0; i < largeFiles.length; i++) {
+    const relPath = largeFiles[i]
+    const absPath = path.join(dir, relPath)
+    const sizeMB = Math.round((await fs.promises.stat(absPath)).size / 1024 / 1024)
+    process.stderr.write(color(`  Streaming [${i + 1}/${largeFiles.length}] ${relPath} (${sizeMB} MB)...\n`, 'cyan'))
+    const blobId = await uploadLargeFile(server, remoteMeta.repoId, relPath, absPath, token)
+    finalFiles[relPath] = blobId
+  }
+
+  // Batch upload small files
+  const totalBatches = Math.ceil(smallFiles.length / BATCH)
+  for (let i = 0; i < smallFiles.length; i += BATCH) {
+    const slice = smallFiles.slice(i, i + BATCH)
+    const batchNum = Math.floor(i / BATCH) + 1
+    process.stderr.write(color(`  Uploading files ${Math.min(i + BATCH, smallFiles.length)}/${smallFiles.length} (batch ${batchNum}/${totalBatches})...\n`, 'cyan'))
+    const entries = slice.map(p => [p, path.join(dir, p)])
+    const blobIds = await uploadSmallBatch(server, remoteMeta.repoId, entries, token)
+    for (const [p, id] of Object.entries(blobIds)) finalFiles[p] = id
+  }
+
+  // Add deletions
+  for (const p of deletions) finalFiles[p] = null
+
+  // Push commit
+  const message = localMeta.pendingCommit?.message || opts.message || 'Push'
+  const commitUrl = new URL(`/api/repositories/${remoteMeta.repoId}/commits`, server).toString()
+  const commitHeaders = { 'Content-Type': 'application/json', ...(token ? { 'Authorization': `Bearer ${token}` } : {}) }
+  const body = { message, files: finalFiles, branchName: remoteMeta.branch, parentCommitId: localMeta.baseCommitId || undefined }
+  const res = await fetch(commitUrl, { method: 'POST', headers: commitHeaders, body: JSON.stringify(body) })
+  if (!res.ok) { const t = await res.text(); throw new Error(`Push failed: ${res.status} ${t.slice(0, 300)}`) }
+  const data = await res.json()
+
+  // Update local metadata — store hashes, not content
+  const newBase = { ...base }
+  for (const p of upserts) { if (local[p]) newBase[p] = local[p].id }
+  for (const p of deletions) { delete newBase[p] }
+  localMeta.baseCommitId = data.id || data.commitId || ''
+  localMeta.baseFiles = newBase
   localMeta.pendingCommit = null
   await fs.promises.writeFile(localPath, JSON.stringify(localMeta, null, 2))
+  process.stderr.write(color(`Push complete: ${upserts.length} file(s) uploaded, ${deletions.length} deleted\n`, 'green'))
   print({ pushed: localMeta.baseCommitId }, opts.json === 'true')
 }