xypriss 9.0.13 → 9.1.1

package/tools/xyp/src/commands/uninstall.rs

@@ -0,0 +1,156 @@
+use std::path::Path;
+use colored::Colorize;
+use anyhow::Result;
+use serde_json::Value;
+
+pub async fn run(packages: Vec<String>, global: bool) -> Result<()> {
+    let current_dir = std::env::current_dir()?;
+    
+    let nm_root = if global {
+        let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
+        Path::new(&home).join(".xpm_global").join("node_modules")
+    } else {
+        current_dir.join("node_modules")
+    };
+
+    if packages.is_empty() {
+        println!("{} No packages specified for uninstallation.", "⚠".bold().yellow());
+        return Ok(());
+    }
+
+    println!("{} Removing packages...", "🗑".bold().red());
+
+    let mut removed_count = 0;
+
+    for pkg_name in &packages {
+        let pkg_path = nm_root.join(pkg_name);
+        
+        if pkg_path.exists() {
+            // Remove the package directory/symlink
+            // Remove the package directory/symlink
+            if pkg_path.is_symlink() || !pkg_path.is_dir() {
+                std::fs::remove_file(&pkg_path)?;
+            } else {
+                std::fs::remove_dir_all(&pkg_path)?;
+            }
+            
+            // Clean up empty scope directory
+            if let Some(parent) = pkg_path.parent() {
+                if parent != nm_root {
+                    if let Ok(mut entries) = std::fs::read_dir(parent) {
+                        if entries.next().is_none() {
+                            let _ = std::fs::remove_dir(parent);
+                        }
+                    }
+                }
+            }
+
+            println!("   {} Removed {}", "-".bold().red(), pkg_name.bold());
+            removed_count += 1;
+
+            // Attempt to clean up binaries
+            if global {
+               let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
+               let global_root = Path::new(&home).join(".xpm_global");
+               cleanup_global_binaries(&global_root, pkg_name).ok();
+            } else {
+               cleanup_binaries(&nm_root, pkg_name).ok();
+            }
+        } else {
+            println!("   {} Package {} not found in {}", "⚠".bold().yellow(), pkg_name.dimmed(), nm_root.display());
+        }
+    }
+
+    // Update package.json (only if local)
+    if !global {
+        let pkg_json_path = current_dir.join("package.json");
+        if pkg_json_path.exists() {
+            update_package_json(&pkg_json_path, &packages)?;
+        }
+    }
+
+    println!();
+    if removed_count > 0 {
+        println!("{} Uninstall complete ({} removed)", "✓".bold().green(), removed_count);
+    } else {
+        println!("{} Nothing to remove", "✓".bold().green());
+    }
+    println!("{}", "   Powered by Nehonix™ & XyPriss Engine".truecolor(100, 100, 100).italic());
+
+    Ok(())
+}
+
+fn cleanup_global_binaries(global_root: &Path, pkg_name: &str) -> Result<()> {
+    let bin_dir = global_root.join("bin");
+    if !bin_dir.exists() { return Ok(()); }
+
+    for entry in std::fs::read_dir(bin_dir)? {
+        let entry = entry?;
+        let path = entry.path();
+        
+        // Check if it's a symlink
+        if let Ok(target) = std::fs::read_link(&path) {
+            let target_str = target.to_string_lossy();
+            
+            // More precise match for binary links
+            let pattern = format!("{0}node_modules{0}{1}{0}", std::path::MAIN_SEPARATOR, pkg_name);
+            let pattern_end = format!("{0}node_modules{0}{1}", std::path::MAIN_SEPARATOR, pkg_name);
+            
+            if target_str.contains(&pattern) || target_str.ends_with(&pattern_end) {
+                std::fs::remove_file(&path)?;
+                println!("   {} Removed binary link: {}", "🔗".bold().magenta(), path.file_name().unwrap().to_string_lossy());
+            }
+        }
+    }
+    Ok(())
+}
+
+fn cleanup_binaries(nm_root: &Path, pkg_name: &str) -> Result<()> {
+    let bin_dir = nm_root.join(".bin");
+    if !bin_dir.exists() { return Ok(()); }
+
+    for entry in std::fs::read_dir(bin_dir)? {
+        let entry = entry?;
+        let path = entry.path();
+        
+        if let Ok(target) = std::fs::read_link(&path) {
+            let target_str = target.to_string_lossy();
+            
+            // Precise match for relative or absolute paths
+            let pattern = format!("{0}{1}{0}", std::path::MAIN_SEPARATOR, pkg_name);
+            let pattern_rel = format!("..{0}{1}{0}", std::path::MAIN_SEPARATOR, pkg_name);
+            
+            if target_str.contains(&pattern) || target_str.contains(&pattern_rel) || 
+               target_str.ends_with(&format!("{}{}", std::path::MAIN_SEPARATOR, pkg_name)) {
+                std::fs::remove_file(path)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn update_package_json(path: &Path, packages: &[String]) -> Result<()> {
+    let content = std::fs::read_to_string(path)?;
+    let mut json: Value = serde_json::from_str(&content)?;
+
+    let sections = vec!["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"];
+    let mut modified = false;
+
+    for section in sections {
+        if let Some(deps) = json.get_mut(section).and_then(|v| v.as_object_mut()) {
+            for pkg in packages {
+                if deps.remove(pkg).is_some() {
+                    modified = true;
+                }
+            }
+        }
+    }
+
+    if modified {
+        let new_content = serde_json::to_string_pretty(&json)?;
+        std::fs::write(path, new_content)?;
+        println!("   {} Updated package.json", "✎".bold().blue());
+    }
+
+    Ok(())
+}

package/tools/xyp/src/core/cas.rs

@@ -0,0 +1,250 @@
+use std::path::{Path, PathBuf};
+use std::fs;
+use std::io::{Read, Write};
+use anyhow::{Context, Result};
+
+
+#[derive(Clone)]
+pub struct Cas {
+    pub base_path: PathBuf,
+    dir_cache: std::sync::Arc<dashmap::DashSet<String>>,
+}
+
+impl Cas {
+    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
+        let base_path = path.as_ref().to_path_buf();
+        if !base_path.exists() {
+            Self::create_dir_all_secure(&base_path).context("Failed to create CAS base directory")?;
+        }
+        Self::create_dir_all_secure(base_path.join("files")).context("Failed to create XCAS files directory")?;
+        Self::create_dir_all_secure(base_path.join("indices")).context("Failed to create XCAS indices directory")?;
+        Self::create_dir_all_secure(base_path.join("metadata")).context("Failed to create XCAS metadata directory")?;
+        Self::create_dir_all_secure(base_path.join("temp")).context("Failed to create XCAS temp directory")?;
+        Self::create_dir_all_secure(base_path.join("virtual_store")).context("Failed to create XCAS virtual_store directory")?;
+        
+        Ok(Self { 
+            base_path,
+            dir_cache: std::sync::Arc::new(dashmap::DashSet::with_capacity(4096)),
+        })
+    }
+
+    pub fn create_dir_all_secure<P: AsRef<Path>>(path: P) -> Result<()> {
+        let path = path.as_ref();
+        fs::create_dir_all(path)?;
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            if let Ok(meta) = fs::metadata(path) {
+                let current_mode = meta.permissions().mode();
+                // Ensure the directory is at least 0o755 (owner rwx, others rx)
+                // We preserve suid/sgid/sticky bits and file type bits
+                if (current_mode & 0o777) != 0o755 {
+                    let mut perms = meta.permissions();
+                    perms.set_mode((current_mode & !0o777) | 0o755);
+                    let _ = fs::set_permissions(path, perms);
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_file_path(&self, hash: &str) -> PathBuf {
+        let (prefix, rest) = hash.split_at(2);
+        let (sub_prefix, final_hash) = rest.split_at(2);
+        self.base_path
+            .join("files")
+            .join(prefix)
+            .join(sub_prefix)
+            .join(final_hash)
+    }
+
+    pub fn store_stream<R: Read>(&self, reader: R, is_executable: bool) -> Result<String> {
+        let mut reader = std::io::BufReader::with_capacity(128 * 1024, reader);
+        // Optimized for small files (common in JS/TS packages)
+        let mut buffer = Vec::with_capacity(64 * 1024);
+        let mut chunk = [0u8; 16384];
+        let mut total_read = 0;
+        let small_file_limit = 64 * 1024;
+
+        while total_read < small_file_limit {
+            let n = reader.read(&mut chunk)?;
+            if n == 0 { break; }
+            buffer.extend_from_slice(&chunk[..n]);
+            total_read += n;
+        }
+
+        if total_read < small_file_limit {
+            // Memory path: fast hash + direct write
+            let hash = blake3::hash(&buffer);
+            let hash_hex = hash.to_hex();
+            let dest_path = self.get_file_path(hash_hex.as_str());
+            
+            if dest_path.exists() {
+                #[cfg(unix)] {
+                    use std::os::unix::fs::PermissionsExt;
+                    if let Ok(meta) = fs::metadata(&dest_path) {
+                        let current_mode = meta.permissions().mode();
+                        let target_mode = if is_executable { 0o755 } else { 0o644 };
+                        if (current_mode & 0o777) != target_mode {
+                            let mut perms = meta.permissions();
+                            perms.set_mode((current_mode & !0o777) | target_mode);
+                            let _ = fs::set_permissions(&dest_path, perms);
+                        }
+                    }
+                }
+                return Ok(hash_hex.to_string());
+            }
+            
+            self.ensure_parent_dirs(&hash_hex)?;
+            
+            // Atomic write using temp file to avoid race conditions
+            let temp_dir = self.base_path.join("temp");
+            let temp_path = temp_dir.join(uuid::Uuid::new_v4().to_string());
+            
+            fs::write(&temp_path, &buffer).context("Writing small file to temp CAS")?;
+            
+            #[cfg(unix)] {
+                use std::os::unix::fs::PermissionsExt;
+                let mode = if is_executable { 0o755 } else { 0o644 };
+                let _ = fs::set_permissions(&temp_path, fs::Permissions::from_mode(mode));
+            }
+
+            if let Err(e) = fs::rename(&temp_path, &dest_path) {
+                if e.raw_os_error() == Some(18) || e.kind() == std::io::ErrorKind::Other { // 18 is EXDEV on Linux
+                    if let Err(copy_err) = fs::copy(&temp_path, &dest_path) {
+                        let _ = fs::remove_file(&temp_path);
+                        return Err(anyhow::Error::from(copy_err)).context("Falling back to copy for cross-device rename");
+                    }
+                    let _ = fs::remove_file(&temp_path);
+                } else if dest_path.exists() {
+                    // If destination exists now, someone else finished it, that's fine
+                    let _ = fs::remove_file(&temp_path);
+                    return Ok(hash_hex.to_string());
+                } else {
+                    return Err(anyhow::Error::from(e)).context("Moving small file to CAS final destination");
+                }
+            }
+
+            return Ok(hash_hex.to_string());
+        }
+
+        // Streaming path for large files (e.g. Bun binary)
+        let temp_dir = self.base_path.join("temp");
+        let temp_path = temp_dir.join(uuid::Uuid::new_v4().to_string());
+        
+        let mut temp_file = fs::File::create(&temp_path).context("Creating temp file in CAS")?;
+        let mut hasher = blake3::Hasher::new();
+        
+        // Write the first chunk already read
+        hasher.update(&buffer);
+        temp_file.write_all(&buffer)?;
+
+        let mut streaming_buffer = [0u8; 131072]; // 128KB buffer
+        loop {
+            let n = reader.read(&mut streaming_buffer)?;
+            if n == 0 { break; }
+            hasher.update(&streaming_buffer[..n]);
+            temp_file.write_all(&streaming_buffer[..n])?;
+        }
+        
+        temp_file.sync_all()?;
+        let hash = hasher.finalize().to_hex();
+        let dest_path = self.get_file_path(hash.as_str());
+        
+        if dest_path.exists() {
+            #[cfg(unix)] {
+                use std::os::unix::fs::PermissionsExt;
+                if let Ok(meta) = fs::metadata(&dest_path) {
+                    let current_mode = meta.permissions().mode();
+                    let target_mode = if is_executable { 0o755 } else { 0o644 };
+                    if (current_mode & 0o777) != target_mode {
+                        let mut perms = meta.permissions();
+                        perms.set_mode((current_mode & !0o777) | target_mode);
+                        let _ = fs::set_permissions(&dest_path, perms);
+                    }
+                }
+            }
+            let _ = fs::remove_file(&temp_path);
+            return Ok(hash.to_string());
+        }
+        
+        self.ensure_parent_dirs(hash.as_str())?;
+        
+        if let Err(e) = fs::rename(&temp_path, &dest_path) {
+            if e.raw_os_error() == Some(18) || e.kind() == std::io::ErrorKind::Other {
+                if let Err(copy_err) = fs::copy(&temp_path, &dest_path) {
+                    let _ = fs::remove_file(&temp_path);
+                    return Err(anyhow::Error::from(copy_err)).context("Falling back to copy for large file cross-device rename");
+                }
+                let _ = fs::remove_file(&temp_path);
+            } else if !dest_path.exists() {
+                return Err(anyhow::Error::from(e)).context("Moving temp file to CAS final destination");
+            } else {
+                let _ = fs::remove_file(&temp_path);
+            }
+        }
+        
+        #[cfg(unix)] {
+            use std::os::unix::fs::PermissionsExt;
+            let mode = if is_executable { 0o755 } else { 0o644 };
+            let _ = fs::set_permissions(&dest_path, fs::Permissions::from_mode(mode));
+        }
+        
+        Ok(hash.to_string())
+    }
+
+    fn ensure_parent_dirs(&self, hash: &str) -> Result<()> {
+        let prefix = &hash[0..2];
+        let sub_prefix = &hash[2..4];
+        let key = format!("{}/{}", prefix, sub_prefix);
+        
+        if !self.dir_cache.contains(&key) {
+            let parent = self.base_path.join("files").join(prefix).join(sub_prefix);
+            Self::create_dir_all_secure(&parent).context(format!("Failed to create CAS directory: {}", parent.display()))?;
+            self.dir_cache.insert(key);
+        }
+        Ok(())
+    }
+
+
+
+    pub fn store_index(&self, name: &str, version: &str, index: &std::collections::HashMap<String, String>) -> Result<()> {
+        let path = self.base_path.join("indices").join(format!("{}@{}.json", name.replace("/", "+"), version));
+        if let Some(parent) = path.parent() {
+             Self::create_dir_all_secure(parent)?;
+        }
+        let data = serde_json::to_string_pretty(index)?;
+        fs::write(path, data)?;
+        Ok(())
+    }
+
+    pub fn get_index(&self, name: &str, version: &str) -> Result<Option<std::collections::HashMap<String, String>>> {
+        let path = self.base_path.join("indices").join(format!("{}@{}.json", name.replace("/", "+"), version));
+        if !path.exists() {
+            return Ok(None);
+        }
+        let data = fs::read_to_string(path)?;
+        let index = serde_json::from_str(&data)?;
+        Ok(Some(index))
+    }
+
+    pub fn store_metadata(&self, name: &str, version: &str, metadata: &serde_json::Value) -> Result<()> {
+        let path = self.base_path.join("metadata").join(format!("{}@{}.json", name.replace("/", "+"), version));
+        if let Some(parent) = path.parent() {
+             Self::create_dir_all_secure(parent)?;
+        }
+        let data = serde_json::to_string_pretty(metadata)?;
+        fs::write(path, data)?;
+        Ok(())
+    }
+
+    pub fn get_metadata(&self, name: &str, version: &str) -> Result<Option<serde_json::Value>> {
+        let path = self.base_path.join("metadata").join(format!("{}@{}.json", name.replace("/", "+"), version));
+        if !path.exists() {
+            return Ok(None);
+        }
+        let data = fs::read_to_string(path)?;
+        let metadata = serde_json::from_str(&data)?;
+        Ok(Some(metadata))
+    }
+}

package/tools/xyp/src/core/config.rs

@@ -0,0 +1,87 @@
+use std::sync::atomic::{AtomicUsize, AtomicU64, Ordering};
+use std::time::Duration;
+use std::sync::Arc;
+
+pub struct DynamicConfig {
+    /// Current max concurrency (changes dynamically)
+    pub concurrency: AtomicUsize,
+    /// Base timeout in seconds
+    pub timeout_secs: AtomicU64,
+    /// Average latency observed (in milliseconds)
+    pub avg_latency_ms: AtomicU64,
+    /// Performance history to adjust values
+    pub total_requests: AtomicU64,
+    pub total_errors: AtomicU64,
+}
+
+impl DynamicConfig {
+    pub fn new() -> Self {
+        Self {
+            concurrency: AtomicUsize::new(32), // Start at 32 (faster for modern connections)
+            timeout_secs: AtomicU64::new(28), // Reduced for faster failover/retries - 15
+            avg_latency_ms: AtomicU64::new(500), // assume good latency initially
+            total_requests: AtomicU64::new(0),
+            total_errors: AtomicU64::new(0),
+        }
+    }
+
+    /// Record a request execution time and adjust settings
+    pub fn record_request(&self, duration: Duration, is_error: bool) {
+        let ms = duration.as_millis() as u64;
+        self.total_requests.fetch_add(1, Ordering::Relaxed);
+        
+        if is_error {
+            self.total_errors.fetch_add(1, Ordering::Relaxed);
+            let current = self.concurrency.load(Ordering::Relaxed);
+            // On error, back off but not too much if it was a timeout
+            if current > 8 {
+                self.concurrency.store(current / 2, Ordering::Relaxed);
+            }
+        } else {
+            let prev_avg = self.avg_latency_ms.load(Ordering::Relaxed);
+            let new_avg = (prev_avg * 7 + ms) / 8; 
+            self.avg_latency_ms.store(new_avg, Ordering::Relaxed);
+
+            let current = self.concurrency.load(Ordering::Relaxed);
+            // Aggressive ramp up: if it works, try more.
+            // Even if it's "slow" (e.g. 1s), we want MORE concurrency to hide that 1s.
+            if current < 512 {
+                if ms < 200 {
+                    self.concurrency.fetch_add(16, Ordering::Relaxed);
+                } else if ms < 1000 {
+                    self.concurrency.fetch_add(8, Ordering::Relaxed);
+                } else {
+                    self.concurrency.fetch_add(2, Ordering::Relaxed);
+                }
+            }
+        }
+    }
+
+    pub fn get_concurrency(&self) -> usize {
+        let latency = self.avg_latency_ms.load(Ordering::Relaxed);
+        let current = self.concurrency.load(Ordering::Relaxed);
+        
+        // Even on slow networks, metadata (small JSON) benefits from high concurrency
+        // to hide Round-Trip Time (RTT).
+        if latency > 5000 {
+            current.min(16) // Extreme latency: still allow some parallelism
+        } else if latency > 2000 {
+            current.min(32)
+        } else if latency > 1000 {
+            current.min(64)
+        } else if latency > 500 {
+            current.min(96)
+        } else {
+            current.min(128) // Cap metadata concurrency at 128 even on fast networks
+        }.max(8)
+    }
+
+    pub fn get_timeout(&self, attempt: u32) -> Duration {
+        let base = self.timeout_secs.load(Ordering::Relaxed);
+        // Be significantly more patient on each retry
+        let patience_factor = 1 + (attempt as u64 * 2);
+        Duration::from_secs(base * patience_factor)
+    }
+}
+
+pub type GlobalConfig = Arc<DynamicConfig>;

package/tools/xyp/src/core/extractor.rs

@@ -0,0 +1,85 @@
+use std::io::Read;
+use anyhow::Result;
+use flate2::read::GzDecoder;
+use tar::Archive;
+use crate::core::cas::Cas;
+
+pub struct StreamingExtractor<'a> {
+    cas: &'a Cas,
+}
+
+impl<'a> StreamingExtractor<'a> {
+    pub fn new(cas: &'a Cas) -> Self {
+        Self { cas }
+    }
+
+    /// Extracts a .tgz stream into the CAS and returns a map of filenames to their hashes
+    /// Uses flate2 with pure Rust backend for maximum portability
+    pub fn extract<R: Read>(&self, reader: R) -> Result<std::collections::HashMap<String, String>> {
+        use std::io::BufReader;
+        
+        // Large buffer for optimal decompression speed
+        let buf_reader = BufReader::with_capacity(4 * 1024 * 1024, reader); // 4MB buffer
+        
+        // GzDecoder with pure Rust backend
+        let gz = GzDecoder::new(buf_reader);
+        let mut archive = Archive::new(gz);
+        let mut file_map = std::collections::HashMap::with_capacity(512);
+        
+        for entry in archive.entries()? {
+            let mut entry = entry?;
+            
+            if !entry.header().entry_type().is_file() {
+                continue;
+            }
+            
+            let path = entry.path()?.to_string_lossy().to_string();
+            let is_executable = entry.header().mode().ok()
+                .map(|mode| (mode & 0o111) != 0)
+                .unwrap_or(false);
+            
+            // Store in CAS
+            let hash = self.cas.store_stream(&mut entry, is_executable)?;
+            file_map.insert(path, hash);
+        }
+
+        Ok(file_map)
+    }
+
+    /// Alternative: Extract with immediate callback for eager processing
+    pub fn extract_with_callback<R, F>(&self, reader: R, mut callback: F) -> Result<std::collections::HashMap<String, String>>
+    where
+        R: Read,
+        F: FnMut(&str, &str), // (path, hash) callback for eager processing
+    {
+        use std::io::BufReader;
+        
+        let buf_reader = BufReader::with_capacity(4 * 1024 * 1024, reader);
+        let gz = GzDecoder::new(buf_reader);
+        let mut archive = Archive::new(gz);
+        let mut file_map = std::collections::HashMap::with_capacity(512);
+
+        for entry in archive.entries()? {
+            let mut entry = entry?;
+            
+            if !entry.header().entry_type().is_file() {
+                continue;
+            }
+
+            let path = entry.path()?.to_string_lossy().to_string();
+            let is_executable = entry.header().mode()
+                .ok()
+                .map(|mode| (mode & 0o111) != 0)
+                .unwrap_or(false);
+
+            let hash = self.cas.store_stream(&mut entry, is_executable)?;
+            
+            // Immediate callback for eager consumers
+            callback(&path, &hash);
+            
+            file_map.insert(path, hash);
+        }
+
+        Ok(file_map)
+    }
+}