mirror-dedupe 0.1.0__py3-none-any.whl

mirror_dedupe/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+"""
+__init__.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+__version__ = '0.1.0'

mirror_dedupe/cli.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""
+cli.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+import os
+import sys
+import subprocess
+import atexit
+import signal
+import argparse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections import defaultdict
+
+from .config import load_config
+from .utils import (acquire_lock, release_lock, signal_handler, 
+                    get_disk_usage, format_bytes)
+from .dedupe import expand_distributions
+from .orchestrate import (run_orchestrator_mode, sync_mirrors, collect_files,
+                          analyse_deduplication, check_existing_files, 
+                          process_files, cleanup_mirrors, print_final_summary)
+
+
+def main():
+    """Main entry point for mirror-dedupe"""
+    parser = argparse.ArgumentParser(
+        description='Mirror repository with global deduplication',
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    
+    parser.add_argument('config_dir', nargs='?', default='/etc/mirror-dedupe',
+                       help='Path to configuration directory (default: /etc/mirror-dedupe)')
+    parser.add_argument('--dry-run', action='store_true',
+                       help='Show what would be done without actually doing it')
+    parser.add_argument('--mirror', type=str,
+                       help='Process only the specified mirror (by name)')
+    parser.add_argument('--dedupe-only', action='store_true',
+                       help='Only run deduplication phase (skip mirror sync)')
+    
+    args = parser.parse_args()
+    
+    # Determine mode and acquire appropriate lock
+    if args.dedupe_only:
+        if not acquire_lock('dedupe'):
+            sys.exit(1)
+        atexit.register(release_lock)
+        signal.signal(signal.SIGTERM, signal_handler)
+        signal.signal(signal.SIGINT, signal_handler)
+    elif args.mirror:
+        if not acquire_lock(args.mirror):
+            sys.exit(1)
+        atexit.register(release_lock)
+        signal.signal(signal.SIGTERM, signal_handler)
+        signal.signal(signal.SIGINT, signal_handler)
+    
+    # Load configuration
+    config_dir = args.config_dir
+    config = load_config(config_dir)
+    mirrors = config.get('mirrors', [])
+    
+    if not mirrors:
+        print("No mirrors defined in configuration")
+        sys.exit(1)
+    
+    print(f"\n{'='*60}")
+    print(f"Loaded {len(mirrors)} mirror(s) from configuration")
+    print(f"{'='*60}")
+    
+    # Orchestrator mode: spawn subprocess for each mirror
+    if not args.mirror and not args.dedupe_only:
+        run_orchestrator_mode(mirrors, config_dir, args.dry_run)
+        # This function exits, so we never reach here
+    
+    # Filter mirrors if --mirror specified
+    if args.mirror:
+        filtered_mirrors = [m for m in mirrors if m['name'] == args.mirror]
+        if not filtered_mirrors:
+            print(f"ERROR: Mirror '{args.mirror}' not found in configuration")
+            sys.exit(1)
+        mirrors = filtered_mirrors
+        print(f"\n{'='*60}")
+        print(f"SINGLE MIRROR MODE: Processing '{args.mirror}'")
+        print(f"{'='*60}")
+    
+    # Skip mirror sync if --dedupe-only
+    if args.dedupe_only:
+        print(f"\n{'='*60}")
+        print("DEDUPE-ONLY MODE: Skipping mirror sync")
+        print(f"{'='*60}")
+    else:
+        sync_mirrors(mirrors, args.dry_run)
+    
+    # Collect all files needed across all mirrors
+    global_files = collect_files(mirrors)
+    
+    # Analyse deduplication potential
+    hash_to_files, unique_files = analyse_deduplication(global_files)
+    
+    # Check existing files
+    check_existing_files(hash_to_files)
+    
+    # Get initial disk usage
+    print(f"\n{'='*60}")
+    print("Initial disk usage")
+    print(f"{'='*60}")
+    first_dest = mirrors[0]['dest']
+    total, initial_used, free = get_disk_usage(first_dest)
+    print(f"Overall mirror filesystem: Used: {format_bytes(initial_used)}, Free: {format_bytes(free)}")
+    
+    # In single-mirror mode, note that cross-mirror deduplication will be handled separately
+    if args.mirror:
+        print(f"\n{'='*60}")
+        print(f"Single mirror mode: Deduplication will be handled separately")
+        print(f"{'='*60}")
+    
+    # Process files (download and hardlink)
+    downloaded, hardlinked, skipped = process_files(hash_to_files, unique_files, config, args.dry_run)
+    
+    # In single-mirror mode, exit after downloading (skip cleanup and cross-mirror dedup)
+    if args.mirror:
+        print(f"\nMirror '{args.mirror}' sync completed successfully!")
+        sys.exit(0)
+    
+    # Cleanup mirrors
+    cleanup_mirrors(mirrors, global_files, args.dry_run)
+    
+    # Print final summary
+    print_final_summary(mirrors, downloaded, hardlinked, skipped, initial_used)
+
+
+if __name__ == '__main__':
+    main()

mirror_dedupe/config.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""
+config.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+import os
+import sys
+import yaml
+from pathlib import Path
+
+
+def load_config(config_dir: str) -> dict:
+    """Load mirror configuration from config directory
+    
+    Loads main config and all repo definitions from repos.d/
+    """
+    try:
+        # Load main config
+        main_config_path = Path(config_dir) / 'mirror-dedupe.conf'
+        with open(main_config_path, 'r') as f:
+            config = yaml.safe_load(f) or {}
+        
+        # Load repo definitions from repos.d/
+        repos_dir = Path(config_dir) / 'repos.d'
+        mirrors = []
+        
+        if repos_dir.exists() and repos_dir.is_dir():
+            for repo_file in sorted(repos_dir.glob('*.conf')):
+                try:
+                    with open(repo_file, 'r') as f:
+                        mirror = yaml.safe_load(f)
+                        if mirror:
+                            # Prepend repo_root to dest if it's relative
+                            repo_root = config.get('repo_root', '/srv/mirror/repos')
+                            dest = mirror.get('dest', '')
+                            if not os.path.isabs(dest):
+                                mirror['dest'] = os.path.join(repo_root, dest)
+                            mirrors.append(mirror)
+                except Exception as e:
+                    print(f"Warning: Failed to load {repo_file}: {e}")
+        
+        # Add mirrors to config
+        config['mirrors'] = mirrors
+        
+        return config
+    except Exception as e:
+        print(f"Error loading config: {e}")
+        sys.exit(1)

mirror_dedupe/dedupe.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+dedupe.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+import os
+from typing import Set, Tuple
+
+
+def hardlink_file(source: str, dest: str, expected_hash: str = None) -> bool:
+    """Create hardlink from source to dest"""
+    dest_dir = os.path.dirname(dest)
+    os.makedirs(dest_dir, exist_ok=True)
+    
+    try:
+        # Check if source and dest are already the same file (hardlinked)
+        if os.path.exists(dest) and os.path.samefile(source, dest):
+            return True  # Already hardlinked, nothing to do
+        
+        # Remove dest if it exists (whether it's a file, different hardlink, or corrupted)
+        if os.path.exists(dest):
+            os.remove(dest)
+        
+        # Create hardlink
+        os.link(source, dest)
+        return True
+    except Exception as e:
+        print(f"  Error hardlinking {source} -> {dest}: {e}")
+        return False
+
+
+def expand_distributions(distributions: list) -> list:
+    """Expand distribution names to include variants"""
+    expanded = []
+    for dist in distributions:
+        expanded.append(dist)
+        if '-' not in dist:
+            expanded.extend([
+                f"{dist}-updates",
+                f"{dist}-security",
+                f"{dist}-backports",
+                f"{dist}-proposed"
+            ])
+    return expanded
+
+
+def cleanup_pool(dest_base: str, expected_files: Set[str], dry_run: bool = False) -> Tuple[int, int]:
+    """Remove files from pool/ that aren't in the expected list"""
+    print(f"\n{'='*60}")
+    print("Cleaning up pool directory")
+    print(f"{'='*60}")
+    
+    pool_path = os.path.join(dest_base, 'pool')
+    if not os.path.exists(pool_path):
+        print("  No pool directory found")
+        return (0, 0)
+    
+    removed_files = 0
+    removed_dirs = 0
+    
+    # Walk through pool/ and find files to remove
+    for root, dirs, files in os.walk(pool_path, topdown=False):
+        for filename in files:
+            full_path = os.path.join(root, filename)
+            # Get relative path from dest_base
+            rel_path = os.path.relpath(full_path, dest_base)
+            
+            if rel_path not in expected_files:
+                if dry_run:
+                    print(f"  Would remove: {rel_path}")
+                    removed_files += 1
+                else:
+                    try:
+                        os.remove(full_path)
+                        removed_files += 1
+                        
+                        if removed_files % 100 == 0:
+                            print(f"  Removed {removed_files} files...")
+                    except Exception as e:
+                        print(f"  Error removing {rel_path}: {e}")
+        
+        # Remove empty directories
+        for dirname in dirs:
+            dir_path = os.path.join(root, dirname)
+            try:
+                if not os.listdir(dir_path):  # Directory is empty
+                    if dry_run:
+                        removed_dirs += 1
+                    else:
+                        os.rmdir(dir_path)
+                        removed_dirs += 1
+            except:
+                pass  # Directory not empty or other issue
+    
+    if dry_run:
+        print(f"\nWould remove: {removed_files} files, {removed_dirs} directories")
+    else:
+        print(f"\nRemoved: {removed_files} files, {removed_dirs} directories")
+    
+    return (removed_files, removed_dirs)

mirror_dedupe/download.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""
+download.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+import os
+import subprocess
+import hashlib
+from .utils import download_lock, active_downloads
+
+# Check if sha256sum is available at startup
+USE_SHA256SUM = False
+try:
+    result = subprocess.run(['sha256sum', '--version'], capture_output=True)
+    USE_SHA256SUM = result.returncode == 0
+except:
+    pass
+
+
+def download_with_curl(url: str, dest_path: str, timeout: int = 300, progress_info: str = "") -> bool:
+    """Download file with curl, supports resuming partial downloads"""
+    from . import utils
+    
+    dest_dir = os.path.dirname(dest_path)
+    os.makedirs(dest_dir, exist_ok=True)
+    
+    # Show what we're downloading
+    filename = os.path.basename(dest_path)
+    with utils.download_lock:
+        utils.active_downloads += 1
+        current_active = utils.active_downloads
+    print(f"  ⬇ Downloading: {filename} ({current_active} active){progress_info}", flush=True)
+    
+    try:
+        # -C - enables automatic resume of partial downloads
+        cmd = ['curl', '-f', '-L', '-C', '-', '--max-time', str(timeout), '-o', dest_path, url]
+        result = subprocess.run(cmd, capture_output=True)
+        
+        with utils.download_lock:
+            utils.active_downloads -= 1
+            remaining = utils.active_downloads
+        
+        if result.returncode == 0:
+            print(f"  [OK] Completed: {filename} ({remaining} remaining)", flush=True)
+        else:
+            print(f"  [FAIL] Failed: {filename} ({remaining} remaining)", flush=True)
+        
+        return result.returncode == 0
+    except Exception as e:
+        with utils.download_lock:
+            utils.active_downloads -= 1
+            remaining = utils.active_downloads
+        print(f"  [ERROR] Error: {filename} ({remaining} remaining) - {e}", flush=True)
+        return False
+
+
+def verify_sha256(file_path: str, expected_hash: str, buffer_size: int = 1048576) -> bool:
+    """Verify file SHA256 hash using sha256sum or Python hashlib"""
+    if USE_SHA256SUM:
+        # Use fast sha256sum command
+        try:
+            result = subprocess.run(['sha256sum', file_path], 
+                                  capture_output=True, text=True)
+            if result.returncode == 0:
+                actual_hash = result.stdout.split()[0]
+                return actual_hash == expected_hash
+            return False
+        except:
+            return False
+    else:
+        # Use Python hashlib
+        try:
+            sha256 = hashlib.sha256()
+            with open(file_path, 'rb') as f:
+                for chunk in iter(lambda: f.read(buffer_size), b''):
+                    sha256.update(chunk)
+            return sha256.hexdigest() == expected_hash
+        except:
+            return False

mirror_dedupe/indices.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""
+indices.py
+
+  Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+"""
+
+import os
+import gzip
+from typing import Dict, Set
+
+
+def read_gzipped_file(filepath: str) -> str:
+    """Read and decompress a gzipped file from local filesystem"""
+    try:
+        with gzip.open(filepath, 'rt', encoding='utf-8') as f:
+            return f.read()
+    except Exception as e:
+        print(f"  Error reading {filepath}: {e}")
+        return None
+
+
+def parse_packages_file(content: str) -> Dict[str, Dict[str, str]]:
+    """
+    Parse Packages file and extract package info.
+    Returns: {filename: {'sha256': hash, 'size': size, 'package': name}}
+    """
+    packages = {}
+    current_package = {}
+    
+    for line in content.split('\n'):
+        line = line.rstrip()
+        
+        if not line:
+            if 'filename' in current_package and 'sha256' in current_package:
+                filename = current_package['filename']
+                packages[filename] = {
+                    'sha256': current_package['sha256'],
+                    'size': current_package.get('size', '0'),
+                    'package': current_package.get('package', 'unknown')
+                }
+            current_package = {}
+            continue
+        
+        if ':' in line and not line.startswith(' '):
+            key, value = line.split(':', 1)
+            key = key.strip().lower()
+            value = value.strip()
+            
+            if key == 'package':
+                current_package['package'] = value
+            elif key == 'filename':
+                current_package['filename'] = value
+            elif key == 'sha256':
+                current_package['sha256'] = value
+            elif key == 'size':
+                current_package['size'] = value
+    
+    return packages
+
+
+def parse_sources_file(content: str) -> Dict[str, Dict[str, str]]:
+    """
+    Parse Sources file and extract source package info.
+    Returns: {filename: {'sha256': hash, 'size': size, 'package': name}}
+    """
+    sources = {}
+    current_package = {}
+    current_files = []
+    
+    for line in content.split('\n'):
+        line = line.rstrip()
+        
+        if not line:
+            if 'directory' in current_package and current_files:
+                directory = current_package['directory']
+                for filename, size, sha256 in current_files:
+                    full_path = f"{directory}/{filename}"
+                    sources[full_path] = {
+                        'sha256': sha256,
+                        'size': size,
+                        'package': current_package.get('package', 'unknown')
+                    }
+            current_package = {}
+            current_files = []
+            continue
+        
+        if ':' in line and not line.startswith(' '):
+            key, value = line.split(':', 1)
+            key = key.strip().lower()
+            value = value.strip()
+            
+            if key == 'package':
+                current_package['package'] = value
+                current_package['in_checksums'] = False
+            elif key == 'directory':
+                current_package['directory'] = value
+                current_package['in_checksums'] = False
+            elif key == 'checksums-sha256':
+                current_package['in_checksums'] = True
+            else:
+                current_package['in_checksums'] = False
+        elif line.startswith(' ') and current_package.get('in_checksums'):
+            parts = line.strip().split()
+            if len(parts) >= 3:
+                sha256 = parts[0]
+                size = parts[1]
+                filename = ' '.join(parts[2:])
+                if filename and not filename.endswith(')') and '(' not in filename:
+                    current_files.append((filename, size, sha256))
+    
+    if 'directory' in current_package and current_files:
+        directory = current_package['directory']
+        for filename, size, sha256 in current_files:
+            full_path = f"{directory}/{filename}"
+            sources[full_path] = {
+                'sha256': sha256,
+                'size': size,
+                'package': current_package.get('package', 'unknown')
+            }
+    
+    return sources
+
+
+def parse_release_file(dest_base: str, distribution: str) -> Set[str]:
+    """
+    Parse Release file and return set of available index files.
+    Returns: set of relative paths like 'main/binary-amd64/Packages.gz'
+    """
+    release_path = f"{dest_base}/dists/{distribution}/Release"
+    
+    if not os.path.exists(release_path):
+        return set()
+    
+    available_files = set()
+    in_sha256_section = False
+    
+    try:
+        with open(release_path, 'r') as f:
+            for line in f:
+                line = line.rstrip()
+                
+                # Look for SHA256 section
+                if line.startswith('SHA256:'):
+                    in_sha256_section = True
+                    continue
+                elif line and not line.startswith(' '):
+                    # New section started
+                    in_sha256_section = False
+                
+                # Parse file entries in SHA256 section
+                if in_sha256_section and line.startswith(' '):
+                    parts = line.split()
+                    if len(parts) >= 3:
+                        # Format: " <hash> <size> <path>"
+                        file_path = parts[2]
+                        available_files.add(file_path)
+    except Exception as e:
+        print(f"  Warning: Could not parse Release file: {e}")
+        return set()
+    
+    return available_files
+
+
+def get_packages_index(dest_base: str, distribution: str, component: str, arch: str) -> Dict[str, Dict[str, str]]:
+    """Read and parse local Packages.gz for a specific component/arch"""
+    packages_path = f"{dest_base}/dists/{distribution}/{component}/binary-{arch}/Packages.gz"
+    content = read_gzipped_file(packages_path)
+    
+    if content is None:
+        return {}
+    
+    return parse_packages_file(content)
+
+
+def get_sources_index(dest_base: str, distribution: str, component: str) -> Dict[str, Dict[str, str]]:
+    """Read and parse local Sources.gz for a specific component"""
+    sources_path = f"{dest_base}/dists/{distribution}/{component}/source/Sources.gz"
+    content = read_gzipped_file(sources_path)
+    
+    if content is None:
+        return {}
+    
+    return parse_sources_file(content)