umpaper-fetch 1.0.0__py3-none-any.whl

umpaper_fetch/cli.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""
+Command-line interface for umpaper-fetch package.
+
+This module provides the main entry point for the um-papers command.
+"""
+
+import argparse
+import getpass
+import logging
+import os
+import sys
+from pathlib import Path
+
+from .auth.um_authenticator import UMAuthenticator
+from .scraper.paper_scraper import PaperScraper
+from .downloader.pdf_downloader import PDFDownloader
+from .utils.zip_creator import ZipCreator
+from .utils.logger import setup_logger
+
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Download all past year papers for a UM subject code",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  um-papers
+  um-papers --username 24012345 --subject-code WIA1005
+  um-papers --username 24056789 --subject-code WXES1116 --show-browser
+  um-papers --no-location-prompt --output-dir "C:/Downloads"
+        """
+    )
+    
+    parser.add_argument(
+        '--username', '-u',
+        help='UM username (without @siswa.um.edu.my)',
+        type=str
+    )
+    
+    parser.add_argument(
+        '--subject-code', '-s',
+        help='Subject code to search for (e.g., WIA1005)',
+        type=str
+    )
+    
+    parser.add_argument(
+        '--output-dir', '-o',
+        help='Output directory for downloads (default: ./downloads)',
+        default='./downloads',
+        type=str
+    )
+    
+    parser.add_argument(
+        '--no-location-prompt',
+        help='Skip location selection prompt and use default output directory',
+        action='store_true'
+    )
+    
+    parser.add_argument(
+        '--show-browser',
+        help='Show browser window (default is headless mode)',
+        action='store_true'
+    )
+    
+    parser.add_argument(
+        '--browser', '-b',
+        help='Browser to use (auto, chrome, edge). Default: edge',
+        choices=['auto', 'chrome', 'edge'],
+        default='edge',
+        type=str
+    )
+    
+    parser.add_argument(
+        '--timeout',
+        help='Session timeout in seconds (default: 30)',
+        default=30,
+        type=int
+    )
+    
+    parser.add_argument(
+        '--max-retries',
+        help='Maximum retry attempts (default: 3)',
+        default=3,
+        type=int
+    )
+    
+    parser.add_argument(
+        '--verbose', '-v',
+        help='Enable verbose logging',
+        action='store_true'
+    )
+    
+    parser.add_argument(
+        '--version',
+        action='version',
+        version='%(prog)s 1.0.0'
+    )
+    
+    return parser.parse_args()
+
+
+def get_credentials(username=None):
+    """Get user credentials securely."""
+    if not username:
+        username = input("Enter your UM username (without @siswa.um.edu.my): ").strip()
+    
+    if not username:
+        print("Error: Username cannot be empty")
+        sys.exit(1)
+    
+    password = getpass.getpass("Enter your UM password: ")
+    
+    if not password:
+        print("Error: Password cannot be empty")
+        sys.exit(1)
+    
+    return username, password
+
+
+def get_subject_code(subject_code=None):
+    """Get subject code from user."""
+    if not subject_code:
+        subject_code = input("Enter subject code (e.g., WIA1005): ").strip().upper()
+    
+    if not subject_code:
+        print("Error: Subject code cannot be empty")
+        sys.exit(1)
+    
+    return subject_code
+
+
+def get_download_location(default_output_dir):
+    """
+    Get custom download location from user.
+    
+    Args:
+        default_output_dir (Path): Default output directory
+        
+    Returns:
+        Path: User-chosen download location or default
+    """
+    print(f"\n📂 Download Location Settings")
+    print("="*50)
+    print(f"Default location: {default_output_dir.absolute()}")
+    print("\nOptions:")
+    print("1. Use default location (downloads folder)")
+    print("2. Choose custom location")
+    
+    while True:
+        choice = input("\nSelect option (1 or 2): ").strip()
+        
+        if choice == '1':
+            print(f"✅ Using default location: {default_output_dir.absolute()}")
+            return default_output_dir
+        
+        elif choice == '2':
+            while True:
+                custom_path = input("\nEnter custom download path: ").strip()
+                
+                if not custom_path:
+                    print("❌ Path cannot be empty. Please try again.")
+                    continue
+                
+                try:
+                    # Convert to Path object and expand user home directory (~)
+                    custom_dir = Path(custom_path).expanduser()
+                    
+                    # Try to create the directory if it doesn't exist
+                    custom_dir.mkdir(parents=True, exist_ok=True)
+                    
+                    # Test if we can write to this directory
+                    test_file = custom_dir / "test_write.tmp"
+                    try:
+                        test_file.write_text("test")
+                        test_file.unlink()  # Delete test file
+                        print(f"✅ Custom location set: {custom_dir.absolute()}")
+                        return custom_dir
+                    except Exception as write_error:
+                        print(f"❌ Cannot write to this location: {write_error}")
+                        print("Please choose a different path or check permissions.")
+                        
+                except Exception as path_error:
+                    print(f"❌ Invalid path: {path_error}")
+                    print("Please enter a valid directory path.")
+        
+        else:
+            print("❌ Please enter '1' or '2'.")
+
+
+def main():
+    """Main execution function."""
+    args = parse_arguments()
+    
+    # Setup logging
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logger = setup_logger(log_level)
+    
+    try:
+        # Create default output directory
+        default_output_dir = Path(args.output_dir)
+        default_output_dir.mkdir(parents=True, exist_ok=True)
+        
+        logger.info("=== UM Past Year Paper Downloader ===")
+        
+        # Get credentials
+        username, password = get_credentials(args.username)
+        
+        # Get subject code
+        subject_code = get_subject_code(args.subject_code)
+        
+        # Get download location
+        if args.no_location_prompt:
+            output_dir = default_output_dir
+            logger.info(f"Using default output directory: {output_dir.absolute()}")
+        else:
+            output_dir = get_download_location(default_output_dir)
+        
+        # Show configuration summary
+        print(f"\n📋 Configuration Summary")
+        print("="*50)
+        print(f"Username: {username}")
+        print(f"Subject Code: {subject_code}")
+        print(f"Output Directory: {output_dir.absolute()}")
+        print(f"Browser: {args.browser}")
+        print(f"Headless Mode: {not args.show_browser}")
+        print(f"Timeout: {args.timeout}s")
+        print(f"Max Retries: {args.max_retries}")
+        
+        # Confirm before proceeding
+        print(f"\n🚀 Ready to start downloading papers for {subject_code}")
+        confirm = input("Continue? (y/N): ").strip().lower()
+        
+        if confirm not in ['y', 'yes']:
+            print("❌ Operation cancelled by user")
+            sys.exit(0)
+        
+        print("\n" + "="*60)
+        print("🔄 Starting download process...")
+        print("="*60)
+        
+        # Step 1: Authentication
+        logger.info("Step 1: Authenticating with UM portal...")
+        authenticator = UMAuthenticator(
+            headless=not args.show_browser,
+            browser=args.browser,
+            timeout=args.timeout
+        )
+        
+        session = authenticator.login(username, password)
+        if not session:
+            logger.error("❌ Authentication failed")
+            sys.exit(1)
+        
+        logger.info("✅ Authentication successful")
+        
+        # Step 2: Search for papers
+        logger.info(f"Step 2: Searching for papers with subject code: {subject_code}")
+        scraper = PaperScraper(session)
+        papers = scraper.search_papers(subject_code)
+        
+        if not papers:
+            logger.warning(f"❌ No papers found for subject code: {subject_code}")
+            print(f"\n❌ No papers found for subject code: {subject_code}")
+            print("Please check the subject code and try again.")
+            sys.exit(1)
+        
+        logger.info(f"✅ Found {len(papers)} papers")
+        print(f"✅ Found {len(papers)} papers")
+        
+        # Step 3: Download papers
+        logger.info("Step 3: Downloading papers...")
+        downloader = PDFDownloader(session, output_dir, max_retries=args.max_retries)
+        downloaded_files = downloader.download_papers(papers)
+        
+        if not downloaded_files:
+            logger.error("❌ No papers were downloaded successfully")
+            sys.exit(1)
+        
+        logger.info(f"✅ Downloaded {len(downloaded_files)} papers")
+        
+        # Step 4: Create ZIP archive
+        logger.info("Step 4: Creating ZIP archive...")
+        zip_creator = ZipCreator()
+        zip_path = zip_creator.create_zip(downloaded_files, subject_code, output_dir)
+        
+        if zip_path:
+            logger.info(f"✅ ZIP archive created: {zip_path}")
+            print(f"\n🎉 Success! All papers downloaded and zipped:")
+            print(f"📦 ZIP file: {zip_path}")
+            print(f"📁 Individual files: {output_dir / subject_code}")
+        else:
+            logger.warning("⚠️ ZIP creation failed, but individual files are available")
+            print(f"\n⚠️ Papers downloaded but ZIP creation failed")
+            print(f"📁 Individual files: {output_dir / subject_code}")
+        
+        # Cleanup
+        authenticator.cleanup()
+        
+        print(f"\n✅ Download completed successfully!")
+        print(f"Total papers downloaded: {len(downloaded_files)}")
+        
+    except KeyboardInterrupt:
+        logger.info("❌ Operation cancelled by user (Ctrl+C)")
+        print("\n❌ Operation cancelled by user")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"❌ Unexpected error: {e}")
+        print(f"\n❌ An error occurred: {e}")
+        print("Check the logs for more details.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main() 

umpaper_fetch/downloader/__init__.py

@@ -0,0 +1 @@
+# Downloader package 

umpaper_fetch/downloader/pdf_downloader.py

@@ -0,0 +1,207 @@
+"""
+PDF Downloader Module
+
+Handles downloading PDF files with progress tracking, retry logic,
+and concurrent download management.
+"""
+
+import logging
+import os
+import time
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+import requests
+
+
+class PDFDownloader:
+    """Downloads PDF files with progress tracking and retry logic."""
+    
+    def __init__(self, session, output_dir, max_retries=3, max_workers=4):
+        """Initialize the PDF downloader."""
+        self.session = session
+        self.output_dir = Path(output_dir)
+        self.max_retries = max_retries
+        self.max_workers = max_workers
+        self.logger = logging.getLogger(__name__)
+        
+        # Ensure output directory exists
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def download_papers(self, papers):
+        """
+        Download multiple papers with progress tracking.
+        
+        Args:
+            papers (list): List of PaperInfo objects to download
+            
+        Returns:
+            list: List of successfully downloaded file paths
+        """
+        if not papers:
+            return []
+        
+        self.logger.info(f"Starting download of {len(papers)} papers...")
+        downloaded_files = []
+        
+        # Create progress bar
+        with tqdm(total=len(papers), desc="Downloading papers", unit="file") as pbar:
+            # Use ThreadPoolExecutor for concurrent downloads
+            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                # Submit all download tasks
+                future_to_paper = {
+                    executor.submit(self._download_paper, paper): paper
+                    for paper in papers
+                }
+                
+                # Process completed downloads
+                for future in as_completed(future_to_paper):
+                    paper = future_to_paper[future]
+                    try:
+                        file_path = future.result()
+                        if file_path:
+                            downloaded_files.append(file_path)
+                            self.logger.debug(f"Downloaded: {paper.filename}")
+                        else:
+                            self.logger.warning(f"Failed to download: {paper.title}")
+                    except Exception as e:
+                        self.logger.error(f"Error downloading {paper.title}: {e}")
+                    finally:
+                        pbar.update(1)
+        
+        self.logger.info(f"Downloaded {len(downloaded_files)}/{len(papers)} papers successfully")
+        return downloaded_files
+    
+    def _download_paper(self, paper):
+        """
+        Download a single paper with retry logic.
+        
+        Args:
+            paper (PaperInfo): Paper information object
+            
+        Returns:
+            str: Path to downloaded file, or None if failed
+        """
+        for attempt in range(self.max_retries + 1):
+            try:
+                file_path = self._download_file(paper.download_url, paper.filename)
+                if file_path and self._verify_download(file_path):
+                    return file_path
+                
+            except Exception as e:
+                self.logger.warning(
+                    f"Download attempt {attempt + 1} failed for {paper.title}: {e}"
+                )
+                
+                if attempt < self.max_retries:
+                    # Wait before retry with exponential backoff
+                    wait_time = 2 ** attempt
+                    time.sleep(wait_time)
+        
+        self.logger.error(f"Failed to download after {self.max_retries + 1} attempts: {paper.title}")
+        return None
+    
+    def _download_file(self, url, filename):
+        """
+        Download a single file from URL.
+        
+        Args:
+            url (str): Download URL
+            filename (str): Target filename
+            
+        Returns:
+            str: Path to downloaded file
+        """
+        file_path = self.output_dir / filename
+        
+        # Avoid re-downloading if file already exists and is valid
+        if file_path.exists() and self._verify_download(file_path):
+            self.logger.debug(f"File already exists: {filename}")
+            return str(file_path)
+        
+        # Start download
+        response = self.session.get(url, stream=True, timeout=60)
+        response.raise_for_status()
+        
+        # Check if response is actually a PDF
+        content_type = response.headers.get('content-type', '').lower()
+        if 'pdf' not in content_type and 'application/octet-stream' not in content_type:
+            self.logger.warning(f"Unexpected content type for {filename}: {content_type}")
+        
+        # Write file with progress tracking
+        total_size = int(response.headers.get('content-length', 0))
+        
+        with open(file_path, 'wb') as f:
+            if total_size > 0:
+                # Track download progress for large files
+                downloaded = 0
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded += len(chunk)
+            else:
+                # For files without content-length header
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+        
+        self.logger.debug(f"Downloaded {filename} ({file_path.stat().st_size} bytes)")
+        return str(file_path)
+    
+    def _verify_download(self, file_path):
+        """
+        Verify that the downloaded file is valid.
+        
+        Args:
+            file_path (str or Path): Path to the downloaded file
+            
+        Returns:
+            bool: True if file is valid, False otherwise
+        """
+        try:
+            file_path = Path(file_path)
+            
+            # Check if file exists and has content
+            if not file_path.exists() or file_path.stat().st_size == 0:
+                return False
+            
+            # Basic PDF validation - check PDF header
+            with open(file_path, 'rb') as f:
+                header = f.read(8)
+                if not header.startswith(b'%PDF'):
+                    self.logger.warning(f"File does not appear to be a PDF: {file_path.name}")
+                    # Don't reject non-PDF files completely, might be valid documents
+                    # return False
+            
+            return True
+            
+        except Exception as e:
+            self.logger.warning(f"Error verifying file {file_path}: {e}")
+            return False
+    
+    def cleanup_failed_downloads(self):
+        """Remove any incomplete or corrupted downloads."""
+        cleaned_count = 0
+        
+        for file_path in self.output_dir.glob('*.pdf'):
+            if not self._verify_download(file_path):
+                try:
+                    file_path.unlink()
+                    cleaned_count += 1
+                    self.logger.debug(f"Removed invalid file: {file_path.name}")
+                except Exception as e:
+                    self.logger.warning(f"Could not remove invalid file {file_path}: {e}")
+        
+        if cleaned_count > 0:
+            self.logger.info(f"Cleaned up {cleaned_count} invalid downloaded files")
+    
+    def get_download_stats(self):
+        """Get statistics about downloaded files."""
+        pdf_files = list(self.output_dir.glob('*.pdf'))
+        total_size = sum(f.stat().st_size for f in pdf_files)
+        
+        return {
+            'count': len(pdf_files),
+            'total_size_mb': total_size / (1024 * 1024),
+            'files': [f.name for f in pdf_files]
+        } 

umpaper_fetch/scraper/__init__.py

@@ -0,0 +1 @@
+# Scraper package