umpaper-fetch 1.0.0__py3-none-any.whl

downloader/pdf_downloader.py

@@ -0,0 +1,207 @@
+"""
+PDF Downloader Module
+
+Handles downloading PDF files with progress tracking, retry logic,
+and concurrent download management.
+"""
+
+import logging
+import os
+import time
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+import requests
+
+
+class PDFDownloader:
+    """Downloads PDF files with progress tracking and retry logic."""
+    
+    def __init__(self, session, output_dir, max_retries=3, max_workers=4):
+        """Initialize the PDF downloader."""
+        self.session = session
+        self.output_dir = Path(output_dir)
+        self.max_retries = max_retries
+        self.max_workers = max_workers
+        self.logger = logging.getLogger(__name__)
+        
+        # Ensure output directory exists
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def download_papers(self, papers):
+        """
+        Download multiple papers with progress tracking.
+        
+        Args:
+            papers (list): List of PaperInfo objects to download
+            
+        Returns:
+            list: List of successfully downloaded file paths
+        """
+        if not papers:
+            return []
+        
+        self.logger.info(f"Starting download of {len(papers)} papers...")
+        downloaded_files = []
+        
+        # Create progress bar
+        with tqdm(total=len(papers), desc="Downloading papers", unit="file") as pbar:
+            # Use ThreadPoolExecutor for concurrent downloads
+            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                # Submit all download tasks
+                future_to_paper = {
+                    executor.submit(self._download_paper, paper): paper
+                    for paper in papers
+                }
+                
+                # Process completed downloads
+                for future in as_completed(future_to_paper):
+                    paper = future_to_paper[future]
+                    try:
+                        file_path = future.result()
+                        if file_path:
+                            downloaded_files.append(file_path)
+                            self.logger.debug(f"Downloaded: {paper.filename}")
+                        else:
+                            self.logger.warning(f"Failed to download: {paper.title}")
+                    except Exception as e:
+                        self.logger.error(f"Error downloading {paper.title}: {e}")
+                    finally:
+                        pbar.update(1)
+        
+        self.logger.info(f"Downloaded {len(downloaded_files)}/{len(papers)} papers successfully")
+        return downloaded_files
+    
+    def _download_paper(self, paper):
+        """
+        Download a single paper with retry logic.
+        
+        Args:
+            paper (PaperInfo): Paper information object
+            
+        Returns:
+            str: Path to downloaded file, or None if failed
+        """
+        for attempt in range(self.max_retries + 1):
+            try:
+                file_path = self._download_file(paper.download_url, paper.filename)
+                if file_path and self._verify_download(file_path):
+                    return file_path
+                
+            except Exception as e:
+                self.logger.warning(
+                    f"Download attempt {attempt + 1} failed for {paper.title}: {e}"
+                )
+                
+                if attempt < self.max_retries:
+                    # Wait before retry with exponential backoff
+                    wait_time = 2 ** attempt
+                    time.sleep(wait_time)
+        
+        self.logger.error(f"Failed to download after {self.max_retries + 1} attempts: {paper.title}")
+        return None
+    
+    def _download_file(self, url, filename):
+        """
+        Download a single file from URL.
+        
+        Args:
+            url (str): Download URL
+            filename (str): Target filename
+            
+        Returns:
+            str: Path to downloaded file
+        """
+        file_path = self.output_dir / filename
+        
+        # Avoid re-downloading if file already exists and is valid
+        if file_path.exists() and self._verify_download(file_path):
+            self.logger.debug(f"File already exists: {filename}")
+            return str(file_path)
+        
+        # Start download
+        response = self.session.get(url, stream=True, timeout=60)
+        response.raise_for_status()
+        
+        # Check if response is actually a PDF
+        content_type = response.headers.get('content-type', '').lower()
+        if 'pdf' not in content_type and 'application/octet-stream' not in content_type:
+            self.logger.warning(f"Unexpected content type for {filename}: {content_type}")
+        
+        # Write file with progress tracking
+        total_size = int(response.headers.get('content-length', 0))
+        
+        with open(file_path, 'wb') as f:
+            if total_size > 0:
+                # Track download progress for large files
+                downloaded = 0
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded += len(chunk)
+            else:
+                # For files without content-length header
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+        
+        self.logger.debug(f"Downloaded {filename} ({file_path.stat().st_size} bytes)")
+        return str(file_path)
+    
+    def _verify_download(self, file_path):
+        """
+        Verify that the downloaded file is valid.
+        
+        Args:
+            file_path (str or Path): Path to the downloaded file
+            
+        Returns:
+            bool: True if file is valid, False otherwise
+        """
+        try:
+            file_path = Path(file_path)
+            
+            # Check if file exists and has content
+            if not file_path.exists() or file_path.stat().st_size == 0:
+                return False
+            
+            # Basic PDF validation - check PDF header
+            with open(file_path, 'rb') as f:
+                header = f.read(8)
+                if not header.startswith(b'%PDF'):
+                    self.logger.warning(f"File does not appear to be a PDF: {file_path.name}")
+                    # Don't reject non-PDF files completely, might be valid documents
+                    # return False
+            
+            return True
+            
+        except Exception as e:
+            self.logger.warning(f"Error verifying file {file_path}: {e}")
+            return False
+    
+    def cleanup_failed_downloads(self):
+        """Remove any incomplete or corrupted downloads."""
+        cleaned_count = 0
+        
+        for file_path in self.output_dir.glob('*.pdf'):
+            if not self._verify_download(file_path):
+                try:
+                    file_path.unlink()
+                    cleaned_count += 1
+                    self.logger.debug(f"Removed invalid file: {file_path.name}")
+                except Exception as e:
+                    self.logger.warning(f"Could not remove invalid file {file_path}: {e}")
+        
+        if cleaned_count > 0:
+            self.logger.info(f"Cleaned up {cleaned_count} invalid downloaded files")
+    
+    def get_download_stats(self):
+        """Get statistics about downloaded files."""
+        pdf_files = list(self.output_dir.glob('*.pdf'))
+        total_size = sum(f.stat().st_size for f in pdf_files)
+        
+        return {
+            'count': len(pdf_files),
+            'total_size_mb': total_size / (1024 * 1024),
+            'files': [f.name for f in pdf_files]
+        } 

scraper/__init__.py

@@ -0,0 +1 @@
+# Scraper package 

scraper/paper_scraper.py

@@ -0,0 +1,316 @@
+"""
+Paper Scraper Module
+
+Handles searching for papers by subject code and extracting download URLs
+and metadata from the UM exam paper repository.
+"""
+
+import logging
+import re
+from urllib.parse import urljoin, parse_qs, urlparse
+from bs4 import BeautifulSoup
+import requests
+
+
+class PaperInfo:
+    """Data class for storing paper information."""
+    
+    def __init__(self, title, download_url, year=None, semester=None, paper_type=None):
+        self.title = title
+        self.download_url = download_url
+        self.year = year
+        self.semester = semester
+        self.paper_type = paper_type
+        self.filename = self._generate_filename()
+    
+    def _generate_filename(self):
+        """Generate a clean filename for the paper."""
+        # Extract useful parts from title - remove subject code and semester/year info to avoid duplication
+        title_to_clean = self.title
+        
+        # Remove subject code pattern from title (e.g., "WIA1005 (Semester 1, 2024)")
+        title_to_clean = re.sub(r'[A-Z]{2,4}\d{4}\s*\([^)]+\)\s*', '', title_to_clean)
+        
+        # Clean the remaining title
+        clean_title = re.sub(r'[^\w\s-]', '', title_to_clean)
+        clean_title = re.sub(r'\s+', '_', clean_title.strip())
+        
+        # Add year and semester if available
+        parts = []
+        if self.year:
+            parts.append(f"Y{self.year}")
+        if self.semester:
+            parts.append(f"S{self.semester}")
+        if self.paper_type:
+            parts.append(self.paper_type)
+        
+        if parts:
+            filename = f"{'_'.join(parts)}_{clean_title}.pdf"
+        else:
+            filename = f"{clean_title}.pdf"
+        
+        # Ensure filename is not too long
+        if len(filename) > 100:
+            filename = filename[:95] + ".pdf"
+        
+        return filename
+    
+    def __str__(self):
+        return f"PaperInfo(title='{self.title}', year={self.year}, semester={self.semester})"
+
+
+class PaperScraper:
+    """Scrapes exam papers from UM repository."""
+    
+    def __init__(self, session):
+        """Initialize scraper with authenticated session."""
+        self.session = session
+        self.base_url = "https://exampaper-um-edu-my.eu1.proxy.openathens.net"
+        self.search_url = f"{self.base_url}/cgi/search"
+        self.logger = logging.getLogger(__name__)
+    
+    def search_papers(self, subject_code, max_results=100):
+        """
+        Search for papers by subject code.
+        
+        Args:
+            subject_code (str): Subject code to search for
+            max_results (int): Maximum number of results to return
+            
+        Returns:
+            list[PaperInfo]: List of paper information objects
+        """
+        self.logger.info(f"Searching for papers with subject code: {subject_code}")
+        
+        papers = []
+        
+        try:
+            # Use the correct search URL and parameters based on the actual form
+            search_params = {
+                'q': subject_code,
+                '_action_search': 'Search',
+                '_order': 'bytitle',
+                'basic_srchtype': 'ALL',
+                '_satisfyall': 'ALL'
+            }
+            
+            self.logger.info(f"Performing search with params: {search_params}")
+            
+            # Perform search request using GET (like the form does)
+            response = self.session.get(
+                "https://exampaper-um-edu-my.eu1.proxy.openathens.net/cgi/search",
+                params=search_params,
+                timeout=30
+            )
+            
+            if response.status_code != 200:
+                self.logger.error(f"Search request failed: {response.status_code}")
+                return papers
+            
+            # Parse search results
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            # Check if we got results
+            results_text = soup.find('div', class_='ep_search_controls')
+            if results_text:
+                text = results_text.get_text()
+                self.logger.info(f"Search results info: {text}")
+                
+                # Extract number of results
+                import re
+                match = re.search(r'(\d+)\s+of\s+(\d+)', text)
+                if match:
+                    total_results = int(match.group(2))
+                    self.logger.info(f"Found {total_results} total results")
+                else:
+                    self.logger.warning("Could not determine number of results")
+            
+            papers = self._parse_search_results(soup, subject_code)
+            
+            self.logger.info(f"Successfully extracted {len(papers)} papers for {subject_code}")
+            
+        except Exception as e:
+            self.logger.error(f"Error searching for papers: {e}")
+        
+        return papers[:max_results]
+    
+    def _parse_search_results(self, soup, subject_code):
+        """Parse search results from HTML."""
+        papers = []
+        
+        # Look for the results table
+        results_table = soup.find('table', class_='ep_paginate_list')
+        if not results_table:
+            self.logger.warning("No results table found with class 'ep_paginate_list'")
+            return papers
+        
+        # Find all result rows
+        result_rows = results_table.find_all('tr', class_='ep_search_result')
+        self.logger.info(f"Found {len(result_rows)} result rows")
+        
+        for i, row in enumerate(result_rows, 1):
+            try:
+                self.logger.info(f"Processing result {i}...")
+                paper_info = self._extract_paper_info_from_row(row, subject_code)
+                if paper_info:
+                    papers.append(paper_info)
+                    self.logger.info(f"✅ Extracted: {paper_info.title}")
+                else:
+                    self.logger.warning(f"❌ Could not extract info from result {i}")
+            except Exception as e:
+                self.logger.warning(f"Error parsing result {i}: {e}")
+                continue
+        
+        return papers
+    
+    def _extract_paper_info_from_row(self, row, subject_code):
+        """Extract paper information from a search result row."""
+        try:
+            # Get all cells in the row
+            cells = row.find_all('td')
+            if len(cells) < 2:
+                self.logger.warning("Row doesn't have enough cells")
+                return None
+            
+            # The main content is in the second cell
+            content_cell = cells[1]
+            
+            # Extract the title and basic info
+            # Pattern: "WIA1005 (Semester X, YEAR) Title"
+            text_content = content_cell.get_text(strip=True)
+            self.logger.info(f"Row content: {text_content[:100]}...")
+            
+            # Extract semester and year
+            semester_year_match = re.search(r'\(Semester (\d+), (\d{4})\)', text_content)
+            if semester_year_match:
+                semester = semester_year_match.group(1)
+                year = semester_year_match.group(2)
+            else:
+                semester = None
+                year = None
+                self.logger.warning("Could not extract semester/year info")
+            
+            # Find the main paper link (usually the title link)
+            title_link = content_cell.find('a', href=True)
+            if title_link:
+                title = title_link.get_text(strip=True)
+                # Remove italic formatting
+                title = re.sub(r'[/*]', '', title)
+                paper_url = urljoin(self.base_url, title_link.get('href'))
+                self.logger.info(f"Found title link: {title}")
+            else:
+                self.logger.warning("No title link found")
+                return None
+            
+            # Look for direct PDF download link
+            download_url = None
+            
+            # Check the third cell (if exists) for PDF links
+            if len(cells) > 2:
+                pdf_cell = cells[2]
+                pdf_links = pdf_cell.find_all('a', href=True)
+                for link in pdf_links:
+                    href = link.get('href')
+                    if href and '.pdf' in href.lower():
+                        download_url = urljoin(self.base_url, href)
+                        self.logger.info(f"Found direct PDF link: {download_url}")
+                        break
+            
+            # If no direct PDF link found, try to get it from the paper page
+            if not download_url:
+                self.logger.info("No direct PDF link found, checking paper page...")
+                download_url = self._get_download_url(paper_url)
+            
+            if not download_url:
+                self.logger.warning(f"No download URL found for: {title}")
+                return None
+            
+            # Generate a clean title without redundant info (year/semester will be in filename prefix)
+            clean_title = f"{subject_code} {title}"
+            
+            paper_type = self._determine_paper_type(title)
+            
+            return PaperInfo(
+                title=clean_title,
+                download_url=download_url,
+                year=year,
+                semester=semester,
+                paper_type=paper_type
+            )
+            
+        except Exception as e:
+            self.logger.warning(f"Error extracting paper info: {e}")
+            return None
+    
+    def _determine_paper_type(self, title):
+        """Determine the type of paper from the title."""
+        title_lower = title.lower()
+        
+        if 'final' in title_lower:
+            return 'Final'
+        elif 'mid' in title_lower or 'midterm' in title_lower:
+            return 'Midterm'
+        elif 'quiz' in title_lower:
+            return 'Quiz'
+        elif 'test' in title_lower:
+            return 'Test'
+        else:
+            return 'Exam'
+    
+    def _get_download_url(self, paper_url):
+        """Get the actual PDF download URL from the paper page."""
+        try:
+            self.logger.info(f"Getting download URL from: {paper_url}")
+            response = self.session.get(paper_url, timeout=15)
+            if response.status_code != 200:
+                self.logger.warning(f"Failed to access paper page: {response.status_code}")
+                return None
+            
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            # Method 1: Look for direct PDF download links
+            download_links = soup.find_all('a', href=True)
+            
+            for link in download_links:
+                href = link.get('href')
+                link_text = link.get_text(strip=True).lower()
+                
+                # Look for PDF files or download links
+                if href and ('.pdf' in href.lower() or 
+                           'download' in href.lower() or 
+                           'download' in link_text or
+                           'pdf' in link_text):
+                    download_url = urljoin(self.base_url, href)
+                    self.logger.info(f"Found download link: {download_url}")
+                    return download_url
+            
+            # Method 2: Look for repository-specific patterns
+            # UM repository often uses /id/eprint/XXXXX/1/filename.pdf
+            eprint_links = soup.find_all('a', href=re.compile(r'/\d+/\d+/.*\.pdf$', re.I))
+            if eprint_links:
+                download_url = urljoin(self.base_url, eprint_links[0].get('href'))
+                self.logger.info(f"Found eprint PDF: {download_url}")
+                return download_url
+            
+            # Method 3: Look for any PDF links
+            pdf_links = soup.find_all('a', href=re.compile(r'\.pdf$', re.I))
+            if pdf_links:
+                download_url = urljoin(self.base_url, pdf_links[0].get('href'))
+                self.logger.info(f"Found PDF link: {download_url}")
+                return download_url
+            
+            # Method 4: Check for embedded objects or iframes
+            objects = soup.find_all(['object', 'embed', 'iframe'])
+            for obj in objects:
+                src = obj.get('src') or obj.get('data')
+                if src and '.pdf' in src.lower():
+                    download_url = urljoin(self.base_url, src)
+                    self.logger.info(f"Found embedded PDF: {download_url}")
+                    return download_url
+            
+            self.logger.warning(f"No download URL found on page: {paper_url}")
+        
+        except Exception as e:
+            self.logger.warning(f"Error getting download URL for {paper_url}: {e}")
+        
+        return None 

umpaper_fetch/__init__.py

@@ -0,0 +1,26 @@
+"""
+umpaper-fetch: Automated downloader for University Malaya past year exam papers.
+
+This package provides tools to automatically download past year exam papers
+from University Malaya's repository through an automated browser interface.
+"""
+
+__version__ = "1.0.0"
+__author__ = "Marcus Mah"
+__email__ = "marcusmah6969@gmail.com"
+__description__ = "Automated downloader for University Malaya past year exam papers"
+
+# Import main classes for easier access
+from .auth.um_authenticator import UMAuthenticator
+from .scraper.paper_scraper import PaperScraper
+from .downloader.pdf_downloader import PDFDownloader
+from .utils.zip_creator import ZipCreator
+from .utils.logger import setup_logger
+
+__all__ = [
+    'UMAuthenticator',
+    'PaperScraper', 
+    'PDFDownloader',
+    'ZipCreator',
+    'setup_logger'
+] 

umpaper_fetch/auth/__init__.py

@@ -0,0 +1 @@
+# Authentication package 

umpaper_fetch/auth/chrome_fix.py

@@ -0,0 +1,119 @@
+"""
+Chrome Driver Fix Module
+
+Handles Chrome driver setup issues, particularly the Win32 application error
+that occurs due to architecture mismatches.
+"""
+
+import os
+import platform
+import logging
+from webdriver_manager.chrome import ChromeDriverManager
+
+
+def get_chrome_driver_path():
+    """
+    Get Chrome driver path with proper architecture handling.
+    
+    Returns:
+        str: Path to the Chrome driver executable
+    """
+    logger = logging.getLogger(__name__)
+    
+    try:
+        # Determine system architecture
+        is_64bit = platform.machine().endswith('64')
+        system = platform.system()
+        
+        logger.info(f"System: {system}, 64-bit: {is_64bit}")
+        
+        # Force specific Chrome driver version/architecture if needed
+        if system == "Windows" and is_64bit:
+            # Try to get the latest driver for Windows 64-bit
+            driver_manager = ChromeDriverManager()
+            driver_path = driver_manager.install()
+            
+            # Verify the driver is executable
+            if os.path.exists(driver_path) and os.access(driver_path, os.X_OK):
+                logger.info(f"Chrome driver ready: {driver_path}")
+                return driver_path
+            else:
+                logger.warning(f"Chrome driver not executable: {driver_path}")
+                # Try to fix permissions
+                try:
+                    os.chmod(driver_path, 0o755)
+                    if os.access(driver_path, os.X_OK):
+                        logger.info("Fixed Chrome driver permissions")
+                        return driver_path
+                except Exception as perm_error:
+                    logger.error(f"Could not fix permissions: {perm_error}")
+        else:
+            # For other systems, use default behavior
+            driver_path = ChromeDriverManager().install()
+            return driver_path
+            
+    except Exception as e:
+        logger.error(f"Chrome driver setup failed: {e}")
+        raise
+    
+    raise Exception("Could not setup Chrome driver")
+
+
+def test_chrome_driver(driver_path):
+    """
+    Test if the Chrome driver is working properly.
+    
+    Args:
+        driver_path (str): Path to Chrome driver
+        
+    Returns:
+        bool: True if driver works, False otherwise
+    """
+    logger = logging.getLogger(__name__)
+    
+    try:
+        import subprocess
+        
+        # Test if the driver can start
+        result = subprocess.run(
+            [driver_path, '--version'], 
+            capture_output=True, 
+            text=True, 
+            timeout=10
+        )
+        
+        if result.returncode == 0:
+            logger.info(f"Chrome driver test passed: {result.stdout.strip()}")
+            return True
+        else:
+            logger.error(f"Chrome driver test failed: {result.stderr}")
+            return False
+            
+    except Exception as e:
+        logger.error(f"Chrome driver test error: {e}")
+        return False
+
+
+def cleanup_chrome_cache():
+    """Clean up problematic Chrome driver cache."""
+    logger = logging.getLogger(__name__)
+    
+    try:
+        import shutil
+        from pathlib import Path
+        
+        # Get the cache directory
+        cache_dir = Path.home() / '.wdm' / 'drivers' / 'chromedriver'
+        
+        if cache_dir.exists():
+            logger.info(f"Cleaning Chrome driver cache: {cache_dir}")
+            shutil.rmtree(cache_dir)
+            logger.info("Chrome driver cache cleaned")
+            return True
+        else:
+            logger.info("No Chrome driver cache to clean")
+            return True
+            
+    except Exception as e:
+        logger.error(f"Could not clean Chrome driver cache: {e}")
+        return False