longcat-scraper 0.1.0__py3-none-any.whl

longcat_scraper/__init__.py

@@ -0,0 +1,141 @@
+import requests
+import time
+import logging
+from pathlib import Path
+
+# ✅ Logging setup
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler("scraper.log"),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# ✅ URLs to scrape
+urls = [
+    "https://invest-tracing.com/",
+    "https://invest-tracing.com/latest-hyips.html",
+    "https://invest-tracing.com/scam-hyips.html",
+]
+
+# ✅ Robust Headers
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+}
+
+# ✅ Retry settings
+MAX_RETRIES = 3
+RETRY_DELAY = 5  # seconds
+TIMEOUT = 30  # seconds
+
+# ✅ Output folder
+OUTPUT_DIR = Path("scraped_pages")
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+
+def download_page(url: str, retries: int = MAX_RETRIES) -> str | None:
+    """Download page with retry logic"""
+    for attempt in range(1, retries + 1):
+        try:
+            logger.info(f"⏳ Downloading: {url} (Attempt {attempt}/{retries})")
+            
+            r = requests.get(
+                url,
+                headers=headers,
+                timeout=TIMEOUT,
+                allow_redirects=True
+            )
+            r.raise_for_status()  # Raise error for bad status codes
+            
+            logger.info(f"✅ Success! Status: {r.status_code}, Size: {len(r.text)} bytes")
+            return r.text
+            
+        except requests.exceptions.Timeout:
+            logger.warning(f"⏱️ Timeout on attempt {attempt}")
+            
+        except requests.exceptions.ConnectionError:
+            logger.warning(f"🔌 Connection error on attempt {attempt}")
+            
+        except requests.exceptions.HTTPError as e:
+            logger.warning(f"❌ HTTP Error: {e}")
+            if r.status_code == 404:
+                logger.error(f"🚫 Page not found: {url}")
+                return None  # No point retrying 404
+                
+        except requests.exceptions.RequestException as e:
+            logger.warning(f"⚠️ Request error: {e}")
+        
+        # Wait before retry
+        if attempt < retries:
+            wait_time = RETRY_DELAY * attempt  # Exponential backoff
+            logger.info(f"⏳ Waiting {wait_time}s before retry...")
+            time.sleep(wait_time)
+    
+    logger.error(f"❌ Failed after {retries} attempts: {url}")
+    return None
+
+
+def save_page(content: str, filename: str) -> bool:
+    """Save page content to file"""
+    try:
+        filepath = OUTPUT_DIR / filename
+        with open(filepath, "w", encoding="utf-8") as f:
+            f.write(content)
+        logger.info(f"💾 Saved: {filepath}")
+        return True
+    except Exception as e:
+        logger.error(f"❌ Failed to save {filename}: {e}")
+        return False
+
+
+def main():
+    """Main function to scrape all pages"""
+    logger.info("=" * 50)
+    logger.info("🚀 Starting Web Scraper")
+    logger.info(f"📄 Total URLs: {len(urls)}")
+    logger.info("=" * 50)
+    
+    results = {
+        "success": 0,
+        "failed": 0,
+        "skipped": 0
+    }
+    
+    for i, url in enumerate(urls, start=1):
+        logger.info(f"\n{'─' * 40}")
+        logger.info(f"[{i}/{len(urls)}] Processing: {url}")
+        
+        # Download page
+        content = download_page(url)
+        
+        if content:
+            filename = f"page_{i}.html"
+            if save_page(content, filename):
+                results["success"] += 1
+            else:
+                results["failed"] += 1
+        else:
+            results["failed"] += 1
+        
+        # Small delay between requests (be polite to server)
+        if i < len(urls):
+            time.sleep(2)
+    
+    # Summary
+    logger.info("\n" + "=" * 50)
+    logger.info("📊 SCRAPING SUMMARY")
+    logger.info(f"✅ Success: {results['success']}")
+    logger.info(f"❌ Failed: {results['failed']}")
+    logger.info(f"⏭️ Skipped: {results['skipped']}")
+    logger.info("=" * 50)
+
+
+if __name__ == "__main__":
+    main()

longcat_scraper-0.1.0.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: longcat-scraper
+Version: 0.1.0
+Summary: A robust web scraper built with Longcat AI
+Home-page: https://github.com/yourusername/longcat-scraper
+Author: BotMaster
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Longcat Scraper 🚀
+
+A robust web scraper generated and managed by **Longcat AI**.
+
+## Features
+- ✅ Multi-URL scraping
+- ✅ Retry logic with exponential backoff
+- ✅ Detailed logging
+- ✅ Error handling for timeouts and connection issues
+
+## Installation
+```bash
+pip install longcat-scraper
+```
+
+## Usage
+After installation, run:
+```bash
+longcat-scrape
+```
+OR use it in your code:
+```python
+import longcat_scraper
+longcat_scraper.main()
+```

longcat_scraper-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+longcat_scraper/__init__.py,sha256=UKpTeVuGnzsJXlPQuvlCfsETso5TbL8kHDsN1yt3_Ck,4273
+longcat_scraper-0.1.0.dist-info/METADATA,sha256=g4FQtDOOTixkQw10FPVcKfHJX781oNCNV8jRyaAmweU,1046
+longcat_scraper-0.1.0.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+longcat_scraper-0.1.0.dist-info/entry_points.txt,sha256=2eUl0KQzgk4MiMbvfVLu2y8C4wVlWfQA8AokKCmbYgU,56
+longcat_scraper-0.1.0.dist-info/top_level.txt,sha256=f4hsz5QelaUa6l_Z29l5R9e5WuESCAoZVNO67GimPeg,16
+longcat_scraper-0.1.0.dist-info/RECORD,,

longcat_scraper-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

longcat_scraper-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+longcat-scrape = longcat_scraper:main

longcat_scraper-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+longcat_scraper