sitemapxml 0.1.0__tar.gz

sitemapxml-0.1.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: sitemapxml
+Version: 0.1.0
+Summary: A CLI tool to parse sitemaps and extract URL metadata into a CSV format.
+Author-email: Amal Alexander <amalalex95@gmail.com>
+Project-URL: Homepage, https://www.linkedin.com/in/amal-alexander-305780131/
+Project-URL: Bug Tracker, https://www.linkedin.com/in/amal-alexander-305780131/
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+
+# sitemapxml
+
+`sitemapxml` is a powerful and fast command-line tool that extracts all URLs from a given XML sitemap, fetches each URL, and generates a comprehensive CSV report containing:
+- Extract all sitemap URLs
+- HTTP Status Code
+- Title Tag
+- Meta Description
+- Content Length
+- Canonical URL
+
+## Installation
+
+Install via pip:
+
+```bash
+pip install sitemapxml
+```
+
+## Usage
+
+Simply run the CLI command and pass the URL of the sitemap:
+
+```bash
+sitemapxml https://example.com/sitemap.xml
+```
+
+This will automatically create a `sitemap_report.csv` file in your current directory containing all the extracted metrics. You can also specify an output file:
+
+```bash
+sitemapxml https://example.com/sitemap.xml -o my_report.csv
+```
+
+## Author
+- Email: amalalex95@gmail.com
+- LinkedIn: [Amal Alexander](https://www.linkedin.com/in/amal-alexander-305780131/)

sitemapxml-0.1.0/README.md

@@ -0,0 +1,35 @@
+# sitemapxml
+
+`sitemapxml` is a powerful and fast command-line tool that extracts all URLs from a given XML sitemap, fetches each URL, and generates a comprehensive CSV report containing:
+- Extract all sitemap URLs
+- HTTP Status Code
+- Title Tag
+- Meta Description
+- Content Length
+- Canonical URL
+
+## Installation
+
+Install via pip:
+
+```bash
+pip install sitemapxml
+```
+
+## Usage
+
+Simply run the CLI command and pass the URL of the sitemap:
+
+```bash
+sitemapxml https://example.com/sitemap.xml
+```
+
+This will automatically create a `sitemap_report.csv` file in your current directory containing all the extracted metrics. You can also specify an output file:
+
+```bash
+sitemapxml https://example.com/sitemap.xml -o my_report.csv
+```
+
+## Author
+- Email: amalalex95@gmail.com
+- LinkedIn: [Amal Alexander](https://www.linkedin.com/in/amal-alexander-305780131/)

sitemapxml-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sitemapxml"
+version = "0.1.0"
+authors = [
+  { name="Amal Alexander", email="amalalex95@gmail.com" },
+]
+description = "A CLI tool to parse sitemaps and extract URL metadata into a CSV format."
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "requests",
+    "beautifulsoup4"
+]
+
+[project.urls]
+"Homepage" = "https://www.linkedin.com/in/amal-alexander-305780131/"
+"Bug Tracker" = "https://www.linkedin.com/in/amal-alexander-305780131/"
+
+[project.scripts]
+sitemapxml = "sitemapxml.cli:main"

sitemapxml-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sitemapxml-0.1.0/src/sitemapxml/__init__.py

@@ -0,0 +1,6 @@
+"""
+sitemapxml package
+Extract URLs from a sitemap and get metadata.
+"""
+
+__version__ = "0.1.0"

sitemapxml-0.1.0/src/sitemapxml/cli.py

@@ -0,0 +1,129 @@
+import argparse
+import csv
+import concurrent.futures
+import requests
+from bs4 import BeautifulSoup
+import sys
+import threading
+
+def get_sitemap_urls(url, headers):
+    print(f"Fetching sitemap: {url}")
+    try:
+        response = requests.get(url, headers=headers, timeout=15)
+        response.raise_for_status()
+    except Exception as e:
+        print(f"Error fetching sitemap {url}: {e}")
+        return []
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    
+    urls = []
+    # If the sitemap is an index, fetch sub-sitemaps
+    sitemaps = soup.find_all('sitemap')
+    for sitemap in sitemaps:
+        loc = sitemap.find('loc')
+        if loc and loc.text:
+            urls.extend(get_sitemap_urls(loc.text.strip(), headers))
+            
+    # If the sitemap contains urls
+    url_tags = soup.find_all('url')
+    for url_tag in url_tags:
+        loc = url_tag.find('loc')
+        if loc and loc.text:
+            urls.append(loc.text.strip())
+            
+    return list(set(urls))
+
+def process_url(url, headers):
+    result = {
+        'url': url,
+        'status_code': None,
+        'title': None,
+        'description': None,
+        'content_length': None,
+        'canonical': None
+    }
+    try:
+        response = requests.get(url, headers=headers, timeout=15)
+        result['status_code'] = response.status_code
+        result['content_length'] = len(response.content)
+        
+        # Only parse HTML pages for meta tags
+        content_type = response.headers.get('Content-Type', '')
+        if 'text/html' in content_type:
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            # Extract Title
+            if soup.title and soup.title.string:
+                result['title'] = soup.title.string.strip()
+                
+            # Extract Description
+            desc_tag = soup.find('meta', attrs={'name': 'description'})
+            if not desc_tag:
+                desc_tag = soup.find('meta', attrs={'property': 'og:description'})
+            if desc_tag and desc_tag.get('content'):
+                result['description'] = desc_tag['content'].strip()
+                
+            # Extract Canonical
+            canonical_tag = soup.find('link', attrs={'rel': 'canonical'})
+            if canonical_tag and canonical_tag.get('href'):
+                result['canonical'] = canonical_tag['href'].strip()
+                
+    except requests.exceptions.RequestException as e:
+        result['status_code'] = str(e)
+    except Exception:
+        pass
+        
+    return result
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract URLs and metadata from an XML API Sitemap.")
+    parser.add_argument("url", help="The URL of the sitemap.xml to parse")
+    parser.add_argument("-o", "--output", default="sitemap_report.csv", help="Output CSV file name (default: sitemap_report.csv)")
+    parser.add_argument("-w", "--workers", type=int, default=10, help="Number of concurrent workers (default: 10)")
+    
+    args = parser.parse_args()
+    headers = {'User-Agent': 'sitemapxml/0.1.0'}
+    
+    print(f"Starting sitemap extraction from: {args.url}")
+    urls = get_sitemap_urls(args.url, headers)
+    
+    if not urls:
+        print("No URLs found in the sitemap or failed to fetch.")
+        sys.exit(1)
+        
+    print(f"Found {len(urls)} URLs. Starting metadata extraction with {args.workers} concurrent workers...")
+    
+    results = []
+    processed = 0
+    total = len(urls)
+    lock = threading.Lock()
+    
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
+        future_to_url = {executor.submit(process_url, u, headers): u for u in urls}
+        for future in concurrent.futures.as_completed(future_to_url):
+            data = future.result()
+            results.append(data)
+            with lock:
+                processed += 1
+                if processed % 10 == 0 or processed == total:
+                    print(f"Progress: {processed}/{total} URLs processed", end='\r')
+                    
+    print("\nExtraction complete! Saving output...")
+    
+    # Save to CSV
+    fieldnames = ['url', 'status_code', 'title', 'description', 'content_length', 'canonical']
+    
+    try:
+        with open(args.output, mode='w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            for row in results:
+                writer.writerow(row)
+        print(f"Report saved to {args.output}")
+    except Exception as e:
+        print(f"Error saving to CSV: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

sitemapxml-0.1.0/src/sitemapxml.egg-info/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: sitemapxml
+Version: 0.1.0
+Summary: A CLI tool to parse sitemaps and extract URL metadata into a CSV format.
+Author-email: Amal Alexander <amalalex95@gmail.com>
+Project-URL: Homepage, https://www.linkedin.com/in/amal-alexander-305780131/
+Project-URL: Bug Tracker, https://www.linkedin.com/in/amal-alexander-305780131/
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+
+# sitemapxml
+
+`sitemapxml` is a powerful and fast command-line tool that extracts all URLs from a given XML sitemap, fetches each URL, and generates a comprehensive CSV report containing:
+- Extract all sitemap URLs
+- HTTP Status Code
+- Title Tag
+- Meta Description
+- Content Length
+- Canonical URL
+
+## Installation
+
+Install via pip:
+
+```bash
+pip install sitemapxml
+```
+
+## Usage
+
+Simply run the CLI command and pass the URL of the sitemap:
+
+```bash
+sitemapxml https://example.com/sitemap.xml
+```
+
+This will automatically create a `sitemap_report.csv` file in your current directory containing all the extracted metrics. You can also specify an output file:
+
+```bash
+sitemapxml https://example.com/sitemap.xml -o my_report.csv
+```
+
+## Author
+- Email: amalalex95@gmail.com
+- LinkedIn: [Amal Alexander](https://www.linkedin.com/in/amal-alexander-305780131/)

sitemapxml-0.1.0/src/sitemapxml.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+pyproject.toml
+src/sitemapxml/__init__.py
+src/sitemapxml/cli.py
+src/sitemapxml.egg-info/PKG-INFO
+src/sitemapxml.egg-info/SOURCES.txt
+src/sitemapxml.egg-info/dependency_links.txt
+src/sitemapxml.egg-info/entry_points.txt
+src/sitemapxml.egg-info/requires.txt
+src/sitemapxml.egg-info/top_level.txt

sitemapxml-0.1.0/src/sitemapxml.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sitemapxml-0.1.0/src/sitemapxml.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sitemapxml = sitemapxml.cli:main

sitemapxml-0.1.0/src/sitemapxml.egg-info/requires.txt

@@ -0,0 +1,2 @@
+requests
+beautifulsoup4

sitemapxml-0.1.0/src/sitemapxml.egg-info/top_level.txt

@@ -0,0 +1 @@
+sitemapxml