seoextract 0.1.0__py3-none-any.whl

seoextract/__init__.py

@@ -0,0 +1,3 @@
+from .init import SEOExtract
+
+__all__ = ["SEOExtract"]

seoextract/crawler.py

@@ -0,0 +1,148 @@
+import time
+import requests
+from urllib.parse import urljoin, urlparse
+from urllib.robotparser import RobotFileParser
+from bs4 import BeautifulSoup
+
+
+HEADERS = {
+    "User-Agent": "SEOExtractHF/1.0 (SEO Audit Bot; +https://github.com/Britto1221/seoextracthf)"
+}
+
+TIMEOUT = 10  # seconds per request
+
+
+# ── Robots.txt Checker ────────────────────────────────────────────────────────
+
+def _load_robots(base_url: str) -> RobotFileParser | None:
+    """
+    Returns a RobotFileParser if robots.txt is reachable and returns 200.
+    Returns None if unreachable or blocked — caller treats None as allow-all.
+    """
+    robots_url = urljoin(base_url, "/robots.txt")
+    try:
+        resp = requests.get(robots_url, headers=HEADERS, timeout=5)
+        if resp.status_code != 200:
+            return None   # can't read robots.txt → allow everything
+        rp = RobotFileParser()
+        rp.set_url(robots_url)
+        rp.read()
+        return rp
+    except Exception:
+        return None
+
+
+# ── Single Page Fetch ─────────────────────────────────────────────────────────
+
+def fetch_page(url: str, session: requests.Session) -> dict:
+    """
+    Fetch a single URL.
+    Returns: { url, status_code, html, response_time_ms, final_url }
+    On failure returns status_code 0 and empty html.
+    """
+    try:
+        start = time.time()
+        response = session.get(url, headers=HEADERS, timeout=TIMEOUT, allow_redirects=True)
+        elapsed = (time.time() - start) * 1000  # ms
+
+        content_type = response.headers.get("Content-Type", "")
+        html = response.text if "text/html" in content_type else ""
+
+        return {
+            "url"             : url,
+            "final_url"       : response.url,
+            "status_code"     : response.status_code,
+            "html"            : html,
+            "response_time_ms": round(elapsed, 2),
+        }
+
+    except requests.exceptions.Timeout as e:
+        return {
+            "url": url,
+            "final_url": url,
+            "status_code": 408,
+            "html": "",
+            "response_time_ms": 0.0,
+            "error": str(e),
+        }
+    except requests.exceptions.TooManyRedirects:
+        return {"url": url, "final_url": url, "status_code": 310, "html": "", "response_time_ms": 0.0}
+    except requests.exceptions.ConnectionError:
+        return {"url": url, "final_url": url, "status_code": 0,   "html": "", "response_time_ms": 0.0}
+    except Exception:
+        return {"url": url, "final_url": url, "status_code": 0,   "html": "", "response_time_ms": 0.0}
+
+
+# ── Internal Link Extractor ───────────────────────────────────────────────────
+
+def _extract_internal_links(html: str, base_url: str) -> list[str]:
+    """Extract all internal hrefs from a page's HTML."""
+    soup = BeautifulSoup(html, "lxml")
+    base_domain = urlparse(base_url).netloc
+    links = []
+
+    for tag in soup.find_all("a", href=True):
+        href = tag["href"].strip()
+        if not href or href.startswith("#") or href.startswith("mailto:") or href.startswith("tel:"):
+            continue
+
+        full_url = urljoin(base_url, href)
+        parsed   = urlparse(full_url)
+
+        # Keep only http/https same-domain links
+        if parsed.scheme in ("http", "https") and parsed.netloc == base_domain:
+            # Normalize: remove fragment
+            clean = parsed._replace(fragment="").geturl()
+            links.append(clean)
+
+    return list(set(links))
+
+
+# ── BFS Crawler ───────────────────────────────────────────────────────────────
+
+def crawl(seed_url: str, max_pages: int = 20) -> list[dict]:
+    """
+    BFS crawl starting from seed_url.
+    Returns list of fetch results (one dict per page).
+    Respects robots.txt and max_pages cap.
+    """
+    # Normalize seed
+    parsed_seed = urlparse(seed_url)
+
+    if not parsed_seed.scheme:
+        seed_url = "https://" + seed_url
+
+    parsed_seed = urlparse(seed_url)
+    base_url = f"{parsed_seed.scheme}://{parsed_seed.netloc}"
+
+    robots    = _load_robots(base_url)
+    session   = requests.Session()
+    visited   = set()
+    queue     = [seed_url]
+    results   = []
+
+    while queue and len(results) < max_pages:
+        url = queue.pop(0)
+
+        # Skip already visited
+        if url in visited:
+            continue
+        visited.add(url)
+
+        # Respect robots.txt (None means unreachable → allow all)
+        if robots is not None and not robots.can_fetch(HEADERS["User-Agent"], url):
+            continue
+
+        # Fetch the page
+        result = fetch_page(url, session)
+        results.append(result)
+
+        # Only follow links from successful HTML pages
+        if result["status_code"] == 200 and result["html"]:
+            new_links = _extract_internal_links(result["html"], base_url)
+            for link in new_links:
+                if link not in visited and link not in queue:
+                    queue.append(link)
+
+    session.close()
+    return results

seoextract/init.py

@@ -0,0 +1,84 @@
+from datetime import datetime
+from .crawler import crawl
+from .parser  import parse
+from .rules   import detect_issues
+from .scorer  import score_site
+from .models  import AuditResult, Severity,SafeBrowsingResult
+from .safe_browsing import check_safe_browsing
+
+
+class SEOExtract:
+
+    @staticmethod
+    def audit(
+        url: str,
+        max_pages: int = 20,
+        safe_browsing_api_key: str | None = None,
+    ) -> AuditResult:
+        """
+        Main entry point.
+
+        Usage:
+            from seoextracthf import SEOExtract
+            result = SEOExtract.audit("https://example.com")
+
+        Returns an AuditResult object with:
+            result.site_score
+            result.grade
+            result.pages        → list of PageData
+            result.issues       → list of SEOIssue
+        """
+        safe_result = check_safe_browsing(url, api_key=safe_browsing_api_key)
+        safe_browsing = SafeBrowsingResult(**safe_result)
+
+        if safe_browsing.is_safe is False:
+            return AuditResult(
+                url=url,
+                audit_date=datetime.now().strftime("%Y-%m-%d %H:%M"),
+                pages_crawled=0,
+                site_score=0.0,
+                grade="F",
+                total_issues=0,
+                critical_count=0,
+                warning_count=0,
+                info_count=0,
+                pages=[],
+                issues=[],
+                safe_browsing=safe_browsing,
+            )
+        # 1. Crawl
+        raw_pages = crawl(url, max_pages=max_pages)
+
+        # 2. Parse each page
+        pages = [parse(r) for r in raw_pages]
+
+        # 3. Detect issues
+        issues = detect_issues(pages)
+
+        # 4. Score
+        site_score, grade = score_site(pages, issues)
+
+        # 5. Count by severity
+        critical = sum(1 for i in issues if i.severity == Severity.CRITICAL)
+        warning  = sum(1 for i in issues if i.severity == Severity.WARNING)
+        info     = sum(1 for i in issues if i.severity == Severity.INFO)
+
+        return AuditResult(
+            url           = url,
+            audit_date    = datetime.now().strftime("%Y-%m-%d %H:%M"),
+            pages_crawled = len(pages),
+            site_score    = site_score,
+            grade         = grade,
+            total_issues  = len(issues),
+            critical_count= critical,
+            warning_count = warning,
+            info_count    = info,
+            pages         = pages,
+            issues        = issues,
+            safe_browsing=safe_browsing,
+        )
+    
+
+if __name__ == "__main__":
+    result = SEOExtract.audit("https://rootpro.in/")
+    print(result.model_dump_json(indent=2))

seoextract/models.py

@@ -0,0 +1,116 @@
+from pydantic import BaseModel, HttpUrl
+from typing import Optional
+from enum import Enum
+
+
+# ── Severity Levels ──────────────────────────────────────────────────────────
+
+class Severity(str, Enum):
+    CRITICAL = "CRITICAL"
+    WARNING  = "WARNING"
+    INFO     = "INFO"
+
+class SafeBrowsingResult(BaseModel):
+    is_safe: bool | None
+    threats: list[str] = []
+    error: str | None = None
+# ── Issue Types ───────────────────────────────────────────────────────────────
+
+class IssueType(str, Enum):
+    MISSING_VIEWPORT = "Missing Viewport Meta Tag"
+    META_TOO_SHORT = "Meta Description Too Short"
+    MISSING_TITLE         = "Missing Title"
+    TITLE_TOO_SHORT       = "Title Too Short"
+    TITLE_TOO_LONG        = "Title Too Long"
+    DUPLICATE_TITLE       = "Duplicate Title"
+    MISSING_META          = "Missing Meta Description"
+    META_TOO_LONG         = "Meta Description Too Long"
+    MISSING_H1            = "Missing H1 Tag"
+    MULTIPLE_H1           = "Multiple H1 Tags"
+    THIN_CONTENT          = "Thin Content"
+    MISSING_ALT_TEXT      = "Missing Image Alt Text"
+    BROKEN_LINK           = "Broken Internal Link"
+    MISSING_CANONICAL     = "Missing Canonical Tag"
+    POOR_INTERNAL_LINKING = "Poor Internal Linking"
+    NO_SCHEMA             = "No Schema Markup"
+    MISSING_ROBOTS_META   = "Missing Robots Meta Tag"
+    DUPLICATE_META = "Duplicate Meta Description"
+
+
+# ── Single SEO Issue ──────────────────────────────────────────────────────────
+
+class SEOIssue(BaseModel):
+    page_url      : str
+    issue_type    : IssueType
+    severity      : Severity
+    current_value : str        # what was found (or empty string if missing)
+    suggestion    : str        # one-line hint for fixing it
+
+
+# ── Per-Page Extracted Data ───────────────────────────────────────────────────
+
+class PageData(BaseModel):
+    url              : str
+    status_code      : int
+    response_time_ms : float
+
+    # Title
+    title            : Optional[str] = None
+    title_length     : int = 0
+
+    # Meta
+    meta_description        : Optional[str] = None
+    meta_description_length : int = 0
+
+    # Robots + Canonical
+    robots_meta : Optional[str] = None   # e.g. "noindex, nofollow"
+    canonical   : Optional[str] = None
+
+    # Headings
+    h1_tags : list[str] = []
+    h2_tags : list[str] = []
+    h3_tags : list[str] = []
+    h1_count: int = 0
+    h2_count: int = 0
+    h3_count: int = 0
+
+    # Content
+    word_count: int = 0
+
+    # Images
+    total_images        : int = 0
+    images_missing_alt  : int = 0
+
+    # Links
+    internal_links : list[str] = []
+    external_links : list[str] = []
+    internal_count : int = 0
+    external_count : int = 0
+
+    # Schema
+    schema_found : bool = False
+
+    # Open Graph
+    og_title       : Optional[str] = None
+    og_description : Optional[str] = None
+
+    # Scoring
+    page_score : float = 0.0
+
+    final_url: Optional[str] = None
+
+    viewport: Optional[str] = None
+
+class AuditResult(BaseModel):
+    url           : str
+    audit_date    : str
+    pages_crawled : int
+    site_score    : float
+    grade         : str          # A / B / C / D / F
+    total_issues  : int
+    critical_count: int
+    warning_count : int
+    info_count    : int
+    pages         : list[PageData]
+    issues        : list[SEOIssue]
+    safe_browsing: SafeBrowsingResult

seoextract/parser.py

@@ -0,0 +1,140 @@
+import json
+from urllib.parse import urljoin, urlparse
+from bs4 import BeautifulSoup
+
+from .models import PageData
+
+
+def parse(fetch_result: dict) -> PageData:
+    """
+    Takes one fetch_result dict from crawler.
+    Returns a fully populated PageData object.
+    """
+    url              = fetch_result["url"]
+    final_url        = fetch_result.get("final_url", url)
+    status_code      = fetch_result["status_code"]
+    response_time_ms = fetch_result["response_time_ms"]
+    html             = fetch_result.get("html", "")
+
+    # Base for resolving relative URLs
+    parsed_base  = urlparse(final_url)
+    base_domain  = parsed_base.netloc
+    base_url     = f"{parsed_base.scheme}://{base_domain}"
+
+    # If no HTML (error page, non-HTML, timeout) return minimal PageData
+    if not html:
+        return PageData(
+            url=url,
+            status_code=status_code,
+            response_time_ms=response_time_ms,
+        )
+
+    soup = BeautifulSoup(html, "lxml")
+
+    # ── Title ─────────────────────────────────────────────────────────────────
+    title_tag    = soup.find("title")
+    title        = title_tag.get_text(strip=True) if title_tag else None
+    title_length = len(title) if title else 0
+
+    # ── Meta Description ──────────────────────────────────────────────────────
+    meta_tag = soup.find("meta", attrs={"name": lambda n: n and n.lower() == "description"})
+    meta_description        = meta_tag["content"].strip() if meta_tag and meta_tag.get("content") else None
+    meta_description_length = len(meta_description) if meta_description else 0
+
+    # ── Robots Meta ───────────────────────────────────────────────────────────
+    robots_tag  = soup.find("meta", attrs={"name": lambda n: n and n.lower() == "robots"})
+    robots_meta = robots_tag["content"].strip() if robots_tag and robots_tag.get("content") else None
+
+    viewport_tag = soup.find("meta", attrs={"name": lambda n: n and n.lower() == "viewport"})
+    viewport = viewport_tag["content"].strip() if viewport_tag and viewport_tag.get("content") else None
+
+    # ── Canonical ─────────────────────────────────────────────────────────────
+    canonical_tag = soup.find("link", attrs={"rel": lambda r: r and "canonical" in r})
+    canonical     = canonical_tag["href"].strip() if canonical_tag and canonical_tag.get("href") else None
+
+    # ── Headings ──────────────────────────────────────────────────────────────
+    h1_tags = [tag.get_text(strip=True) for tag in soup.find_all("h1")]
+    h2_tags = [tag.get_text(strip=True) for tag in soup.find_all("h2")]
+    h3_tags = [tag.get_text(strip=True) for tag in soup.find_all("h3")]
+
+    # ── Word Count ────────────────────────────────────────────────────────────
+    # Remove script and style tags before counting
+    for tag in soup(["script", "style", "noscript"]):
+        tag.decompose()
+    body_text  = soup.get_text(separator=" ", strip=True)
+    word_count = len(body_text.split())
+
+    # ── Images ────────────────────────────────────────────────────────────────
+    all_images        = soup.find_all("img")
+    total_images      = len(all_images)
+    images_missing_alt = sum(
+        1 for img in all_images
+        if not img.get("alt") or img["alt"].strip() == ""
+    )
+
+    # ── Links ─────────────────────────────────────────────────────────────────
+    internal_links = []
+    external_links = []
+
+    for a_tag in soup.find_all("a", href=True):
+        href = a_tag["href"].strip()
+        if not href or href.startswith("#") or href.startswith("mailto:") or href.startswith("tel:"):
+            continue
+
+        full_url = urljoin(final_url, href)
+        link_domain = urlparse(full_url).netloc
+
+        if link_domain == base_domain:
+            internal_links.append(full_url)
+        else:
+            external_links.append(full_url)
+
+    # Deduplicate
+    internal_links = list(set(internal_links))
+    external_links = list(set(external_links))
+
+    # ── Schema Markup ─────────────────────────────────────────────────────────
+    schema_found = False
+    for script_tag in soup.find_all("script", attrs={"type": "application/ld+json"}):
+        try:
+            data = json.loads(script_tag.string or "")
+            if data:
+                schema_found = True
+                break
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+    # ── Open Graph ────────────────────────────────────────────────────────────
+    og_title_tag       = soup.find("meta", attrs={"property": "og:title"})
+    og_description_tag = soup.find("meta", attrs={"property": "og:description"})
+    og_title           = og_title_tag["content"].strip() if og_title_tag and og_title_tag.get("content") else None
+    og_description     = og_description_tag["content"].strip() if og_description_tag and og_description_tag.get("content") else None
+
+    return PageData(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        response_time_ms=response_time_ms,
+        title=title,
+        title_length=title_length,
+        meta_description=meta_description,
+        meta_description_length=meta_description_length,
+        robots_meta=robots_meta,
+        canonical=canonical,
+        h1_tags=h1_tags,
+        h2_tags=h2_tags,
+        h3_tags=h3_tags,
+        h1_count=len(h1_tags),
+        h2_count=len(h2_tags),
+        h3_count=len(h3_tags),
+        word_count=word_count,
+        total_images=total_images,
+        images_missing_alt=images_missing_alt,
+        internal_links=internal_links,
+        external_links=external_links,
+        internal_count=len(internal_links),
+        external_count=len(external_links),
+        schema_found=schema_found,
+        og_title=og_title,
+        og_description=og_description,
+    )

seoextract/rules.py

@@ -0,0 +1,268 @@
+from .models import PageData, SEOIssue, IssueType, Severity
+
+TITLE_MIN        = 50
+TITLE_MAX        = 60
+META_MIN         = 50
+META_MAX         = 160
+THIN_CONTENT     = 300
+MIN_INTERNAL     = 2
+
+
+def _check_title(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if not page.title:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.MISSING_TITLE,
+            severity      = Severity.CRITICAL,
+            current_value = "",
+            suggestion    = "Add a descriptive <title> tag between 30–60 characters.",
+        ))
+    elif page.title_length < TITLE_MIN:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.TITLE_TOO_SHORT,
+            severity      = Severity.WARNING,
+            current_value = page.title,
+            suggestion = f"Title is {page.title_length} chars. Expand to at least {TITLE_MIN} characters.",
+        ))
+    elif page.title_length > TITLE_MAX:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.TITLE_TOO_LONG,
+            severity      = Severity.WARNING,
+            current_value = page.title,
+            suggestion    = f"Title is {page.title_length} chars. Trim to under {TITLE_MAX} characters.",
+        ))
+
+    return issues
+
+def _check_duplicate_meta_descriptions(pages: list[PageData]) -> list[SEOIssue]:
+    issues = []
+    seen = {}
+
+    for page in pages:
+        if not page.meta_description:
+            continue
+
+        meta_lower = page.meta_description.strip().lower()
+
+        if meta_lower in seen:
+            issues.append(SEOIssue(
+                page_url=page.url,
+                issue_type=IssueType.DUPLICATE_META,
+                severity=Severity.WARNING,
+                current_value=page.meta_description,
+                suggestion=f"This meta description duplicates '{seen[meta_lower]}'. Each page should have a unique meta description.",
+            ))
+        else:
+            seen[meta_lower] = page.url
+
+    return issues
+
+def _check_meta(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if not page.meta_description:
+        issues.append(SEOIssue(
+            page_url=page.url,
+            issue_type=IssueType.MISSING_META,
+            severity=Severity.WARNING,
+            current_value="",
+            suggestion="Add a meta description between 50–160 characters summarising the page.",
+        ))
+
+    elif page.meta_description_length < META_MIN:
+        issues.append(SEOIssue(
+            page_url=page.url,
+            issue_type=IssueType.META_TOO_SHORT,
+            severity=Severity.WARNING,
+            current_value=page.meta_description,
+            suggestion=f"Meta description is only {page.meta_description_length} chars. Expand to at least {META_MIN}.",
+        ))
+
+    elif page.meta_description_length > META_MAX:
+        issues.append(SEOIssue(
+            page_url=page.url,
+            issue_type=IssueType.META_TOO_LONG,
+            severity=Severity.WARNING,
+            current_value=page.meta_description,
+            suggestion=f"Meta description is {page.meta_description_length} chars. Trim to under {META_MAX}.",
+        ))
+
+    return issues
+
+
+def _check_headings(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if page.h1_count == 0:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.MISSING_H1,
+            severity      = Severity.CRITICAL,
+            current_value = "0 H1 tags found",
+            suggestion    = "Add exactly one <h1> tag that describes the main topic of this page.",
+        ))
+    elif page.h1_count > 1:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.MULTIPLE_H1,
+            severity      = Severity.WARNING,
+            current_value = f"{page.h1_count} H1 tags: {page.h1_tags}",
+            suggestion    = f"Reduce to one H1 tag. Found: {page.h1_tags}",
+        ))
+
+    return issues
+
+def _check_status_code(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if page.status_code >= 400 or page.status_code == 0:
+        issues.append(SEOIssue(
+            page_url=page.url,
+            issue_type=IssueType.BROKEN_LINK,
+            severity=Severity.CRITICAL,
+            current_value=str(page.status_code),
+            suggestion="Fix this page because it does not return a successful HTTP 200 response.",
+        ))
+
+    return issues
+
+def _check_content(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if page.word_count < THIN_CONTENT and page.status_code == 200:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.THIN_CONTENT,
+            severity      = Severity.WARNING,
+            current_value = f"{page.word_count} words",
+            suggestion    = f"Page has only {page.word_count} words. Aim for at least {THIN_CONTENT} words of meaningful content.",
+        ))
+
+    return issues
+
+
+def _check_images(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if page.images_missing_alt > 0:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.MISSING_ALT_TEXT,
+            severity      = Severity.WARNING,
+            current_value = f"{page.images_missing_alt} of {page.total_images} images missing alt text",
+            suggestion    = f"Add descriptive alt text to all {page.images_missing_alt} images missing it.",
+        ))
+
+    return issues
+
+
+def _check_canonical(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if not page.canonical and page.status_code == 200:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.MISSING_CANONICAL,
+            severity      = Severity.INFO,
+            current_value = "",
+            suggestion    = "Add a <link rel='canonical'> tag to prevent duplicate content issues.",
+        ))
+
+    return issues
+
+
+def _check_internal_linking(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if page.internal_count < MIN_INTERNAL and page.status_code == 200:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.POOR_INTERNAL_LINKING,
+            severity      = Severity.INFO,
+            current_value = f"{page.internal_count} internal links",
+            suggestion    = f"Add at least {MIN_INTERNAL} internal links to help search engines discover related pages.",
+        ))
+
+    return issues
+
+
+def _check_schema(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if not page.schema_found and page.status_code == 200:
+        issues.append(SEOIssue(
+            page_url      = page.url,
+            issue_type    = IssueType.NO_SCHEMA,
+            severity      = Severity.INFO,
+            current_value = "No JSON-LD schema found",
+            suggestion    = "Add Schema.org structured data (JSON-LD) to improve search result appearance.",
+        ))
+
+    return issues
+
+
+# ── Duplicate Title Check (site-level) ───────────────────────────────────────
+
+def _check_duplicate_titles(pages: list[PageData]) -> list[SEOIssue]:
+    issues    = []
+    seen      = {}  # title → first page url
+
+    for page in pages:
+        if not page.title:
+            continue
+        title_lower = page.title.strip().lower()
+        if title_lower in seen:
+            issues.append(SEOIssue(
+                page_url      = page.url,
+                issue_type    = IssueType.DUPLICATE_TITLE,
+                severity      = Severity.CRITICAL,
+                current_value = page.title,
+                suggestion    = f"This title duplicates '{seen[title_lower]}'. Each page needs a unique title.",
+            ))
+        else:
+            seen[title_lower] = page.url
+
+    return issues
+
+def _check_viewport(page: PageData) -> list[SEOIssue]:
+    issues = []
+
+    if not page.viewport and page.status_code == 200:
+        issues.append(SEOIssue(
+            page_url=page.url,
+            issue_type=IssueType.MISSING_VIEWPORT,
+            severity=Severity.WARNING,
+            current_value="",
+            suggestion="Add a viewport meta tag for mobile responsiveness.",
+        ))
+
+    return issues
+
+def detect_issues(pages: list[PageData]) -> list[SEOIssue]:
+    all_issues = []
+
+    for page in pages:
+        all_issues.extend(_check_status_code(page))
+
+        # skip SEO checks if page failed
+        if page.status_code != 200:
+            continue
+
+        all_issues.extend(_check_title(page))
+        all_issues.extend(_check_meta(page))
+        all_issues.extend(_check_headings(page))
+        all_issues.extend(_check_content(page))
+        all_issues.extend(_check_images(page))
+        all_issues.extend(_check_canonical(page))
+        all_issues.extend(_check_internal_linking(page))
+        all_issues.extend(_check_schema(page))
+        all_issues.extend(_check_duplicate_titles(pages))
+        all_issues.extend(_check_duplicate_meta_descriptions(pages))
+        all_issues.extend(_check_duplicate_titles(pages))
+        all_issues.extend(_check_viewport(page))
+
+    return all_issues

seoextract/safe_browsing.py

@@ -0,0 +1,61 @@
+import os
+import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def check_safe_browsing(url: str, api_key: str | None = None) -> dict:
+    # manual key has first priority
+    if api_key is None:
+        api_key = os.getenv("GOOGLE_SAFE_BROWSING_API_KEY")
+
+    if not api_key:
+        return {
+            "is_safe": None,
+            "threats": [],
+            "error": "Google Safe Browsing API key not provided and GOOGLE_SAFE_BROWSING_API_KEY not found in .env",
+        }
+
+    endpoint = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={api_key}"
+
+    payload = {
+        "client": {
+            "clientId": "seoextracthf",
+            "clientVersion": "1.0.0",
+        },
+        "threatInfo": {
+            "threatTypes": [
+                "MALWARE",
+                "SOCIAL_ENGINEERING",
+                "UNWANTED_SOFTWARE",
+                "POTENTIALLY_HARMFUL_APPLICATION",
+            ],
+            "platformTypes": ["ANY_PLATFORM"],
+            "threatEntryTypes": ["URL"],
+            "threatEntries": [{"url": url}],
+        },
+    }
+
+    try:
+        response = requests.post(endpoint, json=payload, timeout=10)
+        response.raise_for_status()
+        data = response.json()
+
+        matches = data.get("matches", [])
+
+        if not matches:
+            return {"is_safe": True, "threats": [], "error": None}
+
+        return {
+            "is_safe": False,
+            "threats": [match.get("threatType") for match in matches],
+            "error": None,
+        }
+
+    except requests.RequestException as e:
+        return {
+            "is_safe": None,
+            "threats": [],
+            "error": str(e),
+        }

seoextract/scorer.py

@@ -0,0 +1,48 @@
+from .models import PageData, SEOIssue, Severity
+
+
+# ── Severity Penalty Weights ──────────────────────────────────────────────────
+
+PENALTY = {
+    Severity.CRITICAL : 20,
+    Severity.WARNING  : 8,
+    Severity.INFO     : 3,
+}
+
+
+def _grade(score: float) -> str:
+    if score >= 90: return "A"
+    if score >= 75: return "B"
+    if score >= 60: return "C"
+    if score >= 40: return "D"
+    return "F"
+
+
+def score_page(page: PageData, issues: list[SEOIssue]) -> float:
+    """
+    Calculate a 0–100 score for a single page.
+    Starts at 100, deducts points per issue by severity.
+    """
+    page_issues = [i for i in issues if i.page_url == page.url]
+    score = 100.0
+    if page.status_code != 200:
+        return 0.0
+    for issue in page_issues:
+        score -= PENALTY[issue.severity]
+
+    return round(max(score, 0.0), 1)
+
+
+def score_site(pages: list[PageData], issues: list[SEOIssue]) -> tuple[float, str]:
+    """
+    Calculate site-level score and grade.
+    Returns: (site_score, grade)
+    """
+    if not pages:
+        return 0.0, "F"
+
+    for page in pages:
+        page.page_score = score_page(page, issues)
+
+    site_score = round(sum(p.page_score for p in pages) / len(pages), 1)
+    return site_score, _grade(site_score)

seoextract-0.1.0.dist-info/METADATA

@@ -0,0 +1,294 @@
+Metadata-Version: 2.4
+Name: seoextract
+Version: 0.1.0
+Summary: A lightweight Python SEO audit engine that returns Pydantic structured output.
+Author: Britto K
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.31.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+
+# SEOExtractHF
+
+<div align="center">
+
+**A lightweight Python SEO audit engine with built-in Google Safe Browsing support.**
+
+Returns validated **Pydantic structured output** that can be directly consumed by AI agents, dashboards, APIs, report generators, and automation pipelines.
+
+</div>
+
+---
+
+## Features
+
+- Website crawler
+- Google Safe Browsing validation
+- Technical SEO auditing
+- Pydantic structured output
+- Page-level SEO metrics
+- Site-level SEO scoring
+- Severity-based issue detection
+- Duplicate title detection
+- Duplicate meta description detection
+- Canonical tag detection
+- Viewport detection
+- Schema.org detection
+- Image alt-text validation
+- Internal linking analysis
+- Thin content detection
+
+---
+
+# Installation
+
+```bash
+pip install seoextracthf
+```
+
+or install from source
+
+```bash
+pip install -e .
+```
+
+---
+
+# Requirements
+
+- Python 3.10+
+- Google Safe Browsing API Key
+
+---
+
+# Google Safe Browsing Setup
+
+SEOExtractHF checks every website against Google's Safe Browsing service **before crawling**.
+
+If Google reports the website as unsafe, crawling is stopped automatically.
+
+If no Google Safe Browsing API key is provided, safe_browsing.is_safe will be None.
+---
+
+## Option 1 (Recommended)
+
+Create a `.env` file.
+
+```text
+.env
+```
+
+Add your API key.
+
+```env
+GOOGLE_SAFE_BROWSING_API_KEY=YOUR_API_KEY
+```
+
+SEOExtractHF automatically loads the API key.
+
+No additional code is required.
+
+---
+
+## Option 2
+
+Pass the API key manually.
+
+```python
+from seoextracthf import SEOExtract
+
+result = SEOExtract.audit(
+    "https://example.com",
+    safe_browsing_api_key="YOUR_API_KEY"
+)
+```
+
+When an API key is supplied manually, the `.env` file is **not used**.
+
+---
+
+# Quick Start
+
+```python
+from seoextracthf import SEOExtract
+
+result = SEOExtract.audit(
+    "https://example.com"
+)
+
+print(result.model_dump_json(indent=2))
+```
+
+---
+
+# Returned Object
+
+SEOExtractHF returns a validated Pydantic model.
+
+```text
+AuditResult
+│
+├── url
+├── audit_date
+├── pages_crawled
+├── site_score
+├── grade
+├── total_issues
+├── critical_count
+├── warning_count
+├── info_count
+├── pages
+├── issues
+└── safe_browsing
+```
+
+---
+
+# Example
+
+```python
+from seoextracthf import SEOExtract
+
+result = SEOExtract.audit("https://example.com")
+
+print(result.site_score)
+print(result.grade)
+print(result.safe_browsing)
+
+for issue in result.issues:
+    print(issue.issue_type)
+```
+
+---
+
+# Safe Browsing Result
+
+```python
+{
+    "is_safe": True,
+    "threats": [],
+    "error": None
+}
+```
+
+If Google reports a threat:
+
+```python
+{
+    "is_safe": False,
+    "threats": [
+        "MALWARE"
+    ],
+    "error": None
+}
+```
+
+SEOExtractHF immediately stops crawling unsafe websites.
+
+---
+
+# Current SEO Checks
+
+## Page Quality
+
+- Title validation
+- Meta description validation
+- H1 validation
+- Thin content detection
+
+## Technical SEO
+
+- Canonical tag
+- Viewport meta tag
+- Schema.org JSON-LD
+- HTTP status validation
+
+## Images
+
+- Missing ALT attributes
+
+## Links
+
+- Internal link analysis
+
+## Site-wide Checks
+
+- Duplicate titles
+- Duplicate meta descriptions
+
+## Security
+
+- Google Safe Browsing validation
+
+---
+
+# Example Output
+
+```python
+AuditResult(
+    site_score=91.0,
+    grade="A",
+    total_issues=4,
+    pages_crawled=15
+)
+```
+
+---
+
+# Project Structure
+
+```
+seoextracthf/
+│
+├── crawler.py
+├── parser.py
+├── rules.py
+├── scorer.py
+├── safe_browsing.py
+├── models.py
+└── __init__.py
+```
+
+---
+
+# Designed For
+
+SEOExtractHF is designed to be used inside:
+
+- AI SEO Agents
+- LangGraph workflows
+- FastAPI applications
+- Streamlit dashboards
+- Report generators
+- CI/CD quality checks
+- Data pipelines
+- SEO automation tools
+
+---
+
+# Dependencies
+
+- beautifulsoup4
+- lxml
+- requests
+- pydantic
+- python-dotenv
+
+---
+
+# License
+
+MIT License
+
+---
+
+# Author
+
+**Britto K**
+
+GitHub:
+
+https://github.com/Britto1221# seoextract

seoextract-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+seoextract/__init__.py,sha256=mYxMjSFt3xUJBhQ2vS_CQ6i-i7EKAr5O5t4RWEjj22Y,56
+seoextract/crawler.py,sha256=-oCnPiPHDKpJiLajRWn8OPqJzgPNkGzJu8AFzO_GKYg,5528
+seoextract/init.py,sha256=5T64sdKW4mJmVz-WziebfOYuX4bofuc9af0GHdXAFS4,2731
+seoextract/models.py,sha256=dtxV9VPiZFzWJiQJMLtKkU-3fMnvXxMWsfwva8q67qM,3912
+seoextract/parser.py,sha256=FqazLOWNA-RhEpHfjUZytO8d1RaMYUV_TuggChhDO1c,7120
+seoextract/rules.py,sha256=yqmb12HT722UsXgHJAxaljzMiyTljWNB3maXB93qkl0,9521
+seoextract/safe_browsing.py,sha256=WiBCM_NuOhpS_xHsO0QLFuO740rMa8okfgSwlxZhKC4,1726
+seoextract/scorer.py,sha256=eOavcwXJbIJRPO9S85f65bZAG2xu1G39dXnyIdeOP60,1394
+seoextract-0.1.0.dist-info/METADATA,sha256=fkeKaE4JmHwXjCfTqLCCuY5W4X6P3QCILBclJ4VZPyg,4609
+seoextract-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+seoextract-0.1.0.dist-info/top_level.txt,sha256=YuTAdTr8NcUNo7p5ygQtcRd42rbDqR0KdFcSUyUuGrw,11
+seoextract-0.1.0.dist-info/RECORD,,

seoextract-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

seoextract-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+seoextract