web-scanner 2.1.4__tar.gz → 2.1.5__tar.gz

{web_scanner-2.1.4 → web_scanner-2.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-scanner
-Version: 2.1.4
+Version: 2.1.5
 Summary: An async Textual TUI for website reconnaissance — DNS, WHOIS, TLS, security, tech-stack and SEO, with no paid APIs.
 Project-URL: Homepage, https://github.com/iamramizk/web-scanner
 Project-URL: Repository, https://github.com/iamramizk/web-scanner
@@ -55,7 +55,7 @@ Description-Content-Type: text/markdown
 _An async Textual TUI for website reconnaissance — DNS, WHOIS, TLS, security, tech-stack and SEO, with no paid APIs._
 
 [![Python](https://img.shields.io/badge/Python-3.11%2B-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/downloads/)
-[![PyPI Downloads](https://img.shields.io/pypi/dm/web-scanner?style=for-the-badge&logo=pypi&logoColor=white&color=006DAD)](https://pypi.org/project/web-scanner/)
+[![PyPI Downloads](https://img.shields.io/pepy/dt/web-scanner?style=for-the-badge&logo=pypi&logoColor=white&color=006DAD)](https://pypi.org/project/web-scanner/)
 [![Release](https://img.shields.io/github/v/release/iamramizk/web-scanner?style=for-the-badge&logo=github&color=2EA44F)](https://github.com/iamramizk/web-scanner/releases/latest)
 
 </div>

{web_scanner-2.1.4 → web_scanner-2.1.5}/README.md

@@ -5,7 +5,7 @@
 _An async Textual TUI for website reconnaissance — DNS, WHOIS, TLS, security, tech-stack and SEO, with no paid APIs._
 
 [![Python](https://img.shields.io/badge/Python-3.11%2B-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/downloads/)
-[![PyPI Downloads](https://img.shields.io/pypi/dm/web-scanner?style=for-the-badge&logo=pypi&logoColor=white&color=006DAD)](https://pypi.org/project/web-scanner/)
+[![PyPI Downloads](https://img.shields.io/pepy/dt/web-scanner?style=for-the-badge&logo=pypi&logoColor=white&color=006DAD)](https://pypi.org/project/web-scanner/)
 [![Release](https://img.shields.io/github/v/release/iamramizk/web-scanner?style=for-the-badge&logo=github&color=2EA44F)](https://github.com/iamramizk/web-scanner/releases/latest)
 
 </div>

{web_scanner-2.1.4 → web_scanner-2.1.5}/webscanner/__init__.py

@@ -1,3 +1,3 @@
 """web-scanner v2 — async website reconnaissance TUI package."""
 
-__version__ = "2.1.4"
+__version__ = "2.1.5"

{web_scanner-2.1.4 → web_scanner-2.1.5}/webscanner/modules/seo.py

@@ -39,11 +39,38 @@ SOCIAL_DOMAINS = (
 )
 
 _STOPWORDS = frozenset(
-    "the a an and or but if of to in on for with as by at from is are was were be "
-    "been being this that these those it its you your we our us they their he she "
-    "his her not no so do does did has have had can will would should could may "
-    "might just than then there here what which who when where how all any some "
-    "more most other into out up down over under about also new get one two".split()
+    (
+        # core articles / conjunctions / prepositions / pronouns
+        "the a an and or but if of to in on for with as by at from is are was were be "
+        "been being this that these those it its you your we our us they their he she "
+        "his her not no so do does did has have had can will would should could may "
+        "might just than then there here what which who when where how all any some "
+        "more most other into out up down over under about also new get one two "
+        # additional pronouns / possessives
+        "him them myself yourself himself herself itself ourselves themselves "
+        "whom whose mine yours ours hers theirs "
+        # modals / auxiliaries
+        "shall must cannot ought need used "
+        # conjunctions / subordinators
+        "nor yet because while since until unless although though whether whereas "
+        "whenever wherever whoever whatever "
+        # prepositions
+        "off per via upon above below beneath between among amongst throughout during "
+        "before after against without within along across behind beyond near toward "
+        "towards onto "
+        # determiners / quantifiers
+        "each every either neither both few many much such own same only very too quite "
+        "rather enough none another several "
+        # common adverbs / fillers
+        "back even still again ever never always often once now thus hence therefore "
+        "however moreover otherwise indeed perhaps maybe actually really simply almost "
+        "already else yes why well like "
+        # contractions (the tokenizer keeps apostrophes, so these survive verbatim)
+        "don't doesn't didn't isn't aren't wasn't weren't won't wouldn't can't couldn't "
+        "shouldn't hasn't haven't hadn't i'm i've i'll i'd you're you've you'll you'd "
+        "we're we've we'll we'd they're they've they'll it's he's she's that's there's "
+        "here's what's who's let's"
+    ).split()
 )
 
 
@@ -99,6 +126,18 @@ def _len_line(text: str, lo: int, hi: int) -> str:
     return f"[{MUTED}]{escape(text)}[/]\n[{colour}]{n} chars · rec. {lo}-{hi}[/]"
 
 
+def _text(el) -> str:
+    """Element text as the browser's `textContent` sees it, whitespace collapsed.
+
+    `get_text(strip=True)` strips each child text node then joins with no
+    separator, so words split across spans/newlines fuse ("One <span>Two</span>
+    <span>Three</span>" -> "OneTwoThree"). Taking the raw text and re-joining on
+    whitespace keeps the gaps that actually exist between nodes without inventing
+    any where the DOM has none (e.g. `<br>`).
+    """
+    return " ".join(el.get_text().split())
+
+
 class SeoModule(ScanModule):
     name = "seo"
     label = "SEO"
@@ -141,7 +180,7 @@ class SeoModule(ScanModule):
 
         # --- Content ---
         title_el = soup.find("title")
-        title = title_el.get_text(strip=True) if title_el else None
+        title = _text(title_el) if title_el else None
         desc_el = soup.find("meta", attrs={"name": "description"})
         desc = (desc_el.get("content") or "").strip() if desc_el else None
         content: dict[str, object] = {
@@ -149,7 +188,7 @@ class SeoModule(ScanModule):
             "Description": _len_line(desc, *DESC_RANGE) if desc else "-",
         }
         for lvl in range(1, 4):
-            content[f"H{lvl}"] = [h.get_text(strip=True) for h in soup.find_all(f"h{lvl}")]
+            content[f"H{lvl}"] = [_text(h) for h in soup.find_all(f"h{lvl}")]
         content["Socials"] = sorted({
             tag["href"].rstrip("/")
             for tag in soup.find_all("a", href=True)