leadintel 0.3.0__py3-none-any.whl

leadintel/services/local_ai.py

@@ -0,0 +1,186 @@
+"""
+Local AI Service via Ollama.
+
+Communicates with locally running Ollama instances (default http://localhost:11434).
+Strictly local-first:
+- Enforces local loopback interface (localhost / 127.0.0.1 / ::1). Rejects external/cloud hosts.
+- Bypasses ambient environment proxies (trust_env=False) to prevent data exfiltration.
+- Rejects HTTP redirects (follow_redirects=False).
+- Never sends business data to external/cloud services.
+- Never falls back to cloud AI.
+- Handles Ollama offline, uninstalled, or missing model gracefully.
+- Does not download models automatically.
+- Enforces request timeouts and JSON structured responses.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from ..config import settings
+from .url_validator import SecurityValidationError
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_TIMEOUT = 30.0
+
+LOCAL_LOOPBACK_HOSTS = {"localhost", "localhost.localdomain", "127.0.0.1", "::1", "0.0.0.0"}
+
+
+class OllamaError(Exception):
+    """Base exception for Ollama interactions."""
+
+
+class OllamaUnavailableError(OllamaError):
+    """Raised when the Ollama service is unreachable or not running."""
+
+
+class OllamaModelNotFoundError(OllamaError):
+    """Raised when the specified model is not loaded in Ollama."""
+
+
+class LocalAIClient:
+    """HTTP client interacting strictly with locally running Ollama API."""
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        model: Optional[str] = None,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        raw_url = (base_url or settings.ollama_base_url).rstrip("/")
+        parsed = urlparse(raw_url)
+        host = (parsed.hostname or "").lower().strip("[]")
+
+        # Guarantee local-first privacy: reject external cloud or remote endpoints
+        if host and host not in LOCAL_LOOPBACK_HOSTS:
+            raise SecurityValidationError(
+                f"Ollama endpoint '{raw_url}' is not a local loopback address. "
+                "LeadIntel strictly enforces that Ollama must run on the local machine "
+                "(localhost / 127.0.0.1 / ::1) to ensure local-first privacy and prevent data exfiltration."
+            )
+
+        self.base_url = raw_url
+        self.model = model or settings.ollama_model
+        self.timeout = timeout
+
+    async def is_available(self, client: Optional[httpx.AsyncClient] = None) -> bool:
+        """Check if local Ollama server is running and reachable."""
+        endpoint = f"{self.base_url}/api/tags"
+        should_close = False
+        if client is None:
+            client = httpx.AsyncClient(timeout=3.0, trust_env=False, follow_redirects=False)
+            should_close = True
+        try:
+            resp = await client.get(endpoint)
+            return resp.status_code == 200
+        except Exception:
+            return False
+        finally:
+            if should_close:
+                await client.aclose()
+
+    async def list_models(self, client: Optional[httpx.AsyncClient] = None) -> list[str]:
+        """List locally available models in Ollama."""
+        endpoint = f"{self.base_url}/api/tags"
+        should_close = False
+        if client is None:
+            client = httpx.AsyncClient(timeout=5.0, trust_env=False, follow_redirects=False)
+            should_close = True
+        try:
+            resp = await client.get(endpoint)
+            if resp.status_code != 200:
+                raise OllamaUnavailableError(f"Ollama returned HTTP {resp.status_code}")
+            data = resp.json()
+            models = [m.get("name", "") for m in data.get("models", [])]
+            return [m for m in models if m]
+        except (httpx.ConnectError, httpx.ConnectTimeout) as e:
+            raise OllamaUnavailableError(
+                f"Local Ollama server is not running at {self.base_url}. Start it with 'ollama serve'."
+            ) from e
+        except Exception as e:
+            if isinstance(e, OllamaError):
+                raise
+            raise OllamaError(f"Failed to query Ollama models: {e}") from e
+        finally:
+            if should_close:
+                await client.aclose()
+
+    async def check_model_installed(self, model_name: Optional[str] = None, client: Optional[httpx.AsyncClient] = None) -> bool:
+        """Check if a specific model or the default model is installed locally."""
+        target = model_name or self.model
+        available = await self.list_models(client=client)
+        # Match exact name or without tag (e.g. 'llama3:latest' matches 'llama3')
+        return any(m == target or m.split(":")[0] == target for m in available)
+
+    async def generate(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        model: Optional[str] = None,
+        format_json: bool = False,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> str:
+        """
+        Generate completion from local Ollama model.
+
+        Args:
+            prompt: User prompt.
+            system_prompt: Optional system instructions.
+            model: Specific model override.
+            format_json: If True, request JSON formatting from Ollama.
+            client: Optional HTTPX async client.
+        """
+        target_model = model or self.model
+        endpoint = f"{self.base_url}/api/generate"
+
+        payload: dict[str, Any] = {
+            "model": target_model,
+            "prompt": prompt,
+            "stream": False,
+        }
+        if system_prompt:
+            payload["system"] = system_prompt
+        if format_json:
+            payload["format"] = "json"
+
+        should_close = False
+        if client is None:
+            # Enforce trust_env=False to prevent proxy interception, and follow_redirects=False
+            client = httpx.AsyncClient(timeout=self.timeout, trust_env=False, follow_redirects=False)
+            should_close = True
+
+        try:
+            resp = await client.post(endpoint, json=payload)
+            if resp.status_code == 404:
+                raise OllamaModelNotFoundError(
+                    f"Model '{target_model}' not found in local Ollama. Install with 'ollama pull {target_model}'."
+                )
+            if resp.status_code != 200:
+                raise OllamaError(f"Ollama returned HTTP {resp.status_code}: {resp.text}")
+
+            data = resp.json()
+            response_text = data.get("response", "")
+            return response_text.strip()
+
+        except (httpx.ConnectError, httpx.ConnectTimeout) as e:
+            raise OllamaUnavailableError(
+                f"Local Ollama server is not running at {self.base_url}. Start it with 'ollama serve'."
+            ) from e
+        except httpx.ReadTimeout as e:
+            raise OllamaError(f"Ollama request timed out after {self.timeout}s.") from e
+        except Exception as e:
+            if isinstance(e, OllamaError):
+                raise
+            raise OllamaError(f"Ollama inference error: {e}") from e
+        finally:
+            if should_close:
+                await client.aclose()
+
+
+# Default singleton instance
+local_ai_client = LocalAIClient()

leadintel/services/robots_checker.py

@@ -0,0 +1,81 @@
+"""
+Robots.txt Parser & Checker Service.
+
+Checks robots.txt policies using standard python urllib.robotparser,
+with respectful timeouts, caching, and User-Agent matching.
+"""
+
+from __future__ import annotations
+
+import logging
+import urllib.robotparser
+from typing import Optional
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from .url_validator import SecurityValidationError, create_safe_client, validate_and_normalize_url
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_USER_AGENT = "LeadIntel/0.1 (Business Intelligence Researcher; +https://github.com/example/leadintel; respectful crawler)"
+_ROBOTS_CACHE: dict[str, tuple[float, urllib.robotparser.RobotFileParser]] = {}
+
+
+class RobotsChecker:
+    """Manages fetching, parsing, and caching robots.txt for websites."""
+
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout: float = 10.0):
+        self.user_agent = user_agent
+        self.timeout = timeout
+
+    async def can_fetch(self, target_url: str, client: Optional[httpx.AsyncClient] = None) -> tuple[bool, Optional[str]]:
+        """
+        Check if target_url can be crawled according to the site's robots.txt.
+
+        Returns:
+            (allowed: bool, reason_or_robots_url: Optional[str])
+        """
+        try:
+            norm_url = validate_and_normalize_url(target_url)
+        except SecurityValidationError as e:
+            return False, f"Blocked by URL security validation: {e}"
+        except Exception as e:
+            return False, f"Invalid URL: {e}"
+
+        parsed = urlparse(norm_url)
+        base_origin = f"{parsed.scheme}://{parsed.netloc}"
+        robots_url = urljoin(base_origin, "/robots.txt")
+
+        rp = urllib.robotparser.RobotFileParser()
+        rp.set_url(robots_url)
+
+        try:
+            should_close = False
+            if client is None:
+                client = create_safe_client(timeout=self.timeout)
+                should_close = True
+
+            try:
+                resp = await client.get(robots_url, headers={"User-Agent": self.user_agent})
+                if resp.status_code == 200:
+                    rp.parse(resp.text.splitlines())
+                elif resp.status_code in (401, 403):
+                    # Access denied to robots.txt usually implies entire site is disallowed or restricted
+                    return False, f"robots.txt returned HTTP {resp.status_code}"
+                else:
+                    # 404 or other errors typically mean crawling is allowed per RFC standards
+                    return True, None
+            finally:
+                if should_close:
+                    await client.aclose()
+
+            allowed = rp.can_fetch(self.user_agent, norm_url)
+            if not allowed:
+                return False, f"Disallowed by rules in {robots_url}"
+            return True, None
+
+        except Exception as e:
+            logger.debug("Failed to check robots.txt for %s: %s", robots_url, e)
+            # Default to allowed on fetch errors (unless server actively blocked with 401/403)
+            return True, None

leadintel/services/seo_auditor.py

@@ -0,0 +1,281 @@
+"""
+Custom Python SEO Audit Service.
+
+Performs deterministic, rule-based SEO evaluation without external paid APIs:
+- Title tag presence, length, and uniqueness.
+- Meta description presence and length.
+- H1 tags presence and uniqueness (detects zero or multiple H1s).
+- Image alt attributes (percentage of images with descriptive alt text).
+- Canonical link tag presence and format.
+- Meta robots tag checking (index/noindex, follow/nofollow).
+- Open Graph tags (og:title, og:description, og:image, og:url).
+- robots.txt and sitemap.xml presence.
+- Internal link counts and basic linking health.
+- HTML page byte size.
+
+Computes a transparent 0-100 composite score with actionable issues and recommendations.
+Note: Transparently communicates that this is an on-page technical audit, not a guarantee of Google ranking.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .url_validator import create_safe_client, validate_and_normalize_url
+
+logger = logging.getLogger(__name__)
+
+
+def audit_html_seo(html_content: str, page_url: str) -> dict[str, Any]:
+    """
+    Perform on-page SEO analysis on the HTML string of page_url.
+
+    Returns structured audit data including checks, score (0-100), issues, and recommendations.
+    """
+    score = 100
+    deductions = 0
+    issues: list[dict[str, str]] = []
+    recommendations: list[str] = []
+    checks: dict[str, Any] = {}
+
+    page_bytes = len(html_content.encode("utf-8"))
+    checks["html_size_bytes"] = page_bytes
+    if page_bytes > 2_000_000:  # 2MB
+        issues.append({"level": "warning", "message": f"HTML document size is large ({page_bytes / 1024:.1f} KB)."})
+        recommendations.append("Reduce page weight and inline assets to improve load time.")
+        deductions += 5
+
+    try:
+        soup = BeautifulSoup(html_content, "html.parser")
+    except Exception as e:
+        return {
+            "seo_score": 0,
+            "disclaimer": "This is a technical on-page SEO assessment and not a complete Google ranking guarantee.",
+            "checks": {"error": f"Failed to parse HTML: {e}"},
+            "issues": [{"level": "error", "message": "HTML could not be parsed."}],
+            "recommendations": ["Fix HTML syntax and structure."],
+        }
+
+    # 1. Title tag
+    title_tag = soup.find("title")
+    title_text = title_tag.get_text().strip() if title_tag else ""
+    checks["title"] = {
+        "present": bool(title_text),
+        "text": title_text,
+        "length": len(title_text),
+    }
+    if not title_text:
+        issues.append({"level": "error", "message": "Page is missing a <title> tag."})
+        recommendations.append("Add a concise, descriptive <title> tag (recommended 30-60 characters).")
+        deductions += 15
+    elif len(title_text) < 15:
+        issues.append({"level": "warning", "message": f"Title tag is very short ({len(title_text)} chars)."})
+        recommendations.append("Expand the title tag to provide clearer context for users and search engines.")
+        deductions += 5
+    elif len(title_text) > 70:
+        issues.append({"level": "warning", "message": f"Title tag may be truncated in search results ({len(title_text)} chars)."})
+        recommendations.append("Keep title tag under 60-70 characters.")
+        deductions += 3
+
+    # 2. Meta description
+    meta_desc = soup.find("meta", attrs={"name": lambda x: x and x.lower() == "description"})
+    desc_text = meta_desc.get("content", "").strip() if meta_desc else ""
+    checks["meta_description"] = {
+        "present": bool(desc_text),
+        "text": desc_text,
+        "length": len(desc_text),
+    }
+    if not desc_text:
+        issues.append({"level": "error", "message": "Page is missing a meta description."})
+        recommendations.append("Add a compelling meta description tag (recommended 120-160 characters).")
+        deductions += 15
+    elif len(desc_text) < 50:
+        issues.append({"level": "warning", "message": f"Meta description is short ({len(desc_text)} chars)."})
+        recommendations.append("Elaborate your meta description to better summarize page content.")
+        deductions += 5
+    elif len(desc_text) > 165:
+        issues.append({"level": "warning", "message": f"Meta description is long ({len(desc_text)} chars) and may be truncated."})
+        deductions += 2
+
+    # 3. Headings (H1 & H2)
+    h1_tags = soup.find_all("h1")
+    h1_texts = [h.get_text().strip() for h in h1_tags if h.get_text().strip()]
+    checks["h1_headings"] = {
+        "count": len(h1_tags),
+        "texts": h1_texts,
+    }
+    if len(h1_tags) == 0:
+        issues.append({"level": "error", "message": "No <h1> heading found on the page."})
+        recommendations.append("Include exactly one primary <h1> heading summarizing the main topic.")
+        deductions += 15
+    elif len(h1_tags) > 1:
+        issues.append({"level": "warning", "message": f"Multiple <h1> headings found ({len(h1_tags)})."})
+        recommendations.append("Consider using a single primary <h1> heading and structure secondary topics with <h2>.")
+        deductions += 5
+
+    h2_tags = soup.find_all("h2")
+    checks["h2_count"] = len(h2_tags)
+    if len(h2_tags) == 0 and len(html_content) > 1500:
+        issues.append({"level": "info", "message": "No <h2> subheadings found."})
+        recommendations.append("Use <h2> subheadings to divide text into readable sections.")
+        deductions += 3
+
+    # 4. Images & Alt attributes
+    images = soup.find_all("img")
+    total_imgs = len(images)
+    missing_alt = [img for img in images if not img.get("alt") or not img.get("alt").strip()]
+    checks["images"] = {
+        "total": total_imgs,
+        "missing_alt": len(missing_alt),
+        "alt_coverage_pct": round(((total_imgs - len(missing_alt)) / total_imgs * 100), 1) if total_imgs else 100.0,
+    }
+    if missing_alt:
+        issues.append({
+            "level": "warning",
+            "message": f"{len(missing_alt)} of {total_imgs} images are missing alt attributes."
+        })
+        recommendations.append("Add descriptive alt text to all informative images for accessibility and image search.")
+        deductions += min(15, len(missing_alt) * 3)
+
+    # 5. Canonical URL
+    canonical_tag = soup.find("link", rel=lambda x: x and "canonical" in x.lower())
+    canonical_href = canonical_tag.get("href", "").strip() if canonical_tag else ""
+    checks["canonical_url"] = {
+        "present": bool(canonical_href),
+        "url": canonical_href,
+    }
+    if not canonical_href:
+        issues.append({"level": "info", "message": "No canonical link element specified."})
+        recommendations.append("Specify a rel='canonical' link to prevent duplicate content issues.")
+        deductions += 5
+
+    # 6. Meta Robots
+    meta_robots = soup.find("meta", attrs={"name": lambda x: x and x.lower() == "robots"})
+    robots_content = meta_robots.get("content", "").strip() if meta_robots else ""
+    checks["meta_robots"] = {
+        "present": bool(robots_content),
+        "content": robots_content,
+        "is_noindex": "noindex" in robots_content.lower(),
+        "is_nofollow": "nofollow" in robots_content.lower(),
+    }
+    if "noindex" in robots_content.lower():
+        issues.append({"level": "warning", "message": "Page has 'noindex' directive in meta robots tag."})
+        deductions += 10
+
+    # 7. Open Graph metadata
+    og_title = soup.find("meta", property="og:title")
+    og_desc = soup.find("meta", property="og:description")
+    og_image = soup.find("meta", property="og:image")
+    checks["open_graph"] = {
+        "has_title": bool(og_title and og_title.get("content")),
+        "has_description": bool(og_desc and og_desc.get("content")),
+        "has_image": bool(og_image and og_image.get("content")),
+    }
+    if not (og_title and og_desc and og_image):
+        missing_og = []
+        if not og_title:
+            missing_og.append("og:title")
+        if not og_desc:
+            missing_og.append("og:description")
+        if not og_image:
+            missing_og.append("og:image")
+        issues.append({"level": "info", "message": f"Incomplete Open Graph tags: missing {', '.join(missing_og)}."})
+        recommendations.append("Add complete Open Graph meta tags (title, description, image) for social sharing previews.")
+        deductions += 5
+
+    # 8. Internal links
+    parsed_base = urlparse(page_url)
+    base_domain = parsed_base.netloc.lower()
+    all_links = soup.find_all("a", href=True)
+    internal_links: list[str] = []
+    external_links: list[str] = []
+
+    for a in all_links:
+        href = a["href"].strip()
+        if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+            continue
+        full_href = urljoin(page_url, href)
+        parsed_target = urlparse(full_href)
+        if parsed_target.netloc.lower() == base_domain:
+            internal_links.append(full_href)
+        elif parsed_target.netloc:
+            external_links.append(full_href)
+
+    checks["links"] = {
+        "total": len(all_links),
+        "internal_count": len(internal_links),
+        "external_count": len(external_links),
+    }
+    if len(internal_links) == 0:
+        issues.append({"level": "warning", "message": "No internal navigation links found on page."})
+        recommendations.append("Ensure key navigation links exist to facilitate crawl depth and user discovery.")
+        deductions += 5
+
+    final_score = max(0, min(100, score - deductions))
+
+    return {
+        "seo_score": final_score,
+        "disclaimer": (
+            "This technical SEO evaluation is based on on-page source standards "
+            "and does not represent a proprietary or full Google ranking algorithm result."
+        ),
+        "checks": checks,
+        "issues": issues,
+        "recommendations": recommendations,
+    }
+
+
+async def run_full_seo_audit(url: str, client: Optional[httpx.AsyncClient] = None) -> dict[str, Any]:
+    """
+    Fetch target page, check robots.txt and sitemap.xml, and perform complete SEO audit.
+    """
+    norm_url = validate_and_normalize_url(url)
+    should_close = False
+    if client is None:
+        client = create_safe_client(timeout=15.0)
+        should_close = True
+
+    try:
+        # Check robots.txt availability
+        parsed = urlparse(norm_url)
+        base_origin = f"{parsed.scheme}://{parsed.netloc}"
+        robots_url = urljoin(base_origin, "/robots.txt")
+        sitemap_url = urljoin(base_origin, "/sitemap.xml")
+
+        robots_ok = False
+        sitemap_ok = False
+
+        try:
+            r_resp = await client.get(robots_url, headers={"User-Agent": "LeadIntel/0.1"})
+            robots_ok = (r_resp.status_code == 200)
+        except Exception:
+            pass
+
+        try:
+            s_resp = await client.get(sitemap_url, headers={"User-Agent": "LeadIntel/0.1"})
+            sitemap_ok = (s_resp.status_code == 200)
+        except Exception:
+            pass
+
+        # Fetch main page
+        page_resp = await client.get(norm_url, headers={"User-Agent": "LeadIntel/0.1"})
+        page_resp.raise_for_status()
+
+        audit = audit_html_seo(page_resp.text, str(page_resp.url))
+        audit["checks"]["robots_txt"] = {"available": robots_ok, "url": robots_url}
+        audit["checks"]["sitemap"] = {"available": sitemap_ok, "url": sitemap_url}
+
+        if not robots_ok:
+            audit["issues"].append({"level": "info", "message": "robots.txt not detected at standard location."})
+        if not sitemap_ok:
+            audit["issues"].append({"level": "info", "message": "sitemap.xml not detected at standard root location."})
+
+        return audit
+    finally:
+        if should_close:
+            await client.aclose()

leadintel/services/social_finder.py

@@ -0,0 +1,119 @@
+"""
+Social Media Profile Finder Service.
+
+Extracts official social media links from HTML documents and handles
+filtering of share buttons, tracking parameters, and unrelated external links.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Optional
+from urllib.parse import urlparse, urlunparse
+
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+# Canonical platform regexes
+_PLATFORMS: dict[str, list[re.Pattern]] = {
+    "linkedin": [
+        re.compile(r"^https?://(www\.)?linkedin\.com/(company|in|school)/[a-zA-Z0-9_\-\.]+/?", re.IGNORECASE)
+    ],
+    "facebook": [
+        re.compile(r"^https?://(www\.)?facebook\.com/[a-zA-Z0-9_\-\.]+/?", re.IGNORECASE)
+    ],
+    "instagram": [
+        re.compile(r"^https?://(www\.)?instagram\.com/[a-zA-Z0-9_\-\.]+/?", re.IGNORECASE)
+    ],
+    "twitter": [
+        re.compile(r"^https?://(www\.)?(twitter|x)\.com/[a-zA-Z0-9_]{1,30}/?", re.IGNORECASE)
+    ],
+    "youtube": [
+        re.compile(r"^https?://(www\.)?youtube\.com/(@[a-zA-Z0-9_\-\.]+|channel/[a-zA-Z0-9_\-]+|c/[a-zA-Z0-9_\-]+)/?", re.IGNORECASE)
+    ],
+    "pinterest": [
+        re.compile(r"^https?://(www\.)?pinterest\.(com|[a-z]{2})/[a-zA-Z0-9_\-\.]+/?", re.IGNORECASE)
+    ],
+}
+
+# Paths/query parameters indicating sharing intent rather than profile links
+_IGNORE_PATTERNS = [
+    re.compile(r"sharer|share\.php|intent/tweet|sharing|dialog/share|pin/create", re.IGNORECASE),
+    re.compile(r"linkedin\.com/shareArticle", re.IGNORECASE),
+]
+
+
+def _clean_social_url(url: str) -> Optional[str]:
+    """Clean query parameters and check if URL is a profile vs sharing link."""
+    for pattern in _IGNORE_PATTERNS:
+        if pattern.search(url):
+            return None
+
+    try:
+        parsed = urlparse(url)
+        # Strip tracking query params
+        path = parsed.path.rstrip("/")
+        if not path or path in ("", "/"):
+            return None
+
+        # Exclude common non-profile paths
+        ignored_paths = {
+            "/login", "/signup", "/home", "/privacy", "/terms", "/help",
+            "/about", "/policies", "/share", "/tr", "/plugins"
+        }
+        if path.lower() in ignored_paths:
+            return None
+
+        clean_url = urlunparse((parsed.scheme.lower(), parsed.netloc.lower(), path, "", "", ""))
+        return clean_url
+    except Exception:
+        return None
+
+
+def extract_social_profiles(html_content: str, base_url: str = "") -> dict[str, list[dict[str, str]]]:
+    """
+    Extract verified social media profiles from HTML content.
+
+    Returns dict mapping platform name to list of dicts:
+    {
+      "linkedin": [{"url": "https://linkedin.com/company/acme", "source_page": base_url}],
+      ...
+    }
+    """
+    results: dict[str, list[dict[str, str]]] = {p: [] for p in _PLATFORMS}
+    seen_urls: set[str] = set()
+
+    if not html_content:
+        return results
+
+    try:
+        soup = BeautifulSoup(html_content, "html.parser")
+    except Exception as e:
+        logger.debug("HTML parsing failed in extract_social_profiles: %s", e)
+        return results
+
+    links = soup.find_all("a", href=True)
+    for a in links:
+        href = a["href"].strip()
+        if not href.startswith("http"):
+            continue
+
+        for platform, patterns in _PLATFORMS.items():
+            matched = False
+            for pat in patterns:
+                if pat.search(href):
+                    cleaned = _clean_social_url(href)
+                    if cleaned and cleaned not in seen_urls:
+                        seen_urls.add(cleaned)
+                        results[platform].append({
+                            "url": cleaned,
+                            "source_page": base_url or "",
+                        })
+                    matched = True
+                    break
+            if matched:
+                break
+
+    return results