seoagents 0.2.0__py3-none-any.whl

seoagent/__init__.py

@@ -0,0 +1,7 @@
+"""SEO Agent: an open-source SEO agent for link building, exposed as an MCP server.
+
+Free mode serves the bundled 25-site library locally and builds ten links, one of them on a DA 90+ site. With an API key it connects to the hosted engine,
+which adds the full 1,245-site library, campaign planning, automatic building and verification.
+"""
+
+__version__ = "0.2.0"

seoagent/agent/browser.py

@@ -0,0 +1,296 @@
+"""Playwright helpers shared by playbooks: session handling and field finding heuristics."""
+from __future__ import annotations
+
+import os
+import re
+from contextlib import contextmanager
+from typing import Iterator, Optional
+
+from playwright.sync_api import Locator, Page, TimeoutError as PWTimeout, sync_playwright
+
+URL_HINTS = ("url", "link", "http", "website", "web address", "address", "long")
+SUBMIT_HINTS = ("shorten", "submit", "add", "create", "go", "generate", "make", "save", "post", "send")
+COOKIE_HINTS = ("accept", "agree", "got it", "allow", "ok", "consent", "i understand")
+
+
+@contextmanager
+def session(headless: bool = True, viewport: tuple[int, int] = (1366, 900)) -> Iterator[Page]:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=headless)
+        ctx = browser.new_context(
+            viewport={"width": viewport[0], "height": viewport[1]},
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+            locale="en-US",
+        )
+        page = ctx.new_page()
+        page.set_default_timeout(15000)
+        try:
+            yield page
+        finally:
+            ctx.close()
+            browser.close()
+
+
+DEFAULT_PROFILE = os.path.join(os.path.expanduser("~"), ".linkengine", "profiles")
+
+
+def profile_dir(name: str = "default") -> str:
+    return os.environ.get("LINKENGINE_PROFILES") or os.path.join(DEFAULT_PROFILE, name)
+
+
+@contextmanager
+def signed_in_session(name: str = "default", headless: bool = False, viewport: tuple[int, int] = (1300, 900),
+                      url: str = "") -> Iterator[Page]:
+    """A browser the person can sign into, and that stays signed in.
+
+    Two things here are not cosmetic. It runs **real Chrome**, not Playwright's bundled Chromium, and it turns off
+    the flag that makes the page see `navigator.webdriver === true`. Google refuses to complete a sign-in in a
+    browser advertising itself as automated -- deliberately, to stop phishing -- and since some sites route their
+    own verification through Google, a person can end up unable to sign into their own account at all. Nothing
+    here defeats a security check: the human types their own password into their own account. It stops the browser
+    falsely announcing itself as a robot while they do it.
+
+    The profile is kept on disk, so this is asked for once and every later build starts already signed in."""
+    d = profile_dir(name)
+    os.makedirs(d, exist_ok=True)
+    with sync_playwright() as p:
+        ctx = p.chromium.launch_persistent_context(
+            d,
+            channel="chrome",
+            headless=headless,
+            viewport={"width": viewport[0], "height": viewport[1]},
+            locale="en-US",
+            args=["--disable-blink-features=AutomationControlled",
+                  "--no-default-browser-check", "--no-first-run",
+                  f"--window-size={viewport[0] + 40},{viewport[1] + 120}"],
+            ignore_default_args=["--enable-automation"],
+        )
+        page = ctx.pages[0] if ctx.pages else ctx.new_page()
+        page.set_default_timeout(30000)
+        if url:
+            try:
+                page.goto(url, wait_until="domcontentloaded")
+            except Exception:
+                pass
+        try:
+            yield page
+        finally:
+            ctx.close()
+
+
+def signed_in_to(page: Page, host: str) -> bool:
+    """Whether the profile holds a live session for this host, read from the cookie the site itself sets."""
+    marker = {"facebook.com": r"c_user=\d+", "x.com": r"auth_token=", "linkedin.com": r"li_at=",
+              "reddit.com": r"reddit_session=", "medium.com": r"sid="}.get(host, r"session")
+    try:
+        return bool(page.evaluate(f"() => !!document.cookie.match(/{marker}/)"))
+    except Exception:
+        return False
+
+
+def quoted_names(texts: list[str]) -> list[str]:
+    """Button and field names the guide quotes, e.g. Click "Shorten URL"."""
+    out: list[str] = []
+    for t in texts:
+        for m in re.findall(r'[“"]([^”"]{2,40})[”"]', t):
+            m = m.strip()
+            if m and m.lower() not in (x.lower() for x in out):
+                out.append(m)
+    return out
+
+
+def dismiss_banners(page: Page) -> None:
+    for hint in COOKIE_HINTS:
+        try:
+            btn = page.get_by_role("button", name=re.compile(rf"^\s*{re.escape(hint)}", re.I)).first
+            if btn.is_visible(timeout=800):
+                btn.click(timeout=2000)
+                page.wait_for_timeout(300)
+                return
+        except Exception:
+            continue
+
+
+# First-run and welcome dialogs sit on top of everything and swallow the click you meant to make. They are not
+# consent banners -- the wording is different and so is the button -- so they need their own pass.
+DIALOG_BUTTONS = ("Use Page", "Not now", "Not Now", "Skip", "Skip for now", "Maybe later", "Got it", "OK",
+                  "Okay", "Done", "Close", "Continue", "Dismiss", "No thanks")
+
+
+def dismiss_dialogs(page: Page, rounds: int = 3) -> list[str]:
+    """Close whatever modal is covering the page, and say what was closed.
+
+    A welcome dialog looks harmless in a screenshot and is fatal to automation: every control behind it is
+    present in the DOM, visible to a query, and impossible to click. Returns the buttons it pressed so a failure
+    later can be read against what was in the way."""
+    closed: list[str] = []
+    for _ in range(rounds):
+        hit = False
+        for name in DIALOG_BUTTONS:
+            try:
+                btn = page.get_by_role("button", name=re.compile(rf"^\s*{re.escape(name)}\s*$", re.I)).first
+                if btn.is_visible(timeout=600):
+                    btn.click(timeout=2500)
+                    closed.append(name)
+                    page.wait_for_timeout(1200)
+                    hit = True
+                    break
+            except Exception:
+                continue
+        if not hit:
+            try:                                  # a dialog with only an X, or one that closes on Escape
+                x = page.locator("div[role=dialog] [aria-label='Close']").first
+                if x.is_visible(timeout=600):
+                    x.click(timeout=2000); closed.append("Close"); page.wait_for_timeout(1000); continue
+            except Exception:
+                pass
+            break
+    return closed
+
+
+def _attr_blob(loc: Locator) -> str:
+    try:
+        return loc.evaluate(
+            "e => [e.type, e.name, e.id, e.placeholder, e.getAttribute('aria-label'), e.title, "
+            "(e.labels && e.labels[0] ? e.labels[0].innerText : '')].join(' ').toLowerCase()"
+        )
+    except Exception:
+        return ""
+
+
+def find_url_input(page: Page) -> Optional[Locator]:
+    """The most likely field for a URL: typed url first, then hinted names, then the largest visible text input."""
+    typed = page.locator("input[type=url]")
+    for i in range(typed.count()):
+        if typed.nth(i).is_visible():
+            return typed.nth(i)
+    cands = page.locator("input[type=text], input:not([type]), textarea")
+    scored: list[tuple[int, Locator]] = []
+    for i in range(min(cands.count(), 40)):
+        loc = cands.nth(i)
+        try:
+            if not loc.is_visible():
+                continue
+        except Exception:
+            continue
+        blob = _attr_blob(loc)
+        s = sum(3 for h in URL_HINTS if h in blob)
+        if "search" in blob or "email" in blob or "password" in blob:
+            s -= 5
+        try:
+            box = loc.bounding_box()
+            if box:
+                s += min(int(box["width"] // 100), 4)
+        except Exception:
+            pass
+        scored.append((s, loc))
+    scored.sort(key=lambda t: -t[0])
+    return scored[0][1] if scored and scored[0][0] > 0 else (scored[0][1] if scored else None)
+
+
+def find_submit(page: Page, near: Optional[Locator] = None, names: Optional[list[str]] = None) -> Optional[Locator]:
+    """A submit control: the input's own form submit first (it is the one wired to the field), then the guide's
+    quoted button name, then generic verbs. Guide names come second because sites often repeat them on
+    decorative call-to-action links elsewhere on the page."""
+    if near is not None:
+        try:
+            form = near.locator("xpath=ancestor::form[1]")
+            if form.count():
+                subs = form.locator("button[type=submit], input[type=submit], button:not([type]), input[type=image], button[type=button]")
+                ranked: list[tuple[int, Locator]] = []
+                wanted = [n.lower() for n in (names or [])]
+                for i in range(min(subs.count(), 8)):
+                    sub = subs.nth(i)
+                    if not sub.is_visible(timeout=300):
+                        continue
+                    try:
+                        label = sub.evaluate("e => (e.innerText || e.value || e.getAttribute('aria-label') || '').trim().toLowerCase()")
+                    except Exception:
+                        label = ""
+                    rank = 0
+                    if label:
+                        rank = 1
+                        if any(h in label for h in SUBMIT_HINTS) or any(w in label for w in wanted):
+                            rank = 2
+                    ranked.append((rank, sub))
+                if ranked:
+                    ranked.sort(key=lambda t: -t[0])
+                    return ranked[0][1]
+        except Exception:
+            pass
+    for name in names or []:
+        for role in ("button", "link"):
+            try:
+                b = page.get_by_role(role, name=re.compile(rf"^\s*{re.escape(name)}\s*$", re.I)).first
+                if b.is_visible(timeout=500):
+                    return b
+            except Exception:
+                pass
+    for hint in SUBMIT_HINTS:
+        try:
+            b = page.get_by_role("button", name=re.compile(hint, re.I)).first
+            if b.is_visible(timeout=300):
+                return b
+        except Exception:
+            pass
+    try:
+        b = page.locator("input[type=submit]").first
+        if b.is_visible(timeout=300):
+            return b
+    except Exception:
+        pass
+    return None
+
+
+def visible_urls(page: Page) -> list[str]:
+    """Every http(s) URL currently shown as a link href, input value or bare text on the page."""
+    try:
+        return page.evaluate(
+            """() => {
+              const out = new Set();
+              for (const a of document.querySelectorAll('a[href]')) out.add(a.href);
+              for (const i of document.querySelectorAll('input, textarea')) { const v = (i.value||'').trim(); if (/^https?:\\/\\//i.test(v)) out.add(v); }
+              const re = /https?:\\/\\/[^\\s"'<>]+/g; const m = document.body.innerText.match(re) || [];
+              for (const x of m) out.add(x.replace(/[).,]+$/, ''));
+              return [...out];
+            }"""
+        )
+    except Exception:
+        return []
+
+
+def bot_gate(page: Page, check_empty: bool = True) -> Optional[str]:
+    """Why an automated browser cannot proceed here, or None: a challenge page, a captcha widget, or nothing rendered.
+    The empty-page signal only means anything on first load; after a submit a short confirmation page is normal."""
+    try:
+        title = (page.title() or "").lower()
+        if "just a moment" in title or "attention required" in title or "access denied" in title:
+            return "bot challenge page"
+        body = (page.inner_text("body") or "")[:2000].lower()
+        if "you have been blocked" in body or "verify you are human" in body and "cloudflare" in body:
+            return "blocked by the site's firewall"
+        has = page.evaluate(
+            "() => ({turn: !!document.querySelector('input[name=cf-turnstile-response], .cf-turnstile, iframe[src*=challenges]'),"
+            " recap: !!document.querySelector('.g-recaptcha, iframe[src*=recaptcha], .h-captcha, iframe[src*=hcaptcha]'),"
+            " empty: (document.body ? document.body.innerText.trim().length : 0) < 20})"
+        )
+        if has["turn"]:
+            return "Cloudflare Turnstile on the form"
+        if has["recap"]:
+            return "captcha on the form"
+        if has["empty"] and check_empty:
+            return "page rendered empty for an automated browser"
+    except Exception:
+        return None
+    return None
+
+
+def safe_goto(page: Page, url: str) -> bool:
+    try:
+        page.goto(url, wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(1200)
+        return True
+    except PWTimeout:
+        return False
+    except Exception:
+        return False

seoagent/agent/executor.py

@@ -0,0 +1,55 @@
+"""Runs one link job on the user's machine: choose a playbook, drive a local browser, save proof, return a JobResult.
+Login-free sites only; every other method is followed by the assistant itself from get_method."""
+from __future__ import annotations
+
+import os
+import time
+import uuid
+
+from ..models import JobResult, Site
+from .browser import bot_gate, session
+from .playbooks import open_submit, shortener
+
+RUNS_DIR = os.environ.get("SEOAGENT_RUNS", os.path.join(os.path.expanduser("~"), ".seoagent", "runs"))
+PLAYBOOKS = {"url_shortener": "shortener", "bookmark_submit": "open_submit"}
+
+
+def can_auto_build(site: Site) -> bool:
+    return site.tier == "A" and site.method in PLAYBOOKS
+
+
+def wait_for_human(page, seconds: int = 180) -> bool:
+    """Headed mode: give the person at the screen time to clear a captcha. True when the gate is gone."""
+    deadline = time.time() + seconds
+    while time.time() < deadline:
+        if not bot_gate(page):
+            return True
+        page.wait_for_timeout(2000)
+    return False
+
+
+def run_job(site: Site, target_url: str, anchor_text: str = "", *, headless: bool = True, description: str = "") -> JobResult:
+    if not can_auto_build(site):
+        return JobResult(slug=site.slug, target_url=target_url, anchor_text=anchor_text, status="manual",
+                         notes=f"{site.method_label} needs an account; follow get_method in your own browser, then verify_link and log_link")
+    job_id = f"{int(time.time())}-{site.slug}-{uuid.uuid4().hex[:6]}"
+    out_dir = os.path.join(RUNS_DIR, job_id)
+    os.makedirs(out_dir, exist_ok=True)
+    proof = os.path.join(out_dir, "proof.png")
+    try:
+        with session(headless=headless) as page:
+            def go():
+                if PLAYBOOKS[site.method] == "shortener":
+                    return shortener.run(page, site, target_url, proof)
+                return open_submit.run(page, site, target_url, anchor_text or site.name, proof, description)
+            r = go()
+            if r.get("status") == "gated" and not headless and wait_for_human(page):
+                r = go()
+    except Exception as e:  # a playbook crash is a failed job, never a crashed server
+        msg = f"{type(e).__name__}: {e}"[:300]
+        if "Executable doesn't exist" in msg or "playwright install" in msg:
+            msg = "the browser is not installed yet: run `playwright install chromium` once, then try again"
+        r = {"status": "failed", "notes": msg}
+    return JobResult(slug=site.slug, target_url=target_url, anchor_text=anchor_text, status=r["status"],
+                     live_url=r.get("live_url"), proof_path=proof if os.path.exists(proof) else None, notes=r.get("notes", ""),
+                     gate=r.get("gate"))

seoagent/agent/playbooks/open_submit.py

@@ -0,0 +1,79 @@
+"""Tier A playbook: open submission forms (no account). Fill URL, title and description, submit, verify."""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from playwright.sync_api import Locator, Page
+
+from ...models import Site
+from ...gates import gate_from_reason
+from ..browser import dismiss_banners, find_submit, find_url_input, quoted_names, safe_goto, visible_urls
+
+
+def _find_text(page: Page, hints: tuple[str, ...], exclude: Optional[Locator] = None) -> Optional[Locator]:
+    cands = page.locator("input[type=text], input:not([type]), textarea")
+    for i in range(min(cands.count(), 40)):
+        loc = cands.nth(i)
+        try:
+            if not loc.is_visible():
+                continue
+            blob = loc.evaluate("e => [e.name, e.id, e.placeholder, e.getAttribute('aria-label')].join(' ').toLowerCase()")
+        except Exception:
+            continue
+        if any(h in blob for h in hints):
+            if exclude is not None:
+                try:
+                    if loc.evaluate("e => e") == exclude.evaluate("e => e"):
+                        continue
+                except Exception:
+                    pass
+            return loc
+    return None
+
+
+def run(page: Page, site: Site, target_url: str, anchor_text: str, proof_path: str, description: str = "") -> dict:
+    if not safe_goto(page, site.url):
+        return {"status": "failed", "notes": "site did not load"}
+    dismiss_banners(page)
+    names = quoted_names([s.text for s in site.steps])
+    # follow an "Add link / Submit" entry point if the form is not on the landing page
+    if find_url_input(page) is None:
+        for name in names + ["Submit", "Add link", "Add URL", "Add site", "Submit URL", "Suggest"]:
+            try:
+                link = page.get_by_role("link", name=re.compile(re.escape(name), re.I)).first
+                if link.is_visible(timeout=400):
+                    link.click(); page.wait_for_timeout(1500); break
+            except Exception:
+                continue
+    field = find_url_input(page)
+    if field is None:
+        page.screenshot(path=proof_path)
+        return {"status": "failed", "notes": "no URL input found"}
+    field.fill(target_url)
+    title = _find_text(page, ("title", "name", "subject"), exclude=field)
+    if title:
+        title.fill(anchor_text)
+    desc = _find_text(page, ("desc", "summary", "about", "comment", "text"), exclude=field)
+    if desc:
+        desc.fill(description or f"{anchor_text} - {target_url}")
+    btn = find_submit(page, near=field, names=names)
+    if btn is not None:
+        btn.click()
+    else:
+        field.press("Enter")
+    page.wait_for_timeout(3500)
+    page.screenshot(path=proof_path)
+    body = ""
+    try:
+        body = page.inner_text("body").lower()
+    except Exception:
+        pass
+    if "captcha" in body or "verify you are human" in body:
+        return {"status": "gated", "notes": "captcha shown after submit", "gate": gate_from_reason("captcha on the form", site.name).model_dump()}
+    urls = visible_urls(page)
+    if any(target_url.rstrip("/") in u for u in urls) or target_url.rstrip("/").lower() in body:
+        return {"status": "placed", "live_url": page.url, "notes": "target URL visible on result page"}
+    if any(w in body for w in ("thank", "submitted", "pending", "review", "approved", "success")):
+        return {"status": "placed", "live_url": page.url, "notes": "submission acknowledged; listing may await moderation"}
+    return {"status": "failed", "notes": "submitted but target URL not found on result page"}

seoagent/agent/playbooks/shortener.py

@@ -0,0 +1,92 @@
+"""Tier A playbook: URL shorteners. Paste the target, submit, capture the short link, verify it resolves."""
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+from playwright.sync_api import Page
+
+from ...models import Site
+from ...gates import gate_from_reason
+from ..browser import bot_gate, dismiss_banners, find_submit, find_url_input, quoted_names, safe_goto, visible_urls
+
+NOT_A_SHORT_LINK = re.compile(r"/(human|captcha|login|signin|signup|register|create|preview|stats?|info|terms|privacy|about|contact|api|blog)\b", re.I)
+
+
+def _host(u: str) -> str:
+    return urlparse(u).netloc.lower().removeprefix("www.")
+
+
+def _same_site(u: str, domain: str) -> bool:
+    host = _host(u)
+    return host == domain or host.endswith("." + domain) or domain.endswith("." + host)
+
+
+def pick_short_link(before: set[str], after: list[str], site_domain: str, site_url: str, target_url: str) -> list[str]:
+    """Candidate short links among URLs that appeared after submitting, best first.
+    A short link lives on the shortener's own domain, has a short opaque path, and does not carry the target as a
+    query parameter or point at a utility page."""
+    tgt = target_url.rstrip("/").lower()
+    cands = []
+    for u in after:
+        if u in before or not _same_site(u, site_domain):
+            continue
+        if u.rstrip("/") == site_url.rstrip("/") or tgt in u.lower():
+            continue
+        p = urlparse(u)
+        segs = [s for s in p.path.split("/") if s]
+        if not segs or len(segs) > 2 or p.query or NOT_A_SHORT_LINK.search(p.path):
+            continue
+        if not re.fullmatch(r"[A-Za-z0-9_\-]{2,32}", segs[-1]):
+            continue
+        cands.append(u)
+    cands.sort(key=lambda u: (len(urlparse(u).path.strip("/").split("/")), len(u)))
+    return cands
+
+
+def verify(page: Page, short: str, target_url: str) -> bool:
+    """The short link resolves to the target, or shows an interstitial that references it."""
+    try:
+        resp = page.request.get(short, max_redirects=5, timeout=15000)
+        if _host(resp.url) == _host(target_url):
+            return True
+        body = resp.text()[:200000]
+        return target_url.rstrip("/") in body
+    except Exception:
+        return False
+
+
+def run(page: Page, site: Site, target_url: str, proof_path: str) -> dict:
+    if not safe_goto(page, site.url):
+        return {"status": "failed", "notes": "site did not load"}
+    dismiss_banners(page)
+    gate = bot_gate(page)
+    if gate:
+        page.screenshot(path=proof_path)
+        return {"status": "gated", "notes": f"needs a human: {gate}", "gate": gate_from_reason(gate, site.name).model_dump()}
+    field = find_url_input(page)
+    if field is None:
+        page.screenshot(path=proof_path)
+        return {"status": "failed", "notes": "no URL input found"}
+    before = set(visible_urls(page))
+    field.click()
+    field.fill(target_url)
+    names = quoted_names([s.text for s in site.steps])
+    btn = find_submit(page, near=field, names=names)
+    if btn is not None:
+        btn.click()
+    else:
+        field.press("Enter")
+    page.wait_for_timeout(3500)
+    gate = bot_gate(page)
+    if gate:
+        page.screenshot(path=proof_path)
+        return {"status": "gated", "notes": f"needs a human: {gate} after submitting", "gate": gate_from_reason(gate, site.name).model_dump()}
+    cands = pick_short_link(before, visible_urls(page), site.domain, site.url, target_url)
+    page.screenshot(path=proof_path)
+    if not cands:
+        return {"status": "failed", "notes": "submitted but no short link appeared"}
+    for short in cands[:3]:
+        if verify(page, short, target_url):
+            return {"status": "placed", "live_url": short, "notes": "short link verified"}
+    return {"status": "unverified", "live_url": cands[0], "notes": "short link captured; could not confirm it resolves to the target (interstitial or script redirect)"}