liferay-docs-scraper 0.1.0__py3-none-any.whl

liferay_docs_scraper/check_regressions.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Flag suspicious content changes in raw/ after a crawl4ai refresh, using git.
+
+Compares the working tree against a given git ref (default: HEAD, i.e. the
+last commit) for every raw/**/*.md file:
+  - Shrank a lot (body text, not counting frontmatter): the signature of a
+    broken/partial fetch overwriting a good page.
+  - Grew a lot: the signature of CONTENT_SELECTOR failing to match and
+    crawl4ai falling back to the whole page (breadcrumb/nav/footer chrome
+    and all) instead of raising an error.
+
+Operates on filter_urls.resolve_docs_dir() (the same shared corpus location
+the scraper writes to and the skill reads from), not the current directory.
+
+Usage:
+    uvx --from liferay-docs-scraper check-regressions [--ref HEAD] [--shrink-threshold 0.5] [--growth-threshold 3.0]
+"""
+
+import argparse
+import subprocess
+
+from .filter_urls import resolve_docs_dir
+
+ROOT = resolve_docs_dir()
+
+
+def body_len(text: str) -> int:
+    """Length of the file excluding the YAML frontmatter block."""
+    parts = text.split("---\n", 2)
+    body = parts[2] if len(parts) == 3 else text
+    return len(body)
+
+
+def git_show(ref: str, path: str) -> str | None:
+    result = subprocess.run(
+        ["git", "show", f"{ref}:{path}"], cwd=ROOT, capture_output=True, text=True,
+    )
+    if result.returncode != 0:
+        return None
+    return result.stdout
+
+
+def changed_raw_files(ref: str) -> list[str]:
+    result = subprocess.run(
+        ["git", "diff", "--name-only", ref, "--", "raw/"],
+        cwd=ROOT, capture_output=True, text=True, check=True,
+    )
+    return [line for line in result.stdout.splitlines() if line.endswith(".md")]
+
+
+def run_check(ref: str = "HEAD", shrink_threshold: float = 0.5, growth_threshold: float = 3.0) -> bool:
+    """Print the regression report; return True iff something looked suspicious."""
+    changed = changed_raw_files(ref)
+    print(f"Archivos .md cambiados en raw/ vs {ref}: {len(changed)}")
+
+    shrunk, grew = [], []
+    for rel_path in changed:
+        full_path = ROOT / rel_path
+        old_text = git_show(ref, rel_path)
+        if old_text is None:
+            continue  # new file, nothing to compare
+        if not full_path.exists():
+            continue  # deleted/moved (e.g. quarantined), handled separately
+
+        new_text = full_path.read_text(encoding="utf-8")
+        old_len = body_len(old_text)
+        new_len = body_len(new_text)
+        if old_len == 0:
+            continue
+        ratio = new_len / old_len
+        if ratio < shrink_threshold:
+            shrunk.append((rel_path, old_len, new_len, ratio))
+        elif ratio > growth_threshold:
+            grew.append((rel_path, old_len, new_len, ratio))
+
+    if shrunk:
+        print(f"\nSOSPECHOSOS ({len(shrunk)}) -- perdieron más del "
+              f"{(1 - shrink_threshold) * 100:.0f}% del contenido:")
+        for rel_path, old_len, new_len, ratio in sorted(shrunk, key=lambda x: x[3]):
+            print(f"  {rel_path}: {old_len} -> {new_len} chars ({ratio:.0%})")
+    else:
+        print("\nNinguno por debajo del umbral de encogimiento -- sin señales de pérdida de contenido.")
+
+    if grew:
+        print(f"\nSOSPECHOSOS ({len(grew)}) -- crecieron más de {growth_threshold:.0f}x "
+              f"(posible fallback a la página completa sin selector):")
+        for rel_path, old_len, new_len, ratio in sorted(grew, key=lambda x: -x[3]):
+            print(f"  {rel_path}: {old_len} -> {new_len} chars ({ratio:.1f}x)")
+
+    return bool(shrunk or grew)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ref", default="HEAD")
+    parser.add_argument("--shrink-threshold", type=float, default=0.5,
+                         help="Flag files whose body shrank below this fraction of the original.")
+    parser.add_argument("--growth-threshold", type=float, default=3.0,
+                         help="Flag files whose body grew beyond this multiple of the original.")
+    args = parser.parse_args()
+    run_check(args.ref, args.shrink_threshold, args.growth_threshold)
+
+
+if __name__ == "__main__":
+    main()

liferay_docs_scraper/classify_pages.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Navigation-vs-content heuristic, shared with crawl4ai_pipeline.py.
+
+An "index"/navigation page is one whose body (frontmatter stripped) is
+short and consists mostly of links to subpages -- no substantial technical
+content of its own. Everything else is "content".
+
+Heuristic (no API calls, pure text analysis):
+  - Strip Markdown link syntax down to visible text (`[text](url)` -> `text`)
+    and strip heading/emphasis markup (`#`, `*`, `_`, backticks).
+  - total_words: word count of that visible text.
+  - link_ratio: fraction of those words that come from inside a Markdown
+    link's link-text span.
+  - "index" iff total_words < INDEX_MAX_WORDS and link_ratio >= INDEX_MIN_LINK_RATIO.
+"""
+
+import re
+
+INDEX_MAX_WORDS = 150
+INDEX_MIN_LINK_RATIO = 0.5
+
+LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)]*)\)")
+MARKUP_RE = re.compile(r"[#*_`\\]")
+
+
+def analyze_body(body: str) -> tuple[int, float]:
+    links = LINK_RE.findall(body)
+    link_word_count = sum(len(text.split()) for text, _url in links)
+
+    visible = LINK_RE.sub(lambda m: m.group(1), body)
+    visible = MARKUP_RE.sub("", visible)
+    total_words = len(visible.split())
+
+    link_ratio = (link_word_count / total_words) if total_words else 0.0
+    return total_words, link_ratio
+
+
+def classify(total_words: int, link_ratio: float) -> str:
+    if total_words < INDEX_MAX_WORDS and link_ratio >= INDEX_MIN_LINK_RATIO:
+        return "index"
+    return "content"

liferay_docs_scraper/filter_urls.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Shared URL/capability utilities for the crawl4ai_pipeline.py corpus.
+
+Capability classification (matching learn.liferay.com/w/dxp URLs to one of
+the 14 capabilities listed on /w/dxp/index, plus the self-hosted prune
+rules), the URL->filename/frontmatter helpers used when writing pages to
+raw/{capability}/*.md, and resolve_docs_dir() -- the one place that decides
+where that raw/ corpus actually lives on disk.
+"""
+
+import hashlib
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+
+CAPABILITIES = {
+    "cloud": "/w/dxp/cloud",
+    "search": "/w/dxp/search",
+    "self-hosted": "/w/dxp/self-hosted-installation-and-upgrades",
+    "sites": "/w/dxp/sites",
+    "security": "/w/dxp/security-and-administration",
+    "development": "/w/dxp/development",
+    "commerce": "/w/dxp/commerce",
+    "personalization": "/w/dxp/personalization",
+    "low-code": "/w/dxp/low-code",
+    "content-management-system": "/w/dxp/content-management-system",
+    "digital-asset-management": "/w/dxp/digital-asset-management",
+    "integration": "/w/dxp/integration",
+    "ai": "/w/dxp/ai",
+    "getting-started": "/w/dxp/getting-started",
+}
+
+# All 14 capabilities listed on https://learn.liferay.com/w/dxp/index are in
+# scope now; nothing under /w/dxp is deliberately excluded anymore.
+OUT_OF_SCOPE_PREFIXES: list[str] = []
+
+# (rule label, substring whose presence -- followed by more path -- excludes the URL)
+SELF_HOSTED_PRUNE_RULES = [
+    (
+        "deprecations-and-breaking-changes-reference subpage",
+        "/upgrading-liferay/deprecations-and-breaking-changes-reference/",
+    ),
+    (
+        "installing-earlier-liferay-versions-on-application-servers subpage",
+        "/installing-earlier-liferay-versions-on-application-servers/",
+    ),
+    (
+        "cne-aws-ready subpage",
+        "/cloud-native-experience/cne-cloud-provider-ready/cne-aws-ready/",
+    ),
+    (
+        "cne-gcp-ready subpage",
+        "/cloud-native-experience/cne-cloud-provider-ready/cne-gcp-ready/",
+    ),
+]
+
+
+def normalize(url: str) -> str:
+    """Strip a trailing slash from the path, keep everything else as-is."""
+    parsed = urlparse(url)
+    path = parsed.path
+    if path != "/" and path.endswith("/"):
+        path = path[:-1]
+    return parsed._replace(path=path).geturl()
+
+
+def matches_prefix(path: str, prefix: str) -> bool:
+    return path == prefix or path.startswith(prefix + "/")
+
+
+def prune_reason(path: str) -> str | None:
+    for label, substr in SELF_HOSTED_PRUNE_RULES:
+        if substr in path:
+            return label
+    return None
+
+
+def classify_url(url: str) -> dict:
+    """Classify a single (already-normalized) URL for the capability pipeline.
+
+    Returns a dict with:
+      - capability: matched capability name, or None if out of scope
+      - prune_reason: self-hosted prune rule label, or None
+      - known_out_of_scope: True if it matches one of the known-excluded
+        capabilities rather than being an unrecognized/"odd" URL worth
+        flagging for manual review
+    """
+    path = urlparse(url).path
+    matched_capability = None
+    for name, prefix in CAPABILITIES.items():
+        if matches_prefix(path, prefix):
+            matched_capability = name
+            break
+
+    if matched_capability is None:
+        known_out_of_scope = any(matches_prefix(path, prefix) for prefix in OUT_OF_SCOPE_PREFIXES)
+        return {"capability": None, "prune_reason": None, "known_out_of_scope": known_out_of_scope}
+
+    reason = prune_reason(path) if matched_capability == "self-hosted" else None
+    return {"capability": matched_capability, "prune_reason": reason, "known_out_of_scope": False}
+
+
+def slugify(url: str, prefix: str) -> str:
+    """URL path (with the capability prefix stripped) -> a flat filename stem."""
+    path = urlparse(url).path
+    remainder = path[len(prefix):].strip("/")
+    if not remainder:
+        return "index"
+    return remainder.replace("/", "-")
+
+
+def build_frontmatter(url: str, capability: str, markdown: str) -> str:
+    fetched_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    content_hash = hashlib.sha256(markdown.encode("utf-8")).hexdigest()
+    lines = [
+        "---",
+        f'url: "{url}"',
+        f"capability: {capability}",
+        f'fetched_at: "{fetched_at}"',
+        f'content_hash: "sha256:{content_hash}"',
+        "---",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def _default_data_dir() -> Path:
+    """Per-user app-data directory, one convention per OS, so the corpus
+    lives in the same predictable place regardless of which project you
+    happen to be running the scraper or the skill from."""
+    if sys.platform == "win32":
+        base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
+        return Path(base) / "liferay-docs"
+    if sys.platform == "darwin":
+        return Path.home() / "Library" / "Application Support" / "liferay-docs"
+    # Linux and other Unix-likes: XDG Base Directory spec
+    base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share")
+    return Path(base) / "liferay-docs"
+
+
+def resolve_docs_dir() -> Path:
+    """Where the local corpus (raw/, reports/filtered/) lives: $LIFERAY_DOCS_DIR
+    if set, otherwise the OS-appropriate default data directory. One shared
+    corpus regardless of the current project, unless explicitly overridden --
+    see _default_data_dir() for the per-OS default."""
+    override = os.environ.get("LIFERAY_DOCS_DIR")
+    if override:
+        return Path(override).expanduser()
+    return _default_data_dir()

liferay_docs_scraper/pipeline.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+"""Weekly from-scratch refresh of the learn.liferay.com/w/dxp corpus, crawl4ai-only.
+
+Builds raw/{capability}/*.md under filter_urls.resolve_docs_dir(): the
+$LIFERAY_DOCS_DIR directory if that env var is set, otherwise one shared,
+OS-appropriate per-user data directory (e.g. ~/Library/Application
+Support/liferay-docs on macOS, %LOCALAPPDATA%\\liferay-docs on Windows,
+~/.local/share/liferay-docs on Linux). Deliberately NOT the current working
+directory -- the liferay-expert skill looks in that same shared location
+regardless of which project you're in when you ask a question, so you
+don't end up with a separate copy of the corpus per project.
+
+A single crawl4ai deep crawl handles both URL discovery and content
+extraction:
+
+  - A BFS deep crawl starts at /w/dxp/index and follows every internal link
+    under /w/dxp/*. crawl4ai extracts links from the FULL page regardless of
+    css_selector, so this single crawl gets us both (a) the complete current
+    set of URLs on the site and (b) each page's Markdown scoped to
+    CONTENT_SELECTOR, in one visit per page. That selector (see below) is
+    precise enough that no further chrome-stripping is needed -- what
+    crawl4ai returns is already the final page content.
+  - Each page is classified with filter_urls.py's classify_url (capability
+    prefixes + self-hosted prune rules) and, if in scope, written to
+    raw/{capability}/{slug}.md -- unless classify_pages.py's heuristic
+    (reused here, not duplicated) says it's a pure navigation/TOC page with
+    no substantial content of its own, in which case it goes to
+    raw/_navigation/{capability}/{slug}.md instead. This keeps
+    raw/{capability}/ as signal for the liferay-expert skill, while still
+    preserving the navigation pages (not deleting them) in case they're
+    useful later.
+  - Because every run starts from zero, a page that existed last run but
+    isn't found this run (removed from the site, or now out of scope/pruned)
+    is a *candidate* for quarantine -- but BFS link-following can miss a page
+    that's still live (no longer linked from anywhere our crawl reached,
+    while still resolving directly), so before quarantining anything we do a
+    direct HTTP check on each candidate's own URL. Only a confirmed non-200
+    gets quarantined (moved to raw/_removed/{capability}/{slug}.md, logged to
+    reports/filtered/removed_log.jsonl); anything that still responds, or
+    that we simply couldn't reach to check, is left in place and flagged for
+    manual review instead. If a capability's discovered count drops
+    implausibly (crawl likely failed partway), quarantine for that capability
+    is skipped entirely and flagged for manual review instead of trusting a
+    possibly-broken run.
+  - reports/filtered/{capability}_urls.txt, self-hosted_pruned.txt and
+    summary.json are regenerated from this run's live results, so they always
+    reflect the current corpus (same format filter_urls.py produces).
+  - Once everything above is written, check_regressions.py's run_check()
+    runs automatically against the last git commit, if resolve_docs_dir()
+    is itself a git repo (worth `git init`-ing once, purely as a local
+    diffing tool -- nothing needs to be pushed anywhere). Skipped otherwise,
+    or with --skip-regression-check.
+
+Setup and run (see README.md for the full explanation):
+    uvx --from crawl4ai crawl4ai-setup   # one-time: installs Playwright browsers
+    uvx --python 3.13 --from "git+https://github.com/mordonez/liferay-docs-scraper" liferay-docs-scraper
+    # (not on PyPI yet -- once published, just `uvx liferay-docs-scraper`)
+"""
+
+import argparse
+import asyncio
+import json
+import shutil
+import sys
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .check_regressions import run_check as run_regression_check
+from .classify_pages import analyze_body, classify as classify_navigation
+from .filter_urls import (
+    CAPABILITIES,
+    SELF_HOSTED_PRUNE_RULES,
+    build_frontmatter,
+    classify_url,
+    normalize,
+    resolve_docs_dir,
+    slugify,
+)
+
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, ContentTypeFilter, DomainFilter, FilterChain, URLPatternFilter
+
+ROOT = resolve_docs_dir()
+RAW_DIR = ROOT / "raw"
+REMOVED_DIR = RAW_DIR / "_removed"
+NAVIGATION_DIR = RAW_DIR / "_navigation"
+FILTERED_DIR = ROOT / "reports" / "filtered"
+REMOVED_LOG = FILTERED_DIR / "removed_log.jsonl"
+
+SEED_URL = "https://learn.liferay.com/w/dxp/index"
+ALLOWED_DOMAIN = "learn.liferay.com"
+URL_SCOPE_PATTERN = "*/w/dxp*"
+# learn.liferay.com's article template puts the breadcrumb, sidebar TOC, and
+# the actual article body all inside #main-content, with the maintenance
+# banner and global footer outside it. .learn-article-content is scoped
+# tighter still: just the title, body, and resource-type tags -- no
+# breadcrumb/TOC/"Submit Feedback" chrome to strip afterward.
+CONTENT_SELECTOR = ".learn-article-content"
+
+DEFAULT_MAX_DEPTH = 12
+DEFAULT_MAX_PAGES = 3000
+# If a capability's freshly discovered URL count falls below this fraction of
+# its previous count, treat the run as suspect and skip quarantining orphans
+# for that capability rather than mass-deleting good content on a bad crawl.
+QUARANTINE_SAFETY_RATIO = 0.5
+
+# Some fetches render a client-side error banner instead of the real page
+# (transient rendering/server hiccup) -- crawl4ai still reports these as a
+# "successful" fetch, so we have to catch it ourselves and retry.
+ERROR_MARKERS = ["An unexpected error occurred."]
+MIN_ACCEPTABLE_BODY_LENGTH = 30
+CONTENT_RETRY_ATTEMPTS = 3
+CONTENT_RETRY_DELAY_SECONDS = 3.0
+
+
+@dataclass
+class PageOutcome:
+    url: str
+    capability: str
+    slug: str
+    status: str  # "new" | "updated" | "unchanged"
+    is_navigation: bool = False
+
+
+@dataclass
+class RunStats:
+    discovered_total: int = 0
+    fetch_failed: list[str] = field(default_factory=list)
+    unmatched: list[str] = field(default_factory=list)
+    pruned: list[tuple] = field(default_factory=list)
+    outcomes: dict = field(default_factory=lambda: {name: [] for name in CAPABILITIES})
+
+
+def read_existing_hash(path: Path) -> str | None:
+    if not path.exists():
+        return None
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("content_hash:"):
+                return line.strip()
+    return None
+
+
+def is_broken_content(markdown: str) -> bool:
+    """Detect a client-side error banner or a suspiciously empty fetch."""
+    stripped = markdown.strip()
+    if len(stripped) < MIN_ACCEPTABLE_BODY_LENGTH:
+        return True
+    return any(marker in stripped for marker in ERROR_MARKERS)
+
+
+def build_deep_crawl_config(max_depth: int, max_pages: int) -> CrawlerRunConfig:
+    filter_chain = FilterChain([
+        DomainFilter(allowed_domains=[ALLOWED_DOMAIN]),
+        URLPatternFilter(patterns=[URL_SCOPE_PATTERN]),
+        ContentTypeFilter(allowed_types=["text/html"]),
+    ])
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=max_depth, filter_chain=filter_chain, max_pages=max_pages, include_external=False,
+    )
+    return CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        css_selector=CONTENT_SELECTOR,
+        wait_for=f"css:{CONTENT_SELECTOR}",
+        cache_mode=CacheMode.BYPASS,
+        stream=True,
+        verbose=False,
+    )
+
+
+async def refetch_single_page(crawler: AsyncWebCrawler, url: str) -> str | None:
+    """Re-fetch one URL outside the deep crawl (used when the deep crawl's
+    copy looked broken). Returns the page's Markdown, or None if every
+    attempt still looks broken."""
+    single_config = CrawlerRunConfig(
+        css_selector=CONTENT_SELECTOR, wait_for=f"css:{CONTENT_SELECTOR}", cache_mode=CacheMode.BYPASS,
+    )
+    for attempt in range(1, CONTENT_RETRY_ATTEMPTS):
+        await asyncio.sleep(CONTENT_RETRY_DELAY_SECONDS * attempt)
+        result = await crawler.arun(url=url, config=single_config)
+        if result.success and not is_broken_content(result.markdown.raw_markdown):
+            return result.markdown.raw_markdown
+    return None
+
+
+async def run_crawl(max_depth: int, max_pages: int) -> RunStats:
+    stats = RunStats()
+    config = build_deep_crawl_config(max_depth, max_pages)
+
+    async with AsyncWebCrawler() as crawler:
+        stream = await crawler.arun(url=SEED_URL, config=config)
+        async for result in stream:
+            url = normalize(result.url)
+
+            if not result.success:
+                stats.fetch_failed.append(url)
+                continue
+
+            stats.discovered_total += 1
+            classification = classify_url(url)
+            capability = classification["capability"]
+
+            if capability is None:
+                if not classification["known_out_of_scope"]:
+                    stats.unmatched.append(url)
+                continue
+
+            if classification["prune_reason"] is not None:
+                stats.pruned.append((url, classification["prune_reason"]))
+                continue
+
+            prefix = CAPABILITIES[capability]
+            slug = slugify(url, prefix)
+
+            markdown = result.markdown.raw_markdown
+            if is_broken_content(markdown):
+                markdown = await refetch_single_page(crawler, url)
+                if markdown is None:
+                    # Never overwrite a good existing file with a broken
+                    # fetch -- leave it as-is and flag for a manual retry.
+                    stats.fetch_failed.append(url)
+                    continue
+
+            # Pure navigation/TOC pages (per classify_pages.py's heuristic)
+            # go to raw/_navigation/ instead of raw/{capability}/, so the
+            # corpus a future consultation skill reads stays high-signal.
+            total_words, link_ratio = analyze_body(markdown)
+            is_navigation = classify_navigation(total_words, link_ratio) == "index"
+
+            content_path = RAW_DIR / capability / f"{slug}.md"
+            navigation_path = NAVIGATION_DIR / capability / f"{slug}.md"
+            out_path = navigation_path if is_navigation else content_path
+            other_path = content_path if is_navigation else navigation_path
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+
+            new_content = build_frontmatter(url, capability, markdown) + markdown
+            old_hash_line = read_existing_hash(out_path) or read_existing_hash(other_path)
+            existed_before = out_path.exists() or other_path.exists()
+            out_path.write_text(new_content, encoding="utf-8")
+            if other_path.exists():
+                other_path.unlink()  # reclassified since last run -- drop the stale copy
+            new_hash_line = read_existing_hash(out_path)
+
+            if not existed_before:
+                status = "new"
+            elif old_hash_line == new_hash_line:
+                status = "unchanged"
+            else:
+                status = "updated"
+            stats.outcomes[capability].append(PageOutcome(url, capability, slug, status, is_navigation))
+
+    return stats
+
+
+def read_url_from_file(path: Path) -> str | None:
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("url:"):
+                return line.strip().removeprefix("url:").strip().strip('"')
+    return None
+
+
+def is_confirmed_gone(url: str, timeout: float = 10.0) -> bool:
+    """True only if the URL itself, fetched directly (no BFS involved),
+    confirms it's actually gone (404/410). Any other outcome -- 200, a
+    different error, a timeout, a network hiccup on our end -- is NOT treated
+    as confirmation, since BFS link-following can miss pages that are still
+    live but just unlinked from wherever our crawl reached this run."""
+    request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "Mozilla/5.0"})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            return False  # any successful response means it's still there
+    except urllib.error.HTTPError as exc:
+        return exc.code in (404, 410)
+    except Exception:  # noqa: BLE001 - network errors on our side aren't proof of anything
+        return False
+
+
+def quarantine_orphans(stats: RunStats) -> dict:
+    """Move raw/{capability}/*.md and raw/_navigation/{capability}/*.md files
+    that this run didn't touch to raw/_removed/{capability}/ -- but only
+    after directly confirming the URL is actually gone (see
+    is_confirmed_gone). Orphans that turn out to still be live, or that we
+    couldn't check, are left in place and reported separately so a human
+    can look into the crawl's coverage gap."""
+    quarantined: dict[str, list[str]] = {name: [] for name in CAPABILITIES}
+    still_alive: dict[str, list[str]] = {name: [] for name in CAPABILITIES}
+    skipped_capabilities: list[str] = []
+
+    for capability in CAPABILITIES:
+        content_dir = RAW_DIR / capability
+        navigation_dir = NAVIGATION_DIR / capability
+        on_disk_paths = {p.stem: p for p in content_dir.glob("*.md")}
+        on_disk_paths.update({p.stem: p for p in navigation_dir.glob("*.md")})
+        if not on_disk_paths:
+            continue
+
+        # Only files untouched by this run's outcomes are orphans.
+        current_slugs = {o.slug for o in stats.outcomes[capability]}
+        orphans = set(on_disk_paths) - current_slugs
+
+        previous_count = len(on_disk_paths)
+        new_count = len(current_slugs)
+        if previous_count > 0 and new_count < QUARANTINE_SAFETY_RATIO * previous_count:
+            skipped_capabilities.append(capability)
+            continue
+
+        if not orphans:
+            continue
+
+        removed_dir = REMOVED_DIR / capability
+        removed_dir.mkdir(parents=True, exist_ok=True)
+        removed_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        for slug in sorted(orphans):
+            src = on_disk_paths[slug]
+            url = read_url_from_file(src)
+            if url is None or not is_confirmed_gone(url):
+                still_alive[capability].append(slug)
+                continue
+
+            dst = removed_dir / f"{slug}.md"
+            shutil.move(str(src), str(dst))
+            quarantined[capability].append(slug)
+            with REMOVED_LOG.open("a", encoding="utf-8") as log:
+                log.write(json.dumps({"capability": capability, "slug": slug, "url": url, "removed_at": removed_at}) + "\n")
+
+    return {"quarantined": quarantined, "still_alive": still_alive, "skipped_capabilities": skipped_capabilities}
+
+
+def write_filtered_reports(stats: RunStats) -> None:
+    FILTERED_DIR.mkdir(parents=True, exist_ok=True)
+
+    for capability in CAPABILITIES:
+        urls = sorted(o.url for o in stats.outcomes[capability])
+        (FILTERED_DIR / f"{capability}_urls.txt").write_text("\n".join(urls) + "\n", encoding="utf-8")
+
+    pruned_lines = [f"{url}\t# {reason}" for url, reason in sorted(stats.pruned)]
+    (FILTERED_DIR / "self-hosted_pruned.txt").write_text(
+        "\n".join(pruned_lines) + ("\n" if pruned_lines else ""), encoding="utf-8",
+    )
+
+    navigation_urls = sorted(
+        o.url for outcomes in stats.outcomes.values() for o in outcomes if o.is_navigation
+    )
+    (FILTERED_DIR / "navigation_urls.txt").write_text(
+        "\n".join(navigation_urls) + ("\n" if navigation_urls else ""), encoding="utf-8",
+    )
+
+    prune_counts = {label: 0 for label, _ in SELF_HOSTED_PRUNE_RULES}
+    for _, reason in stats.pruned:
+        prune_counts[reason] += 1
+
+    summary = {
+        "capabilities": {
+            name: {
+                "unique_urls": len(stats.outcomes[name]),
+                "navigation_pages": sum(1 for o in stats.outcomes[name] if o.is_navigation),
+            } for name in CAPABILITIES
+        },
+        "self_hosted_pruned": {"total": len(stats.pruned), "by_rule": prune_counts},
+        "total_in_scope": sum(len(stats.outcomes[name]) for name in CAPABILITIES),
+        "total_navigation_pages": len(navigation_urls),
+        "discovered_total": stats.discovered_total,
+        "fetch_failed_count": len(stats.fetch_failed),
+        "unmatched_count": len(stats.unmatched),
+    }
+    (FILTERED_DIR / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
+
+
+def print_summary(stats: RunStats, quarantine_result: dict) -> None:
+    print(f"\nTotal descubierto bajo /w/dxp: {stats.discovered_total}")
+    if stats.fetch_failed:
+        print(f"Fallos de fetch: {len(stats.fetch_failed)}")
+        for url in stats.fetch_failed:
+            print(f"  - {url}")
+
+    print("\nPor capability (nuevas / actualizadas / sin cambios / navegación):")
+    total_in_scope = 0
+    total_navigation = 0
+    for capability, outcomes in stats.outcomes.items():
+        new = sum(1 for o in outcomes if o.status == "new")
+        updated = sum(1 for o in outcomes if o.status == "updated")
+        unchanged = sum(1 for o in outcomes if o.status == "unchanged")
+        navigation = sum(1 for o in outcomes if o.is_navigation)
+        total_in_scope += len(outcomes)
+        total_navigation += navigation
+        print(f"  {capability:12s}: {len(outcomes):4d} total  "
+              f"({new} nuevas, {updated} actualizadas, {unchanged} sin cambios, {navigation} navegación)")
+    print(f"\nTotal en scope: {total_in_scope} ({total_navigation} en raw/_navigation/, "
+          f"{total_in_scope - total_navigation} en raw/{{capability}}/)")
+
+    print(f"\nSelf-hosted podadas: {len(stats.pruned)}")
+
+    quarantined = quarantine_result["quarantined"]
+    total_quarantined = sum(len(v) for v in quarantined.values())
+    print(f"\nEn cuarentena (URL verificada como caída, HTTP 404/410): {total_quarantined}")
+    for capability, slugs in quarantined.items():
+        if slugs:
+            print(f"  {capability}: {len(slugs)}")
+            for slug in slugs:
+                print(f"    - {slug}")
+
+    still_alive = quarantine_result["still_alive"]
+    total_still_alive = sum(len(v) for v in still_alive.values())
+    if total_still_alive:
+        print(f"\nNo redescubiertas por el BFS pero SIGUEN VIVAS (no se tocaron, revisar cobertura del crawl): {total_still_alive}")
+        for capability, slugs in still_alive.items():
+            if slugs:
+                print(f"  {capability}: {len(slugs)}")
+                for slug in slugs:
+                    print(f"    - {slug}")
+
+    if quarantine_result["skipped_capabilities"]:
+        print("\nADVERTENCIA: cuarentena omitida por caída sospechosa de conteo "
+              "(posible crawl incompleto), revisar a mano:")
+        for capability in quarantine_result["skipped_capabilities"]:
+            print(f"  - {capability}")
+
+    if stats.unmatched:
+        print(f"\nURLs raras (ni en scope ni en descartadas conocidas), {len(stats.unmatched)}:")
+        for url in stats.unmatched:
+            print(f"  {url}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--max-depth", type=int, default=DEFAULT_MAX_DEPTH)
+    parser.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
+    parser.add_argument("--skip-regression-check", action="store_true",
+                         help="Skip the post-run check_regressions.run_check() call (e.g. no git repo yet).")
+    args = parser.parse_args()
+
+    stats = asyncio.run(run_crawl(args.max_depth, args.max_pages))
+    quarantine_result = quarantine_orphans(stats)
+    write_filtered_reports(stats)
+    print_summary(stats, quarantine_result)
+
+    suspicious = False
+    if not args.skip_regression_check:
+        print("\n--- Verificación de regresiones (contra el último commit) ---")
+        suspicious = run_regression_check()
+
+    if stats.fetch_failed or suspicious:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

liferay_docs_scraper-0.1.0.dist-info/METADATA

@@ -0,0 +1,128 @@
+Metadata-Version: 2.4
+Name: liferay-docs-scraper
+Version: 0.1.0
+Summary: Scrape learn.liferay.com/w/dxp into a local Markdown corpus (raw/{capability}/*.md) for the liferay-expert Claude Code skill.
+Requires-Python: <3.14,>=3.10
+Requires-Dist: crawl4ai>=0.9.0
+Description-Content-Type: text/markdown
+
+# liferay-docs-scraper
+
+Scrapes `learn.liferay.com/w/dxp/*` into a local, clean Markdown corpus
+(`raw/{capability}/*.md`) and ships a Claude Code skill (`liferay-expert`)
+that answers Liferay DXP questions by searching and citing that corpus.
+
+**This repo does not ship Liferay's documentation.** It ships the code that
+scrapes it, and a skill that reads whatever you scrape locally. Each user
+builds and refreshes their own copy directly from learn.liferay.com.
+
+## Quickstart
+
+The recommended order for a first-time setup: scrape, then install the
+skill, then ask questions.
+
+**1. Build the corpus (one-time, ~30-40 min):**
+
+```bash
+uvx --from crawl4ai crawl4ai-setup   # one-time, installs Playwright browsers
+uvx --python 3.13 --from "git+https://github.com/mordonez/liferay-docs-scraper" liferay-docs-scraper
+```
+
+Run this from anywhere -- it does not write into your current directory,
+see "Reference: the scraper in detail" below for exactly where it goes.
+
+**2. Install the skill into whatever project you're working in:**
+
+```bash
+npx skills add mordonez/liferay-docs-scraper --skill liferay-expert -a claude-code
+```
+
+You'll see:
+
+```
+◇  Installed 1 skill ───────────────────╮
+│                                       │
+│  ✓ liferay-expert (copied)            │
+│    → ./.claude/skills/liferay-expert  │
+│                                       │
+├───────────────────────────────────────╯
+```
+
+**3. Ask Claude Code a Liferay question**, e.g. "how do I configure a
+synonym set in Liferay search?" The skill finds the corpus, greps the
+`search` capability, reads `search-administration-and-tuning-synonym-sets.md`,
+and answers grounded in that page -- citing
+`https://learn.liferay.com/w/dxp/search/search-administration-and-tuning/synonym-sets`
+as the source.
+
+The corpus is shared across every project where you install the skill (see
+"OS default location" below), so step 1 is only ever needed once per
+machine -- rerun it later just to refresh, not per-project.
+
+**If you install the skill without doing step 1 first** (or its corpus goes
+stale), it notices and tells you what to run rather than guessing or
+answering ungrounded -- it never launches the ~30-40 min scrape on its own
+mid-conversation. See "Step 1/2" in `skills/liferay-expert/SKILL.md` for
+that check.
+
+## Reference: the scraper in detail
+
+Requires Python 3.10-3.13 (crawl4ai's Playwright dependency doesn't yet
+support 3.14) and [uv](https://docs.astral.sh/uv/).
+
+```bash
+# One-time: installs the Playwright/Chromium browser crawl4ai drives
+uvx --from crawl4ai crawl4ai-setup
+
+# From anywhere -- the corpus does NOT go in your current directory.
+# Not on PyPI yet, so install straight from GitHub:
+uvx --python 3.13 --from "git+https://github.com/mordonez/liferay-docs-scraper" liferay-docs-scraper
+```
+
+This takes roughly 30-40 minutes (BFS deep crawl of ~1900 pages across 14
+capabilities) and writes to one shared, per-user location (so it's the same
+corpus no matter which project you're in when the skill looks for it):
+
+| OS | Default location |
+|---|---|
+| macOS | `~/Library/Application Support/liferay-docs/` |
+| Linux | `~/.local/share/liferay-docs/` (or `$XDG_DATA_HOME/liferay-docs`) |
+| Windows | `%LOCALAPPDATA%\liferay-docs\` |
+
+Set `LIFERAY_DOCS_DIR` to override (e.g. to keep a project-local copy instead).
+
+Inside that directory:
+
+- `raw/{capability}/*.md` — the corpus, one file per page
+- `raw/_navigation/{capability}/*.md` — pure TOC pages, kept but deprioritized
+- `raw/_removed/{capability}/*.md` — pages confirmed gone from the live site
+- `reports/filtered/` — URL manifests, self-hosted prune log, run summary
+
+Re-run it anytime (weekly recommended) to refresh: it starts from zero every
+time, so it naturally picks up new pages, updates changed ones, and
+quarantines (never deletes) removed ones. If that directory is (or becomes)
+a git repo -- worth doing once, purely as a local diffing tool, nothing needs
+pushing anywhere -- it also runs `check-regressions` automatically afterward
+and flags any file that shrank by more than half or grew more than 3x versus
+the last commit (signals of a broken fetch); see
+`docs/adr/0001-crawl4ai-based-corpus-pipeline.md` for why that check exists.
+
+## Reference: the skill in detail
+
+```bash
+npx skills add mordonez/liferay-docs-scraper --skill liferay-expert
+```
+
+Or just copy `skills/liferay-expert/SKILL.md` into `.claude/skills/liferay-expert/`
+in any project. Claude Code picks it up automatically; the skill itself
+resolves `$LIFERAY_DOCS_DIR` (or the OS default above) to find the corpus,
+so it works the same regardless of which project you installed it into.
+
+## Why no bundled docs, no embeddings, no vector DB
+
+See `docs/adr/` for the full reasoning. Short version: the corpus is
+Liferay's copyrighted documentation text -- distributing the *tool* that
+scrapes public pages is a different, much lower-risk thing than a third
+party redistributing that text at scale. Plain grep + Read over ~1800
+well-organized Markdown files is fast enough that no search index is needed;
+add one later if that stops being true.

liferay_docs_scraper-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+liferay_docs_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+liferay_docs_scraper/check_regressions.py,sha256=_7k-5wNuxrhRJTbggUufwhcH8ECXByjUUXm_GELwHQ8,4114
+liferay_docs_scraper/classify_pages.py,sha256=yXo6eKLlsFpoM1CKzcZUNEWBN8RcQ1Nc_2qNgqnkLLs,1448
+liferay_docs_scraper/filter_urls.py,sha256=kHrskK2xyPO9tKCrjJUepLEN5k5xunG9I6iFg2J3fxU,5615
+liferay_docs_scraper/pipeline.py,sha256=pnivtmSXP7oGCjV7Ta52ZEjd1655tJMoveTKU7GcWqQ,19959
+liferay_docs_scraper-0.1.0.dist-info/METADATA,sha256=fSSrsFCFYg_Gbs8GEXcJfxR3kZaWDyqM0AFKUEDpzOc,5522
+liferay_docs_scraper-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+liferay_docs_scraper-0.1.0.dist-info/entry_points.txt,sha256=TFz1Q5Aj4QrlDOMZFWCHCN4qKDTqEG985t0JhoLYJVU,140
+liferay_docs_scraper-0.1.0.dist-info/RECORD,,

liferay_docs_scraper-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

liferay_docs_scraper-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+check-regressions = liferay_docs_scraper.check_regressions:main
+liferay-docs-scraper = liferay_docs_scraper.pipeline:main