getdocs 0.1.0__py3-none-any.whl

getdocs/jobs.py

@@ -0,0 +1,204 @@
+"""Jobs: run Crawls as subprocesses and track their state.
+
+Per ADR-0002, a job is the getdocs CLI run with --format jsonl; its stdout
+stream is the event protocol. One subprocess per Crawl sidesteps the
+one-reactor-per-process constraint and isolates crashes.
+"""
+
+import asyncio
+import json
+import sys
+import tempfile
+import uuid
+from dataclasses import dataclass, field
+
+_BOOL_FLAGS = {
+    "allow_backward": "--allow-backward",
+    "allow_subdomains": "--allow-subdomains",
+    "ignore_robots": "--ignore-robots",
+    "keep_html": "--keep-html",
+}
+_VALUE_FLAGS = {
+    "limit": "--limit",
+    "depth": "--depth",
+    "delay": "--delay",
+    "concurrency": "--concurrency",
+    "render": "--render",
+    "selector": "--selector",
+}
+_LIST_FLAGS = {
+    "include_paths": "--include-paths",
+    "exclude_paths": "--exclude-paths",
+}
+_SITEMAP_FLAGS = {"off": "--no-sitemap", "only": "--sitemap-only"}
+
+# JSONL lines carry whole pages (and raw HTML with keep_html).
+_STREAM_LIMIT = 32 * 1024 * 1024
+
+
+def build_args(options: dict, output_dir: str) -> list[str]:
+    seeds = options.get("urls") or [options["url"]]
+    args = ["crawl", *seeds, "--format", "jsonl", "-o", output_dir]
+    for key, flag in _VALUE_FLAGS.items():
+        if options.get(key) is not None:
+            args += [flag, str(options[key])]
+    for key, flag in _BOOL_FLAGS.items():
+        if options.get(key):
+            args.append(flag)
+    for key, flag in _LIST_FLAGS.items():
+        for value in options.get(key) or []:
+            args += [flag, value]
+    sitemap_flag = _SITEMAP_FLAGS.get(options.get("sitemap", ""))
+    if sitemap_flag:
+        args.append(sitemap_flag)
+    return args
+
+
+@dataclass
+class CrawlJob:
+    id: str
+    seeds: list[str]
+    status: str = "running"  # running | completed | failed | cancelled
+    pages: list[dict] = field(default_factory=list)
+    manifest: dict | None = None
+    error: str | None = None
+    webhook_failures: int = 0
+
+
+class JobManager:
+    def __init__(self):
+        self.jobs: dict[str, CrawlJob] = {}
+        self._tasks: dict[str, asyncio.Task] = {}
+        self._processes: dict[str, asyncio.subprocess.Process] = {}
+        self._subscribers: dict[str, list[asyncio.Queue]] = {}
+
+    def start(self, options: dict) -> CrawlJob:
+        seeds = options.get("urls") or [options["url"]]
+        job = CrawlJob(id=uuid.uuid4().hex, seeds=seeds)
+        self.jobs[job.id] = job
+        output_dir = tempfile.mkdtemp(prefix=f"getdocs-{job.id[:8]}-")
+        args = build_args(options, output_dir=output_dir)
+        self._tasks[job.id] = asyncio.ensure_future(
+            self._run(job, args, webhook=options.get("webhook"))
+        )
+        return job
+
+    def get(self, job_id: str) -> CrawlJob | None:
+        return self.jobs.get(job_id)
+
+    def cancel(self, job_id: str) -> CrawlJob | None:
+        """Cancel a running job (terminates its subprocess, keeps partial
+        results). A no-op on finished jobs; None for unknown ids."""
+        job = self.jobs.get(job_id)
+        if job is None:
+            return None
+        if job.status == "running":
+            job.status = "cancelled"
+            process = self._processes.get(job_id)
+            if process is not None and process.returncode is None:
+                process.terminate()
+        return job
+
+    async def wait(self, job_id: str) -> CrawlJob:
+        await self._tasks[job_id]
+        return self.jobs[job_id]
+
+    def _publish(self, job_id: str, event: dict) -> None:
+        for queue in self._subscribers.get(job_id, []):
+            queue.put_nowait(event)
+
+    async def stream(self, job_id: str):
+        """Yield a job's events: a replay of everything so far, then live
+        page events, ending with the manifest (when one was produced).
+
+        The queue is attached in the same event-loop step as the replay
+        snapshot, so no event is missed or duplicated around the boundary.
+        """
+        job = self.jobs[job_id]
+        queue: asyncio.Queue = asyncio.Queue()
+        self._subscribers.setdefault(job_id, []).append(queue)
+        try:
+            replay = list(job.pages)
+            finished = job.status != "running"
+            for record in replay:
+                yield {"type": "page", **record}
+            if finished:
+                if job.manifest is not None:
+                    yield {"type": "manifest", **job.manifest}
+                return
+            while True:
+                event = await queue.get()
+                if event["type"] == "end":
+                    return
+                yield event
+                if event["type"] == "manifest":
+                    return
+        finally:
+            self._subscribers[job_id].remove(queue)
+
+    async def _deliver(self, job: CrawlJob, url: str, payload: dict) -> None:
+        """Bounded-retry webhook POST; failures are recorded, never raised."""
+        import httpx
+
+        for attempt in range(3):
+            try:
+                async with httpx.AsyncClient(timeout=5) as client:
+                    response = await client.post(url, json=payload)
+                if response.status_code < 400:
+                    return
+            except httpx.HTTPError:
+                pass
+            await asyncio.sleep(0.05 * (attempt + 1))
+        job.webhook_failures += 1
+
+    async def _run(self, job: CrawlJob, args: list[str], webhook: str | None = None) -> None:
+        if webhook:
+            await self._deliver(
+                job, webhook, {"event": "started", "id": job.id, "seeds": job.seeds}
+            )
+        process = await asyncio.create_subprocess_exec(
+            sys.executable, "-m", "getdocs", *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            limit=_STREAM_LIMIT,
+        )
+        self._processes[job.id] = process
+        async for line in process.stdout:
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if record.get("type") == "page":
+                page = {k: v for k, v in record.items() if k != "type"}
+                job.pages.append(page)
+                self._publish(job.id, record)
+                if webhook:
+                    await self._deliver(
+                        job, webhook, {"event": "page", "id": job.id, "page": page}
+                    )
+            elif record.get("type") == "manifest":
+                job.manifest = {k: v for k, v in record.items() if k != "type"}
+                self._publish(job.id, record)
+        stderr = await process.stderr.read()
+        returncode = await process.wait()
+        if job.status == "cancelled":
+            pass  # keep the cancelled status and partial pages
+        elif returncode == 0:
+            job.status = "completed"
+        else:
+            job.status = "failed"
+            job.error = stderr.decode(errors="replace").strip()[-2000:] or (
+                f"crawl exited with code {returncode}"
+            )
+        self._publish(job.id, {"type": "end", "status": job.status})
+        if webhook:
+            await self._deliver(
+                job,
+                webhook,
+                {
+                    "event": "completed",
+                    "id": job.id,
+                    "status": job.status,
+                    "manifest": job.manifest,
+                },
+            )

getdocs/navharvest.py

@@ -0,0 +1,242 @@
+"""Nav harvesting: capture the original site's Nav Order and Reading Order.
+
+Per ADR-0004, three signals are harvested from each fetched Page's raw HTML
+and merged at Crawl end: header tabs and sidebar trees build the Nav Order
+(grouping, labels, nesting); prev/next link chains are authoritative for the
+linear Reading Order. Everything is plain dicts so harvests serialize
+directly into the resume state and the Manifest.
+
+Node shape: {"title": str, "url": str | None, "children": [node, ...]}
+Harvest shape: {"tree": [node], "tabs": [node], "prev": url?, "next": url?}
+"""
+
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from getdocs.urlnorm import normalize
+
+_TAB_SELECTORS = [
+    ".md-tabs a.md-tabs__link",        # MkDocs Material tabs
+    "nav.navbar a.navbar__item[href]",  # Docusaurus navbar
+]
+_SIDEBAR_SELECTORS = [
+    "nav.md-nav--primary",             # MkDocs Material
+    "nav.menu",                        # Docusaurus
+    "div.sphinxsidebarwrapper",        # Sphinx
+    "aside nav",
+    "aside",
+    '[class*="sidebar"] nav',
+]
+_PREV_SELECTORS = ['a[rel="prev"]', "a.md-footer__link--prev", "a.pagination-nav__link--prev"]
+_NEXT_SELECTORS = ['a[rel="next"]', "a.md-footer__link--next", "a.pagination-nav__link--next"]
+
+
+def _node(title: str, url: str | None, children: list) -> dict:
+    return {"title": title, "url": url, "children": children}
+
+
+def _own_link(li, nested_ul):
+    for a in li.find_all("a", href=True):
+        if nested_ul is None or nested_ul not in a.parents:
+            return a
+    return None
+
+
+def _parse_list(ul, page_url: str) -> list[dict]:
+    nodes = []
+    for li in ul.find_all("li", recursive=False):
+        nested = li.find("ul")
+        link = _own_link(li, nested)
+        if link is not None:
+            title = link.get_text(strip=True)
+            url = urljoin(page_url, link["href"])
+        else:
+            label = li.find("label")
+            title = (label or li).find(string=True, recursive=bool(label))
+            title = (title or "").strip()
+            url = None
+        children = _parse_list(nested, page_url) if nested else []
+        if title or children:
+            nodes.append(_node(title, url, children))
+    return nodes
+
+
+def _first_href(soup, selectors: list[str], page_url: str) -> str | None:
+    for selector in selectors:
+        el = soup.select_one(selector)
+        if el is not None and el.get("href"):
+            return urljoin(page_url, el["href"])
+    return None
+
+
+def harvest_nav(html: str, page_url: str) -> dict:
+    soup = BeautifulSoup(html, "html.parser")
+
+    tabs = []
+    for selector in _TAB_SELECTORS:
+        links = soup.select(selector)
+        if links:
+            tabs = [
+                _node(a.get_text(strip=True), urljoin(page_url, a["href"]), [])
+                for a in links
+                if a.get("href") and a.get_text(strip=True)
+            ]
+            break
+
+    tree: list[dict] = []
+    for selector in _SIDEBAR_SELECTORS:
+        container = soup.select_one(selector)
+        if container is not None:
+            ul = container if container.name == "ul" else container.find("ul")
+            if ul is not None:
+                tree = _parse_list(ul, page_url)
+                if tree:
+                    break
+
+    return {
+        "tree": tree,
+        "tabs": tabs,
+        "prev": _first_href(soup, _PREV_SELECTORS, page_url),
+        "next": _first_href(soup, _NEXT_SELECTORS, page_url),
+    }
+
+
+# -- merging ----------------------------------------------------------------
+
+
+def _count_nodes(nodes: list[dict]) -> int:
+    return sum(1 + _count_nodes(n["children"]) for n in nodes)
+
+
+def _index_tree(nodes: list[dict], index: dict) -> None:
+    for node in nodes:
+        if node["url"]:
+            index.setdefault(normalize(node["url"]), node)
+        _index_tree(node["children"], index)
+
+
+def _first_seen_merge(skeleton: list[dict], index: dict, other: list[dict]) -> None:
+    """Attach nodes unseen by the skeleton under their (known) parent."""
+
+    def walk(nodes: list[dict], parent_children: list[dict]):
+        for node in nodes:
+            norm = normalize(node["url"]) if node["url"] else None
+            if norm and norm in index:
+                walk(node["children"], index[norm]["children"])
+            elif norm:
+                copy = _node(node["title"], node["url"], [])
+                parent_children.append(copy)
+                index[norm] = copy
+                walk(node["children"], copy["children"])
+            else:
+                walk(node["children"], parent_children)
+
+    walk(other, skeleton)
+
+
+def _attach_tabs(tabs: list[dict], roots: list[dict]) -> list[dict]:
+    """Tabs become the top level; existing roots nest under the tab whose
+    URL path is their prefix. A root that IS a tab's page merges into it."""
+    tab_nodes = [_node(t["title"], t["url"], []) for t in tabs]
+    leftovers = []
+    for root in roots:
+        target = None
+        if root["url"]:
+            for tab in tab_nodes:
+                tab_path = (tab["url"] or "").rstrip("/")
+                if tab_path and (root["url"].rstrip("/") + "/").startswith(tab_path + "/"):
+                    target = tab
+                    break
+        if target is None:
+            leftovers.append(root)
+        elif root["url"] and tab_nodes and normalize(root["url"]) == normalize(target["url"]):
+            target["children"].extend(root["children"])
+        else:
+            target["children"].append(root)
+    return tab_nodes + leftovers
+
+
+def _prune(nodes: list[dict], written: set[str]) -> list[dict]:
+    """Un-crawled nodes keep their label but lose the link; label-only nodes
+    without children are dropped."""
+    result = []
+    for node in nodes:
+        children = _prune(node["children"], written)
+        url = node["url"] if node["url"] and normalize(node["url"]) in written else None
+        if url is None and not children:
+            continue
+        result.append(_node(node["title"], url, children))
+    return result
+
+
+def _traversal(nodes: list[dict]) -> list[str]:
+    urls = []
+    for node in nodes:
+        if node["url"]:
+            urls.append(node["url"])
+        urls.extend(_traversal(node["children"]))
+    return urls
+
+
+def _chain_sequences(harvests: list[dict], written: set[str]) -> list[str]:
+    """Assemble prev/next links into ordered chains of normalized URLs."""
+    next_of: dict[str, str] = {}
+    has_incoming: set[str] = set()
+    for harvest in harvests:
+        page = normalize(harvest["page"])
+        if harvest.get("next"):
+            target = normalize(harvest["next"])
+            next_of.setdefault(page, target)
+            has_incoming.add(target)
+        if harvest.get("prev"):
+            source = normalize(harvest["prev"])
+            next_of.setdefault(source, page)
+            has_incoming.add(page)
+
+    sequence, visited = [], set()
+    heads = [p for p in next_of if p not in has_incoming]
+    for head in heads:
+        current: str | None = head
+        while current and current not in visited:
+            visited.add(current)
+            if current in written:
+                sequence.append(current)
+            current = next_of.get(current)
+    return sequence
+
+
+def merge_harvests(
+    harvests: list[dict], written_urls: list[str]
+) -> tuple[list[dict], list[str]]:
+    """Merge per-page harvests into (nav tree, reading order).
+
+    harvests: [{"page": url, "tree": [...], "tabs": [...], "prev", "next"}]
+    written_urls: Pages actually written, in crawl order (original URLs).
+    """
+    written = {normalize(u) for u in written_urls}
+    original = {normalize(u): u for u in written_urls}
+
+    trees = [h for h in harvests if h["tree"]]
+    skeleton: list[dict] = []
+    if trees:
+        skeleton = max(trees, key=lambda h: _count_nodes(h["tree"]))["tree"]
+        index: dict = {}
+        _index_tree(skeleton, index)
+        for harvest in trees:
+            if harvest["tree"] is not skeleton:
+                _first_seen_merge(skeleton, index, harvest["tree"])
+
+    tabs = next((h["tabs"] for h in harvests if h["tabs"]), [])
+    if tabs:
+        skeleton = _attach_tabs(tabs, skeleton)
+
+    nav = _prune(skeleton, written)
+
+    ordered = _chain_sequences(harvests, written)
+    for url in _traversal(nav) + written_urls:
+        norm = normalize(url)
+        if norm in written and norm not in ordered:
+            ordered.append(norm)
+    reading_order = [original[norm] for norm in dict.fromkeys(ordered)]
+    return nav, reading_order

getdocs/output.py

@@ -0,0 +1,191 @@
+"""Output: Page records to a .md tree with YAML frontmatter, plus the Manifest."""
+
+import json
+import posixpath
+import re
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+import yaml
+
+from getdocs.urlnorm import normalize
+
+
+@dataclass(frozen=True)
+class PageRecord:
+    url: str
+    title: str
+    markdown: str
+    status: int
+    crawled_at: str
+    canonical: str | None = None
+    html: str | None = None
+
+
+class FileTreeWriter:
+    def __init__(self, output_dir: Path):
+        self.output_dir = Path(output_dir)
+        self.page_count = 0
+
+    def path_for(self, url: str) -> Path:
+        # Decode percent-escapes per segment so %20 doesn't end up in file
+        # names (and %2F can't smuggle in extra directory levels).
+        segments = [
+            unquote(segment).replace("/", "_")
+            for segment in urlsplit(url).path.split("/")
+            if segment
+        ]
+        path = "/".join(segments) or "index"
+        return self.output_dir / f"{path}.md"
+
+    def write_page(self, record: PageRecord) -> Path:
+        target = self.path_for(record.url)
+        target.parent.mkdir(parents=True, exist_ok=True)
+
+        frontmatter = {
+            k: v
+            for k, v in asdict(record).items()
+            if k not in ("markdown", "html") and v is not None
+        }
+        target.write_text(
+            "---\n"
+            + yaml.safe_dump(frontmatter, sort_keys=False)
+            + "---\n\n"
+            + record.markdown
+            + "\n"
+        )
+        if record.html is not None:
+            target.with_suffix(".html").write_text(record.html)
+        self.page_count += 1
+        return target
+
+    def write_manifest(
+        self,
+        seeds: list[str],
+        errors: list[dict] | None = None,
+        truncated: bool = False,
+        skipped: list[dict] | None = None,
+        shells: list[str] | None = None,
+        nav: list[dict] | None = None,
+        reading_order: list[str] | None = None,
+        media_skipped: list[dict] | None = None,
+    ) -> Path:
+        target = self.output_dir / "crawl.json"
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(
+            json.dumps(
+                {
+                    "seeds": seeds,
+                    "page_count": self.page_count,
+                    "errors": errors or [],
+                    "skipped": skipped or [],
+                    "shells": shells or [],
+                    "truncated": truncated,
+                    "nav": nav or [],
+                    "reading_order": reading_order or [],
+                    "media_skipped": media_skipped or [],
+                },
+                indent=2,
+            )
+            + "\n"
+        )
+        return target
+
+
+_MD_LINK_RE = re.compile(r"\((https?://[^)\s]+)\)")
+
+
+def relink_pages(writer: FileTreeWriter, written_urls: list[str]) -> None:
+    """Rewrite links between crawled Pages into relative .md paths.
+
+    Runs at Crawl end, when the full set of written Pages is known. Links
+    to anything else — external sites, un-crawled pages, hotlinked media —
+    keep their absolute URLs.
+    """
+    targets = {
+        normalize(url): writer.path_for(url).relative_to(writer.output_dir).as_posix()
+        for url in written_urls
+    }
+
+    for url in written_urls:
+        page_path = writer.path_for(url)
+        page_dir = page_path.relative_to(writer.output_dir).parent.as_posix()
+
+        def rewrite(match):
+            link, _, fragment = match.group(1).partition("#")
+            target = targets.get(normalize(link))
+            if target is None:
+                return match.group(0)
+            relative = posixpath.relpath(target, start=page_dir)
+            return f"({relative}{'#' + fragment if fragment else ''})"
+
+        text = page_path.read_text()
+        rewritten = _MD_LINK_RE.sub(rewrite, text)
+        if rewritten != text:
+            page_path.write_text(rewritten)
+
+
+class AssetStore:
+    """Downloaded Assets land under _media/<host>/<decoded path>."""
+
+    def __init__(self, output_dir: Path):
+        self.output_dir = Path(output_dir)
+
+    def save(self, url: str, body: bytes) -> str:
+        parts = urlsplit(url)
+        segments = [
+            unquote(s).replace("/", "_") for s in parts.path.split("/") if s
+        ] or ["asset"]
+        relpath = "/".join(["_media", parts.netloc, *segments])
+        target = self.output_dir / relpath
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(body)
+        return relpath
+
+
+class JsonlWriter:
+    """One typed JSON record per line; the Manifest is the final record.
+
+    This stream is the process-boundary protocol the future API service
+    consumes (ADR-0002) — record shape changes are contract changes.
+    """
+
+    def __init__(self, stream):
+        self.stream = stream
+        self.page_count = 0
+
+    def _emit(self, record: dict) -> None:
+        self.stream.write(json.dumps(record, ensure_ascii=False) + "\n")
+        self.stream.flush()
+
+    def write_page(self, record: PageRecord) -> None:
+        fields = {k: v for k, v in asdict(record).items() if v is not None}
+        self._emit({"type": "page", **fields})
+        self.page_count += 1
+
+    def write_manifest(
+        self,
+        seeds: list[str],
+        errors: list[dict] | None = None,
+        truncated: bool = False,
+        skipped: list[dict] | None = None,
+        shells: list[str] | None = None,
+        nav: list[dict] | None = None,
+        reading_order: list[str] | None = None,
+        media_skipped: list[dict] | None = None,
+    ) -> None:
+        self._emit(
+            {
+                "type": "manifest",
+                "seeds": seeds,
+                "page_count": self.page_count,
+                "errors": errors or [],
+                "skipped": skipped or [],
+                "shells": shells or [],
+                "truncated": truncated,
+                "nav": nav or [],
+                "reading_order": reading_order or [],
+                "media_skipped": media_skipped or [],
+            }
+        )