pull-cli 0.1.0__py3-none-any.whl

pull_cli/envelope.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from dataclasses import asdict
+from typing import Any
+
+from .errors import PullError
+from .models import WarningRecord
+from .security import redact_value
+
+SCHEMA_VERSION = "1.0"
+
+
+def wants_json(explicit: bool) -> bool:
+    return explicit or os.environ.get("LLM", "").lower() == "true"
+
+
+def request_id() -> str:
+    return time.strftime("req_%Y%m%d_%H%M%S_") + uuid.uuid4().hex[:8]
+
+
+def make_envelope(
+    *,
+    ok: bool,
+    command: str,
+    target: dict[str, Any] | None = None,
+    result: dict[str, Any] | None = None,
+    warnings: list[WarningRecord | dict[str, Any]] | None = None,
+    errors: list[PullError | dict[str, Any]] | None = None,
+    metrics: dict[str, Any] | None = None,
+    request_id_value: str | None = None,
+) -> dict[str, Any]:
+    warning_records = [
+        warning.to_dict() if isinstance(warning, WarningRecord) else warning
+        for warning in (warnings or [])
+    ]
+    error_records = [error.to_record() if isinstance(error, PullError) else error for error in errors or []]
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "request_id": request_id_value or request_id(),
+        "ok": ok,
+        "command": command,
+        "target": target or {},
+        "result": result if ok else None,
+        "warnings": warning_records,
+        "errors": error_records,
+        "metrics": metrics or {},
+    }
+
+
+def emit_json(data: dict[str, Any]) -> None:
+    print(json.dumps(redact_value(data), ensure_ascii=False, separators=(",", ":")))
+
+
+def dataclass_dict(value: Any) -> dict[str, Any]:
+    return asdict(value)

pull_cli/errors.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+EXIT_SUCCESS = 0
+EXIT_VALIDATION = 10
+EXIT_AUTH = 20
+EXIT_SOURCE = 30
+EXIT_STRICT_PARTIAL = 40
+EXIT_IO = 50
+EXIT_INTERNAL = 90
+
+
+@dataclass
+class PullError(Exception):
+    code: str
+    message: str
+    exit_code: int = EXIT_INTERNAL
+    retryable: bool = False
+    suggested_action: str | None = None
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def __str__(self) -> str:
+        return f"{self.code}: {self.message}"
+
+    def to_record(self) -> dict[str, Any]:
+        return {
+            "code": self.code,
+            "message": self.message,
+            "retryable": self.retryable,
+            "suggested_action": self.suggested_action,
+            "details": self.details,
+        }
+
+
+def validation_error(
+    code: str,
+    message: str,
+    *,
+    suggested_action: str | None = None,
+    details: dict[str, Any] | None = None,
+) -> PullError:
+    return PullError(
+        code=code,
+        message=message,
+        exit_code=EXIT_VALIDATION,
+        suggested_action=suggested_action,
+        details=details or {},
+    )

pull_cli/extractor.py

@@ -0,0 +1,344 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from .assets import discover_asset_candidates, download_assets, skipped_asset_warnings
+from .clients.base import ConfluenceClient
+from .crawler import crawl_pages
+from .errors import EXIT_STRICT_PARTIAL, PullError
+from .html_normalizer import normalize_html
+from .links import rewrite_html_links
+from .macros import MacroContext, MacroRegistry
+from .markdown_writer import rendered_html_to_markdown
+from .models import (
+    CommentRecord,
+    ExtractionResult,
+    PageArtifact,
+    PageSummary,
+    PullOptions,
+    WarningRecord,
+)
+from .paths import relative_path, slugify
+from .security import redact_source_url_text, redact_text
+from .writer import (
+    page_markdown_header,
+    prepare_output_dir,
+    write_bundle,
+    write_diagnostics,
+    write_manifest,
+    write_page_artifact,
+)
+
+
+def extract(
+    *,
+    client: ConfluenceClient,
+    root: PageSummary,
+    options: PullOptions,
+) -> ExtractionResult:
+    prepare_output_dir(options.output, force=options.force, clean=options.clean)
+    summaries = crawl_pages(
+        client,
+        root,
+        tree=options.tree,
+        depth=options.depth,
+        max_pages=options.max_pages,
+    )
+    page_paths = _page_paths(summaries, options=options)
+    pages_by_id = {summary.page_id: summary for summary in summaries}
+    registry = MacroRegistry()
+    result = ExtractionResult(
+        output_dir=options.output,
+        manifest_path=options.output / "manifest.yaml",
+        bundle_path=options.output / "bundle.md" if options.write_bundle else None,
+        pages=[],
+        assets=[],
+        warnings=[],
+        links=[],
+        macros=[],
+    )
+    for summary in summaries:
+        page = client.get_page(summary.page_id)
+        page.order = summary.order
+        page.depth = summary.depth
+        page.parent_id = summary.parent_id
+        page.title = page.title or summary.title
+        page.url = page.url or summary.url
+        page_dir = page_paths[page.page_id].removesuffix("/index.md")
+        index_md = f"{page_dir}/index.md"
+        index_html = f"{page_dir}/index.html" if options.write_html else None
+        source_path = f"{page_dir}/source.storage.xml" if options.write_source and page.body_storage else None
+        page_json = f"{page_dir}/page.json"
+        comments, comment_warnings = _collect_comments(client, page.page_id, options=options)
+        comments_path = f"{page_dir}/comments.md" if comments else None
+        rendered = _select_rendered_body(page.body_view, page.body_export_view, page.body_storage)
+        normalized_html, html_warnings = normalize_html(
+            rendered,
+            source_page_id=page.page_id,
+        )
+        attachments = client.list_attachments(page.page_id)
+        candidates = discover_asset_candidates(
+            normalized_html,
+            page_id=page.page_id,
+            attachments=attachments,
+            options=options,
+        )
+        assets, asset_warnings = download_assets(
+            candidates,
+            page_id=page.page_id,
+            page_assets_dir=options.output / page_dir / "assets",
+            page_assets_path=f"{page_dir}/assets",
+            client=client,
+            extract_attachments=options.extract_attachments,
+        )
+        if options.no_assets:
+            asset_warnings.extend(skipped_asset_warnings(normalized_html, page_id=page.page_id))
+        rewritten_html, links, link_warnings = rewrite_html_links(
+            normalized_html,
+            page=page,
+            page_index_path=index_md,
+            pages_by_id=pages_by_id,
+            page_paths=page_paths,
+            assets=assets,
+            rewrite_links=options.rewrite_links,
+        )
+        if options.redact_manifest or options.redact_source_urls:
+            _redact_links(links, redact_source_urls=options.redact_source_urls)
+        if options.redact_source_urls:
+            rewritten_html, _redaction_warnings = normalize_html(
+                rewritten_html,
+                source_page_id=page.page_id,
+                redact_source_urls=True,
+            )
+        macro_context = MacroContext(
+            page_id=page.page_id,
+            attachments=attachments,
+            options=options,
+            child_links=_child_links(page, summaries, page_paths),
+        )
+        macros = registry.convert_all(page.body_storage, macro_context)
+        _enforce_strict_macros(macros, options=options)
+        macro_warnings = [warning for macro in macros for warning in macro.warnings]
+        visible_markdown = rendered_html_to_markdown(rewritten_html)
+        attachment_markdown = _attachment_markdown(assets, page_index_path=index_md)
+        if attachment_markdown:
+            visible_markdown = visible_markdown.rstrip() + "\n\n" + attachment_markdown + "\n"
+        macro_markdown = _macro_recovery_markdown(macros)
+        artifact = PageArtifact(
+            page=page,
+            order=page.order,
+            page_dir=page_dir,
+            index_md=index_md,
+            index_html=index_html,
+            source_path=source_path,
+            page_json=page_json,
+            markdown="",
+            html=rewritten_html,
+            assets=assets,
+            links=links,
+            macros=macros,
+            warnings=[*html_warnings, *asset_warnings, *link_warnings, *macro_warnings, *comment_warnings],
+            comments_path=comments_path,
+            comments=comments,
+        )
+        artifact.markdown = (
+            page_markdown_header(artifact, options=options)
+            + visible_markdown
+            + ("\n\n## Macro Recovery\n\n" + macro_markdown + "\n" if macro_markdown else "")
+        )
+        write_page_artifact(options.output, artifact, options=options)
+        result.pages.append(artifact)
+        result.assets.extend(assets)
+        result.links.extend(links)
+        result.macros.extend(macros)
+        result.warnings.extend(artifact.warnings)
+
+    unresolved = [
+        link.__dict__
+        for link in result.links
+        if link.status == "unresolved" or link.warning == "W_LINK_ANCHOR_UNRESOLVED"
+    ]
+    write_bundle(result, root_title=result.pages[0].page.title if result.pages else root.title, options=options)
+    if options.write_chunks:
+        _write_chunks(result)
+    write_diagnostics(options.output, result.warnings, unresolved)
+    write_manifest(
+        result,
+        options=options,
+        root_page_id=root.page_id,
+        base_url=client.base_url,
+        deployment_type=client.deployment_type,
+    )
+    result.metrics["api_calls"] = client.api_calls
+    result.metrics["pages"] = len(result.pages)
+    result.metrics["assets"] = len(result.assets)
+    return result
+
+
+def _select_rendered_body(view: str | None, export_view: str | None, storage: str | None) -> str:
+    return view or export_view or storage or ""
+
+
+def _macro_recovery_markdown(macros) -> str:
+    blocks = [macro.markdown.strip() for macro in macros if macro.markdown and macro.status != "ignored"]
+    return "\n\n".join(block for block in blocks if block)
+
+
+def _attachment_markdown(assets, *, page_index_path: str) -> str:
+    rows = []
+    for asset in assets:
+        if asset.attachment_id:
+            asset_link = relative_path(page_index_path, asset.local_path)
+            sidecars = (
+                ", ".join(
+                    f"`{sidecar}` ([open]({relative_path(page_index_path, sidecar)}))"
+                    for sidecar in asset.sidecars
+                )
+                or ""
+            )
+            rows.append(
+                "| "
+                + " | ".join(
+                    [
+                        asset.filename,
+                        f"`{asset.local_path}` ([open]({asset_link}))",
+                        asset.media_type or "",
+                        sidecars,
+                    ]
+                )
+                + " |"
+            )
+    if not rows:
+        return ""
+    return "\n".join(
+        [
+            "## Attachments",
+            "",
+            "| Filename | Local path | Media type | Extracted sidecars |",
+            "| --- | --- | --- | --- |",
+            *rows,
+        ]
+    )
+
+
+def _collect_comments(
+    client: ConfluenceClient, page_id: str, *, options: PullOptions
+) -> tuple[list[CommentRecord], list[WarningRecord]]:
+    if not options.comments:
+        return [], []
+    try:
+        return _unique_comments(client.list_comments(page_id)), []
+    except Exception as exc:  # noqa: BLE001
+        return [], [
+            WarningRecord(
+                code="W_COMMENTS_FETCH_FAILED",
+                message="Could not fetch Confluence comments for this page.",
+                source_page_id=page_id,
+                details={"reason": _redacted_warning_reason(exc, options=options)},
+            )
+        ]
+
+
+def _unique_comments(comments: list[CommentRecord]) -> list[CommentRecord]:
+    output: list[CommentRecord] = []
+    seen: set[str] = set()
+    for comment in comments:
+        if comment.comment_id and comment.comment_id in seen:
+            continue
+        if comment.comment_id:
+            seen.add(comment.comment_id)
+        output.append(comment)
+    return output
+
+
+def _redacted_warning_reason(exc: Exception, *, options: PullOptions) -> str:
+    reason = redact_text(str(exc))
+    return redact_source_url_text(reason) if options.redact_source_urls else reason
+
+
+def _redact_links(links, *, redact_source_urls: bool) -> None:
+    from .security import redact_text, sanitize_url
+
+    for link in links:
+        link.original = sanitize_url(link.original, redact_source_url=redact_source_urls) or redact_text(link.original)
+        link.normalized = sanitize_url(link.normalized, redact_source_url=redact_source_urls) or redact_text(link.normalized)
+
+
+def _enforce_strict_macros(macros, *, options: PullOptions) -> None:
+    strict = options.macro_policy == "strict" or options.unknown_macro == "error"
+    if not strict:
+        return
+    failures = [
+        {
+            "macro_id": macro.macro_id,
+            "name": macro.name,
+            "status": macro.status,
+            "warnings": [warning.code for warning in macro.warnings],
+        }
+        for macro in macros
+        if macro.status in {"placeholder", "error"} or macro.warnings
+    ]
+    if failures:
+        raise PullError(
+            code="ERR_INTERNAL_CONVERSION",
+            message="Strict macro policy rejected one or more partial macro conversions.",
+            exit_code=EXIT_STRICT_PARTIAL,
+            suggested_action="Use --macro-policy expand or --unknown-macro warn to allow placeholders.",
+            details={"macros": failures},
+        )
+
+
+def _page_paths(summaries: list[PageSummary], *, options: PullOptions) -> dict[str, str]:
+    layout = options.layout
+    if layout == "auto":
+        layout = "nested" if options.tree else "flat"
+    paths: dict[str, str] = {}
+    by_id = {summary.page_id: summary for summary in summaries}
+    for summary in summaries:
+        segment = f"{summary.order:04d}-{slugify(summary.title, fallback=summary.page_id)}"
+        if layout == "nested" and summary.parent_id and summary.parent_id in paths:
+            parent_dir = str(Path(paths[summary.parent_id]).parent).replace("\\", "/")
+            paths[summary.page_id] = f"{parent_dir}/{segment}/index.md"
+        elif layout == "nested" and summary.parent_id and summary.parent_id in by_id:
+            paths[summary.page_id] = f"pages/{segment}/index.md"
+        else:
+            paths[summary.page_id] = f"pages/{segment}/index.md"
+    return paths
+
+
+def _child_links(
+    page: PageSummary, summaries: list[PageSummary], page_paths: dict[str, str]
+) -> list[tuple[str, str]]:
+    links = []
+    source_index = page_paths.get(page.page_id, "")
+    for summary in summaries:
+        if summary.parent_id == page.page_id:
+            from .paths import relative_path
+
+            links.append((summary.title, relative_path(source_index, page_paths[summary.page_id])))
+    return links
+
+
+def _write_chunks(result: ExtractionResult) -> None:
+    import json
+
+    chunks_path = result.output_dir / "chunks.jsonl"
+    records = []
+    for artifact in result.pages:
+        paragraphs = [block.strip() for block in artifact.markdown.split("\n\n") if block.strip()]
+        for index, paragraph in enumerate(paragraphs, start=1):
+            records.append(
+                {
+                    "schema_version": "1.0",
+                    "chunk_id": f"{artifact.page.page_id}-{index:04d}",
+                    "page_id": artifact.page.page_id,
+                    "title": artifact.page.title,
+                    "source_path": artifact.index_md,
+                    "order": artifact.order,
+                    "text": paragraph,
+                }
+            )
+    chunks_path.write_text(
+        "".join(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" for record in records),
+        encoding="utf-8",
+    )

pull_cli/guide.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+
+def guide_payload() -> dict[str, object]:
+    return {
+        "schema_version": "1.0",
+        "commands": {
+            "pull": {
+                "usage": "pull PAGE_REF [OPTIONS]",
+                "selectors": ["PAGE_REF", "--page-id", "--url", "--space + --title"],
+                "resolution_order": [
+                    "explicit --page-id",
+                    "explicit --url",
+                    "positional URL",
+                    "positional numeric page ID",
+                    "--space + --title",
+                ],
+                "options": {
+                    "scope": ["--tree", "--depth N", "--max-pages N"],
+                    "output": [
+                        "-o/--output PATH",
+                        "--output-mode simple|full",
+                        "--force",
+                        "--clean",
+                        "--html/--no-html",
+                        "--source/--no-source",
+                        "--bundle/--no-bundle",
+                    ],
+                    "assets": ["--assets visible|page|all", "--no-assets", "--extract-attachments", "--diagram-sources"],
+                    "comments": ["--comments"],
+                    "links": ["--rewrite-links/--no-rewrite-links", "--follow-includes", "--follow-links same-tree|same-space|none"],
+                    "agent": ["--json", "LLM=true"],
+                },
+            },
+            "validate": {"usage": "pull validate MANIFEST_OR_OUTPUT_DIR [--json]"},
+            "guide": {"usage": "pull guide [--json]"},
+        },
+        "output_schema": {
+            "default_mode": "simple",
+            "mode_rules": {
+                "simple": "Default. Agent-facing output is the sanitized root AI Markdown, page index.md files, and downloaded assets/sidecars. Control files are still written for validation/provenance but are not linked from the root AI Markdown.",
+                "full": "Writes the current full evidence package: bundle.md, page index.html snapshots, source.storage.xml when available, page.json, manifests, diagnostics, and assets.",
+                "overrides": "--bundle/--no-bundle, --html/--no-html, --source/--no-source, and --chunks override mode defaults.",
+                "clean": "Use --clean when switching modes if you need the physical output tree to contain only files from the new mode.",
+            },
+            "simple_root_files": [
+                "<sanitized-root-page-title>.md",
+                "<sanitized-root-page-title>.yaml",
+                "manifest.yaml",
+                "diagnostics/warnings.jsonl",
+                "diagnostics/unresolved-links.md",
+            ],
+            "simple_per_page": ["index.md", "page.json", "assets/", "comments.md when --comments finds comments"],
+            "full_extra_files": ["bundle.md", "pages/*/index.html", "pages/*/source.storage.xml"],
+            "per_page": ["index.md", "page.json", "assets/", "comments.md when --comments finds comments"],
+            "ai_manifest": "Agent handoff files named from the sanitized root page title, with package-root path rules, hierarchical local page markdown paths, assets, sidecars, and diagnostics pointers.",
+            "manifest_paths": "Manifest and AI manifest paths are package-root-relative. Resolve them against the directory containing the root AI Markdown/YAML file, not the shell current working directory.",
+            "bundle_links": "Local links in bundle.md are rebased to package-root-relative paths.",
+            "comments": "--comments is opt-in. It fetches page and inline comments, writes page-local comments.md sidecars only when comments exist, and links them from agent-facing navigation.",
+        },
+        "json_envelope": {
+            "schema_version": "1.0",
+            "shape": ["schema_version", "request_id", "ok", "command", "target", "result", "warnings", "errors", "metrics"],
+            "failure_result": None,
+        },
+        "error_codes": [
+            "ERR_VALIDATION_REQUIRED",
+            "ERR_VALIDATION_AMBIGUOUS_PAGE",
+            "ERR_VALIDATION_INVALID_URL",
+            "ERR_VALIDATION_OUTPUT_EXISTS",
+            "ERR_AUTH_REQUIRED",
+            "ERR_AUTH_FORBIDDEN",
+            "ERR_AUTH_EXPIRED",
+            "ERR_SOURCE_PAGE_NOT_FOUND",
+            "ERR_SOURCE_BODY_UNAVAILABLE",
+            "ERR_SOURCE_TREE_TOO_LARGE",
+            "ERR_IO_CONNECTION",
+            "ERR_IO_TIMEOUT",
+            "ERR_IO_WRITE_FAILED",
+            "ERR_INTERNAL_CONVERSION",
+            "ERR_INTERNAL_API_RESPONSE",
+        ],
+        "warning_codes": [
+            "W_MACRO_UNKNOWN",
+            "W_MACRO_PARTIAL",
+            "W_MACRO_RENDER_EMPTY",
+            "W_ASSET_DOWNLOAD_FAILED",
+            "W_ASSET_SKIPPED_BY_POLICY",
+            "W_ASSET_DIAGRAM_SOURCE_NOT_FOUND",
+            "W_ATTACHMENT_TEXT_EXTRACTION_FAILED",
+            "W_LINK_UNRESOLVED",
+            "W_LINK_ANCHOR_UNRESOLVED",
+            "W_LINK_EXTERNAL_PRESERVED",
+            "W_PAGE_SKIPPED_PERMISSION",
+            "W_PAGE_SKIPPED_LIMIT",
+            "W_COMMENTS_FETCH_FAILED",
+            "W_BODY_REPRESENTATION_FALLBACK",
+            "W_DYNAMIC_MACRO_SNAPSHOT",
+            "W_SANITIZED_HTML",
+        ],
+        "examples": [
+            "pull 123456 -o pulled",
+            "pull --page-id 123456 --output-mode full -o pulled-full",
+            "pull \"https://example.atlassian.net/wiki/spaces/EA/pages/123456/Architecture\" -o pulled",
+            "pull --page-id 123456 --tree --depth 2 --assets all -o pulled-tree",
+            "pull --page-id 123456 --tree --comments -o pulled-comments",
+            "pull validate pulled-tree",
+            "LLM=true pull --page-id 123456 --json",
+        ],
+        "compatibility_notes": [
+            "PULL_* environment variables take precedence over config file values.",
+            "CONFPUB_URL, CONFPUB_USER, CONFPUB_TOKEN, and CONFPUB_SSL_VERIFY are accepted as compatibility fallbacks.",
+            "The CLI is read-only and does not call LLM services.",
+        ],
+    }

pull_cli/html_normalizer.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, NavigableString
+
+from .models import WarningRecord
+from .security import SECRET_KEY_PATTERN, redact_source_url_text, redact_text, sanitize_url
+
+WRITE_UI_SELECTORS = (
+    ".plugin_attachments_container",
+    ".plugin_attachments_upload_container",
+    ".plugin_attachments_table_container",
+    ".attachments-table-drop-zone",
+    ".download-all-link",
+    ".attachment-buttons",
+    "table.attachments",
+    ".labels-edit-container",
+    ".show-labels-editor",
+    ".editAttachmentLink",
+    ".removeAttachmentLink",
+)
+
+
+def normalize_html(
+    html: str, *, source_page_id: str, redact_source_urls: bool = False
+) -> tuple[str, list[WarningRecord]]:
+    soup = BeautifulSoup(html or "", "lxml")
+    warnings: list[WarningRecord] = []
+    removed_executable = False
+    for tag in soup.find_all(["script", "style", "iframe", "object", "embed", "form"]):
+        tag.decompose()
+        removed_executable = True
+    for selector in WRITE_UI_SELECTORS:
+        for tag in soup.select(selector):
+            tag.decompose()
+            removed_executable = True
+    for tag in soup.find_all("input"):
+        input_type = str(tag.get("type") or "").lower()
+        input_name = str(tag.get("name") or "")
+        if input_type == "hidden" or input_type == "file" or SECRET_KEY_PATTERN.search(input_name):
+            tag.decompose()
+            removed_executable = True
+    for tag in soup.find_all(True):
+        for attr in list(tag.attrs):
+            attr_lower = attr.lower()
+            if attr_lower.startswith("on") or SECRET_KEY_PATTERN.search(attr_lower):
+                del tag.attrs[attr]
+                removed_executable = True
+                continue
+            value = tag.attrs.get(attr)
+            if isinstance(value, str):
+                redacted = (
+                    sanitize_url(value, redact_source_url=redact_source_urls)
+                    if _is_source_url(value)
+                    else redact_text(value)
+                )
+                if redacted != value:
+                    tag.attrs[attr] = redacted
+                    removed_executable = True
+        for attr in ("href", "src", "data-file-src"):
+            value = tag.get(attr)
+            if isinstance(value, str) and value.strip().lower().startswith("javascript:"):
+                del tag.attrs[attr]
+                removed_executable = True
+                continue
+            if isinstance(value, str) and _is_source_url(value):
+                sanitized = sanitize_url(value, redact_source_url=redact_source_urls)
+                if sanitized != value:
+                    tag.attrs[attr] = sanitized
+                    removed_executable = True
+    if redact_source_urls:
+        for node in soup.find_all(string=True):
+            if isinstance(node, NavigableString):
+                redacted = redact_source_url_text(str(node))
+                if redacted != str(node):
+                    node.replace_with(redacted)
+                    removed_executable = True
+        for tag in soup.find_all("img"):
+            src = tag.get("src")
+            if isinstance(src, str) and _is_redacted_url(src) and not _has_accessible_label(tag):
+                tag.decompose()
+                removed_executable = True
+    if removed_executable:
+        warnings.append(
+            WarningRecord(
+                code="W_SANITIZED_HTML",
+                message="Executable or active HTML content was stripped from the rendered page snapshot.",
+                source_page_id=source_page_id,
+            )
+        )
+    body = soup.body or soup
+    return str(body), warnings
+
+
+def soup_from_html(html: str) -> BeautifulSoup:
+    return BeautifulSoup(html or "", "lxml")
+
+
+def _is_source_url(value: str) -> bool:
+    return value.strip().lower().startswith(("http://", "https://", "//", "/wiki/", "/download/"))
+
+
+def _is_redacted_url(value: str) -> bool:
+    return value.strip().lower() in {"<redacted-url>", "&lt;redacted-url&gt;"}
+
+
+def _has_accessible_label(tag) -> bool:
+    for attr in ("alt", "title", "aria-label"):
+        value = tag.get(attr)
+        if isinstance(value, str) and value.strip():
+            return True
+    return False