git2xml 0.1.0__py3-none-any.whl

git2xml/core.py

@@ -0,0 +1,633 @@
+"""Brief-assembly engine: scan -> fetch diffs -> render -> XML.
+
+The orchestration layer. ``build_brief`` drives a GitScanner through the
+phases (validate, scan, fetch diffs concurrently, render each file, emit XML)
+and returns the brief as a string; ``save_brief`` wraps it with an atomic
+write to disk. Concurrency policy (the diff/render semaphores) and
+diff-eligibility rules live here, not in the scanner. The per-file render
+helpers resolve a file's content source (working tree, index, or HEAD blob)
+and delegate formatting to ``utils``.
+"""
+
+import asyncio
+import logging
+import os
+import tempfile
+import time
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+from .git_scanner import GitScanner
+from .models import (
+    NO_DIFF,
+    ChangedFile,
+    DiffOmission,
+    DiffResult,
+    FileStatus,
+    Git2xmlCliConfig,
+    Git2xmlConfig,
+    Git2xmlError,
+    StagingState,
+)
+from .utils import (
+    build_commit_log_xml,
+    decode_bytes_bom_aware,
+    diff_exceeds_limit,
+    escape_xml_attr,
+    format_file_xml,
+    is_binary_bytes,
+    is_binary_file,
+    read_text_bom_aware,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _render_worktree_file_sync(
+    cwd_path: Path,
+    file_obj: ChangedFile,
+    status_label: str,
+    diff: DiffResult,
+    no_content: bool,
+    max_size: int,
+    strict_xml: bool,
+) -> str:
+    """Synchronous working-tree file resolution (symlink/missing/binary/size/content).
+
+    Runs the blocking working-tree resolution off the event loop.
+    """
+    full_path = cwd_path / file_obj.path
+
+    # Avoid including Symlink file content. Replicate Git behaviour with a clear content message
+    if full_path.is_symlink():
+        target = full_path.readlink()
+        symlink_content = f"Symlink pointing to: {target}"
+        return format_file_xml(
+            file_obj.path,
+            None if no_content else symlink_content,
+            status=status_label,
+            diff=diff,
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+
+    # Default: pull from the live working tree
+    if not full_path.exists():
+        return format_file_xml(
+            file_obj.path,
+            None,
+            status="omitted",
+            reason="deleted or unavailable",
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+
+    if is_binary_file(full_path):
+        # No diff passed: binary files are omitted whole (see diff-eligibility note
+        # in build_brief on why their diff was fetched but isn't rendered).
+        return format_file_xml(
+            file_obj.path,
+            None,
+            status=status_label,
+            reason="omitted - binary file detected",
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+
+    if full_path.stat().st_size > max_size:
+        return format_file_xml(
+            file_obj.path,
+            None,
+            status=status_label,
+            reason=f"omitted - file exceeds {max_size} bytes",
+            diff=diff,
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+
+    content = read_text_bom_aware(full_path)
+    return format_file_xml(
+        file_obj.path,
+        None if no_content else content,
+        status=status_label,
+        diff=diff,
+        indent="  ",
+        strict_xml=strict_xml,
+    )
+
+
+def _render_blob_sync(
+    path: str,
+    blob: bytes,
+    is_symlink: bool,
+    status_label: str,
+    diff: DiffResult,
+    no_content: bool,
+    strict_xml: bool,
+) -> str:
+    """Decode an already-fetched blob and format its <file> XML (CPU-bound).
+
+    Runs off the event loop via asyncio.to_thread. The blob bytes are fetched
+    by the async caller; this does only the binary check, BOM-aware decode, and
+    formatting - the work that would otherwise block the loop for large blobs.
+    Mirrors the post-fetch tail of the worktree path.
+    """
+    if is_symlink:
+        target = decode_bytes_bom_aware(blob)
+        symlink_content = f"Symlink pointing to: {target}"
+        return format_file_xml(
+            path,
+            None if no_content else symlink_content,
+            status=status_label,
+            diff=diff,
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+    if is_binary_bytes(blob):
+        # No diff passed: binary files are omitted whole (see diff-eligibility note
+        # in build_brief on why their diff was fetched but isn't rendered).
+        return format_file_xml(
+            path,
+            None,
+            status=status_label,
+            reason="omitted - binary file detected",
+            indent="  ",
+            strict_xml=strict_xml,
+        )
+    content = decode_bytes_bom_aware(blob)
+    return format_file_xml(
+        path,
+        None if no_content else content,
+        status=status_label,
+        diff=diff,
+        indent="  ",
+        strict_xml=strict_xml,
+    )
+
+
+async def _render_file_xml(
+    scanner: GitScanner,
+    cwd_path: Path,
+    file_obj: ChangedFile,
+    mode: str,
+    diff: DiffResult,
+    staged_only: bool,
+    no_content: bool,
+    max_size: int,
+    strict_xml: bool,
+    sem: asyncio.Semaphore,
+    tree_meta: Dict[str, Tuple[int, bool]],
+) -> str:
+    """Resolve one file's content source and return its formatted <file> XML.
+
+    Runs concurrently with other files, bounded by ``sem``.
+    """
+    status_label = file_obj.status.label
+
+    async with sem:
+        if mode == "pr":
+            # PR Briefs pull directly from Git memory (HEAD)
+            if file_obj.status == FileStatus.DELETED:
+                return format_file_xml(
+                    file_obj.path,
+                    None,
+                    status=status_label,
+                    diff=diff,
+                    indent="  ",
+                    strict_xml=strict_xml,
+                )
+
+            # Size and symlink-ness come from the batched ls-tree map, not
+            # per-file git calls. A path absent from the map yields size -1
+            # (intentionally not > max_size, so it falls through to the fetch
+            # below, which returns None and renders the "deleted or unavailable"
+            # omission - do not special-case it here).
+            blob_size, is_symlink = tree_meta.get(file_obj.path, (-1, False))
+            if blob_size > max_size:
+                return format_file_xml(
+                    file_obj.path,
+                    None,
+                    status=status_label,
+                    reason=f"omitted - file exceeds {max_size} bytes",
+                    diff=diff,
+                    indent="  ",
+                    strict_xml=strict_xml,
+                )
+
+            blob = await scanner.get_blob_at_head(file_obj.path)
+            if blob is None:
+                return format_file_xml(
+                    file_obj.path,
+                    None,
+                    status="omitted",
+                    reason="deleted or unavailable at HEAD",
+                    indent="  ",
+                    strict_xml=strict_xml,
+                )
+
+            return await asyncio.to_thread(
+                _render_blob_sync,
+                file_obj.path,
+                blob,
+                is_symlink,
+                status_label,
+                diff,
+                no_content,
+                strict_xml,
+            )
+
+        else:
+            # Commit Briefs
+            if file_obj.status == FileStatus.DELETED:
+                return format_file_xml(
+                    file_obj.path,
+                    None,
+                    status=status_label,
+                    diff=diff,
+                    indent="  ",
+                    strict_xml=strict_xml,
+                )
+
+            if staged_only:
+                # --staged: size + symlink mode come from the batched index map
+                # (mirrors PR mode), not per-file cat-file/ls-files probes. A path
+                # missing from the map (e.g. an unmerged index where write-tree
+                # failed) falls back to the per-file probes. Content is still read
+                # from the index per file.
+                blob_size, is_symlink = tree_meta.get(file_obj.path, (-1, False))
+                if blob_size < 0:
+                    blob_size = await scanner.get_blob_size_at_index(file_obj.path)
+                    is_symlink = await scanner.is_symlink_at_index(file_obj.path)
+                if blob_size > max_size:
+                    return format_file_xml(
+                        file_obj.path,
+                        None,
+                        status=status_label,
+                        reason=f"omitted - file exceeds {max_size} bytes",
+                        diff=diff,
+                        indent="  ",
+                        strict_xml=strict_xml,
+                    )
+
+                blob = await scanner.get_blob_at_index(file_obj.path)
+                if blob is None:
+                    return format_file_xml(
+                        file_obj.path,
+                        None,
+                        status="omitted",
+                        reason="deleted or unavailable in index",
+                        indent="  ",
+                        strict_xml=strict_xml,
+                    )
+
+                return await asyncio.to_thread(
+                    _render_blob_sync,
+                    file_obj.path,
+                    blob,
+                    is_symlink,
+                    status_label,
+                    diff,
+                    no_content,
+                    strict_xml,
+                )
+
+            # Working-tree branch: blocking FS I/O -> run in a thread so it
+            # doesn't stall the event loop and actually runs concurrently.
+            return await asyncio.to_thread(
+                _render_worktree_file_sync,
+                cwd_path,
+                file_obj,
+                status_label,
+                diff,
+                no_content,
+                max_size,
+                strict_xml,
+            )
+
+
+async def build_brief(config: Git2xmlConfig) -> str:
+    """Generate an XML brief of a repository's changes and return it as a string.
+
+    Scans the repository for changes, fetches their diffs concurrently, and
+    assembles an XML document mapping each changed file to its status, diff,
+    and (optionally) full content - formatted for consumption by an LLM.
+
+    Two modes determine what is summarized and where content is read from:
+
+    - ``"commit"``: changes in the working tree relative to HEAD. Content is
+      read from the working tree, or from the git index when ``staged_only``
+      is set. Includes staged, unstaged, and untracked files by default.
+    - ``"pr"``: all changes on the current branch relative to the configuration
+      ``base`` field (a ``base...HEAD`` diff). Content is read from the committed
+      blob at HEAD, and a structured ``<commit_log>`` of the branch's commits is
+      included. Uncommitted changes are not part of a PR brief.
+
+    Args:
+        config: ``Git2xmlConfig`` configuration object.
+
+    Returns:
+        The XML brief output as a string.
+
+    Raises:
+        Git2xmlError: If the repository is invalid, git is not installed, the
+            base ref cannot be resolved, a git command fails or times out, or
+            the output file cannot be written.
+    """
+
+    scanner = GitScanner(config)
+
+    cwd_path = Path(config.repo).resolve()
+    mode = config.command
+    base_branch = config.base
+    verbose = config.verbose
+    staged_only = config.staged
+    strict_xml = config.strict_xml
+    no_untracked = config.no_untracked
+    max_size = config.max_size
+    max_diff_size = config.max_diff_size
+    no_content = config.no_content
+    diff_semaphore_limit = config.diff_semaphore_limit
+    hide_repo_path = config.hide_repo_path
+
+    logger.info(f"Validating repository at {cwd_path}...")
+    await scanner.validate_repository()
+
+    if mode == "commit":
+        logger.info("Scanning commit changes...")
+        scan = await scanner.scan_commit_changes()
+        base_ref = "HEAD"
+
+        # Default to all changed files, filter only if user explicitly asks for --staged
+        if staged_only:
+            active_files = [
+                f
+                for f in scan.files
+                if f.staging in (StagingState.STAGED, StagingState.STAGED_AND_MODIFIED)
+            ]
+        else:
+            active_files = scan.files
+
+        # --no-untracked: drop untracked files (no-op under --staged, which already excludes them)
+        if no_untracked:
+            active_files = [f for f in active_files if f.status != FileStatus.UNTRACKED]
+    else:
+        base_branch = await scanner.resolve_base_ref()
+        logger.info("Scanning pr changes...")
+
+        # Check for uncommitted changes and warn the user
+        if await scanner.has_uncommitted_changes():
+            logger.warning(
+                "You have uncommitted changes! These will NOT be included in the PR brief."
+            )
+
+        scan = await scanner.scan_pr_changes(base_branch)
+        base_ref = f"{base_branch}...HEAD"
+        active_files = scan.files
+
+    if not active_files:
+        if mode == "commit":
+            if staged_only and not scan.has_staged:
+                logger.info(
+                    f"No staged changes in {cwd_path}. "
+                    "Stage files with 'git add' first, or omit --staged to include "
+                    "unstaged and untracked changes."
+                )
+            else:
+                logger.info(f"No staged or unstaged changes to commit in {cwd_path}.")
+        else:
+            logger.info(f"No commits between {base_branch} and HEAD in {cwd_path}.")
+        return ""
+
+    # Files that always get a diff fetched.
+    #
+    # MODIFIED/TYPE_CHANGED include binary files, whose diff the render path then
+    # discards (a binary file is omitted with no <diff>). We fetch it anyway:
+    # binary-ness isn't known until the bytes are sniffed at render time, and
+    # hoisting that sniff ahead of the fetch isn't worth the plumbing. The wasted
+    # fetch is bounded to ~max_diff_size by the streaming cap in
+    # GitScanner._run_git_capped, so the cost is one spawn + a capped read - not
+    # memory, not correctness.
+    diff_eligible_statuses = {
+        FileStatus.MODIFIED,
+        FileStatus.RENAMED,
+        FileStatus.COPIED,
+        FileStatus.DELETED,
+        FileStatus.TYPE_CHANGED,
+    }
+    # Under --no-content, a new file's diff is the only carrier of its change,
+    # so added/untracked files become diff-eligible too. In normal
+    # mode their content carries the change and no diff is needed.
+    if no_content:
+        diff_eligible_statuses |= {FileStatus.ADDED, FileStatus.UNTRACKED}
+
+    diff_eligible = [f for f in active_files if f.status in diff_eligible_statuses]
+    sem = asyncio.Semaphore(diff_semaphore_limit)
+
+    # PR rendering reads size + symlink mode from one batched ls-tree call
+    # instead of two git spawns per file; --staged does the same against the
+    # index (via write-tree). Default commit mode reads from the working tree
+    # per file, so the map stays empty there.
+    scoped = [f.path for f in active_files]
+    tree_meta: Dict[str, Tuple[int, bool]]
+    if mode == "pr":
+        tree_meta = await scanner.get_tree_metadata("HEAD", scoped)
+    elif staged_only:
+        tree_meta = await scanner.get_index_metadata(scoped)
+    else:
+        tree_meta = {}
+
+    async def _fetch_diff(f: ChangedFile) -> Tuple[str, DiffResult]:
+        try:
+            if f.status in (FileStatus.ADDED, FileStatus.UNTRACKED):
+                # Size pre-check stays OUTSIDE the semaphore: it's a metadata
+                # lookup (tree map, with a stat / cat-file fallback), not a diff
+                # fetch, and it must run even when the diff is skipped.
+                if await scanner.new_file_size(f, tree_meta) > max_size:
+                    return f.path, DiffResult(omission=DiffOmission.SIZE_EXCEEDED)
+                untracked = f.status == FileStatus.UNTRACKED
+            else:
+                untracked = False
+
+            # max_diff_size caps the diff, but only as output-shaping, not a
+            # memory guard: unlike content (whose size git reports via
+            # ls-tree/cat-file before we load it), a diff has no size until git
+            # computes it. The cap can only run post-fetch - git has already
+            # produced and buffered the full diff by the time we measure it - so
+            # an oversized diff is dropped from the OUTPUT (DIFF_SIZE_EXCEEDED),
+            # not prevented from being built. This keeps a pathological diff out
+            # of the brief / LLM context, but does not bound peak memory.
+            async with sem:
+                path, diff_text = await scanner.get_single_diff(f, base_ref, untracked=untracked)
+            if diff_exceeds_limit(diff_text, max_diff_size):
+                return path, DiffResult(
+                    omission=DiffOmission.DIFF_SIZE_EXCEEDED, limit=max_diff_size
+                )
+            return path, DiffResult(text=diff_text)
+        except Exception as e:
+            logger.warning("Failed to read diff for %s: %s", f.path, e)
+            return f.path, DiffResult(omission=DiffOmission.FETCH_ERROR)
+
+    diff_tasks = [_fetch_diff(f) for f in diff_eligible]
+    total_diffs = len(diff_tasks)
+    logger.info(f"Reading diffs for {total_diffs} file(s)...")
+    diffs: Dict[str, DiffResult] = {}
+    for i, coro in enumerate(asyncio.as_completed(diff_tasks), 1):
+        path, result = await coro
+        diffs[path] = result
+        if verbose:
+            logger.info(f"  Read diff {i}/{total_diffs}")
+
+    logger.info("Generating XML...")
+    tag_name = f"{mode}_brief"
+    repo_name = cwd_path.name
+
+    out_lines = []
+    # Only --strict-xml is guaranteed well-formed XML 1.0 (CDATA terminators are
+    # split, control chars escaped), so only strict mode gets an XML prologue.
+    # Default mode prioritizes byte-fidelity of code content and may use the
+    # markdown-fenced fallback, which is intentionally not well-formed XML - so
+    # it must NOT carry a declaration that promises conformance it doesn't meet.
+    if strict_xml:
+        out_lines.append('<?xml version="1.0" encoding="UTF-8"?>')
+
+    repo_path = cwd_path.name if hide_repo_path else cwd_path.as_posix()
+    out_lines.append(f'<{tag_name} repo="{escape_xml_attr(repo_path, strict_xml=strict_xml)}">')
+    out_lines.append(f"  <name>{escape_xml_attr(repo_name, strict_xml=strict_xml)}</name>")
+
+    # Inject the structured commit log for PRs
+    if mode == "pr":
+        current_branch = await scanner.get_current_branch()
+        logger.info("Fetching commit history...")
+        commits = await scanner.get_pr_commits(base_branch)
+
+        if commits:
+            if verbose:
+                for j, c in enumerate(commits, 1):
+                    logger.info(f"   Read commit {j}/{len(commits)} [{c['hash']}]: {c['subject']}")
+            commit_log_xml = build_commit_log_xml(
+                commits, current_branch, base_branch, strict_xml=strict_xml
+            )
+            out_lines.append(commit_log_xml)
+        elif verbose:
+            logger.info("   No commits found for this PR.")
+
+    content_sem = asyncio.Semaphore(diff_semaphore_limit)
+    render_tasks = [
+        _render_file_xml(
+            scanner,
+            cwd_path,
+            file_obj,
+            mode,
+            diffs.get(file_obj.path, NO_DIFF),
+            staged_only,
+            no_content,
+            max_size,
+            strict_xml,
+            content_sem,
+            tree_meta,
+        )
+        for file_obj in active_files
+    ]
+    # return_exceptions=True so a single file that raises (e.g. a transient
+    # PermissionError or a file lock while reading the working tree) degrades to
+    # an omitted <file> element instead of aborting the whole brief and losing
+    # every other successfully rendered file. Results are positional, so they
+    # zip back to active_files in order.
+    rendered = await asyncio.gather(*render_tasks, return_exceptions=True)
+    for file_obj, result in zip(active_files, rendered):
+        if isinstance(result, BaseException):
+            # Never swallow cancellation / interrupts (KeyboardInterrupt,
+            # SystemExit, CancelledError) - those are not Exception subclasses.
+            if not isinstance(result, Exception):
+                raise result
+            detail = " ".join(str(result).split())
+            if len(detail) > 200:
+                detail = detail[:197] + "..."
+            reason = f"error rendering file: {type(result).__name__}"
+            if detail:
+                reason += f": {detail}"
+            logger.warning("Failed to render %s: %s", file_obj.path, result)
+            out_lines.append(
+                format_file_xml(
+                    file_obj.path,
+                    None,
+                    status="omitted",
+                    reason=reason,
+                    indent="  ",
+                    strict_xml=strict_xml,
+                )
+            )
+        else:
+            out_lines.append(result)
+
+    out_lines.append(f"</{tag_name}>")
+
+    # Return the joined string instead of writing to disk
+    return "\n".join(out_lines) + "\n"
+
+
+def _swap_temp(temp_path: Path, out_path: Path) -> None:
+    """Atomically move ``temp_path`` onto ``out_path``, retrying briefly.
+
+    ``os.replace`` is atomic, but on Windows antivirus and search-indexer hooks
+    can hold a transient lock on a just-closed file, making the replace fail with
+    a spurious ``PermissionError``. The lock clears in milliseconds, so we retry a
+    few times before giving up and re-raising the last error.
+    """
+    for attempt in range(3):
+        try:
+            os.replace(temp_path, out_path)
+            return
+        except PermissionError:
+            if attempt == 2:
+                raise
+            time.sleep(0.1)
+
+
+async def save_brief(config: Git2xmlCliConfig) -> Optional[Path]:
+    """Generate a brief and atomically write it to disk.
+
+    Returns:
+        Path of the created output file. The brief is written to the configuration ``output`` field, resolved against the
+        current working directory (the directory the command was run from),
+        not against ``cwd`` (the repo). So ``--repo /elsewhere`` still writes
+        the brief into your current directory. Missing parent directories in
+        the configuration ``output`` field are created. If there are no changes
+        to summarize, logs an informational message and writes nothing.
+    """
+    xml_content = await build_brief(config)
+
+    if not xml_content:
+        return None
+
+    out_path = (Path.cwd() / config.output).resolve()
+    temp_path: Optional[Path] = None
+
+    try:
+        # Create any missing parent directories (e.g. --output reports/brief.xml)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to a collision-free temporary file first
+        # Note: NamedTemporaryFile creates at 0600; os.replace preserves it, so the
+        # brief is owner-readable only - intentional, since it can contain source.
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", newline="", dir=out_path.parent, delete=False
+        ) as f:
+            temp_path = Path(f.name)
+            f.write(xml_content)
+
+        # Atomically swap the temporary file into place (retries on transient
+        # Windows AV/indexer locks; see _swap_temp).
+        _swap_temp(temp_path, out_path)
+        temp_path = None
+    except OSError as e:
+        raise Git2xmlError(f"Could not write brief to {out_path}: {e}") from e
+    finally:
+        if temp_path is not None and temp_path.exists():
+            try:
+                temp_path.unlink()
+            except OSError:
+                pass
+
+    logger.info(f"Successfully generated {config.command} brief: {out_path.absolute()}")
+    return out_path