gitingest-tddschn 0.3.1__py3-none-any.whl

gitingest/__init__.py

@@ -0,0 +1,5 @@
+"""Gitingest: A package for ingesting data from Git repositories."""
+
+from gitingest.entrypoint import ingest, ingest_async
+
+__all__ = ["ingest", "ingest_async"]

gitingest/__main__.py

@@ -0,0 +1,204 @@
+"""Command-line interface (CLI) for Gitingest."""
+
+# pylint: disable=no-value-for-parameter
+from __future__ import annotations
+
+import asyncio
+from typing import TypedDict
+
+import click
+from typing_extensions import Unpack
+
+from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_NAME
+from gitingest.entrypoint import ingest_async
+
+# Import logging configuration first to intercept all logging
+from gitingest.utils.logging_config import get_logger
+
+# Initialize logger for this module
+logger = get_logger(__name__)
+
+
+class _CLIArgs(TypedDict):
+    source: str
+    max_size: int
+    exclude_pattern: tuple[str, ...]
+    include_pattern: tuple[str, ...]
+    branch: str | None
+    include_gitignored: bool
+    include_submodules: bool
+    token: str | None
+    output: str | None
+
+
+@click.command()
+@click.argument("source", type=str, default=".")
+@click.option(
+    "--max-size",
+    "-s",
+    default=MAX_FILE_SIZE,
+    show_default=True,
+    help="Maximum file size to process in bytes",
+)
+@click.option("--exclude-pattern", "-e", multiple=True, help="Shell-style patterns to exclude.")
+@click.option(
+    "--include-pattern",
+    "-i",
+    multiple=True,
+    help="Shell-style patterns to include.",
+)
+@click.option("--branch", "-b", default=None, help="Branch to clone and ingest")
+@click.option(
+    "--include-gitignored",
+    is_flag=True,
+    default=False,
+    help="Include files matched by .gitignore and .gitingestignore",
+)
+@click.option(
+    "--include-submodules",
+    is_flag=True,
+    help="Include repository's submodules in the analysis",
+    default=False,
+)
+@click.option(
+    "--token",
+    "-t",
+    envvar="GITHUB_TOKEN",
+    default=None,
+    help=(
+        "GitHub personal access token (PAT) for accessing private repositories. "
+        "If omitted, the CLI will look for the GITHUB_TOKEN environment variable."
+    ),
+)
+@click.option(
+    "--output",
+    "-o",
+    default=None,
+    help="Output file path (default: digest.txt in current directory). Use '-' for stdout.",
+)
+def main(**cli_kwargs: Unpack[_CLIArgs]) -> None:
+    """Run the CLI entry point to analyze a repo / directory and dump its contents.
+
+    Parameters
+    ----------
+    **cli_kwargs : Unpack[_CLIArgs]
+        A dictionary of keyword arguments forwarded to ``ingest_async``.
+
+    Notes
+    -----
+    See ``ingest_async`` for a detailed description of each argument.
+
+    Examples
+    --------
+    Basic usage:
+        $ gitingest
+        $ gitingest /path/to/repo
+        $ gitingest https://github.com/user/repo
+
+    Output to stdout:
+        $ gitingest -o -
+        $ gitingest https://github.com/user/repo --output -
+
+    With filtering:
+        $ gitingest -i "*.py" -e "*.log"
+        $ gitingest --include-pattern "*.js" --exclude-pattern "node_modules/*"
+
+    Private repositories:
+        $ gitingest https://github.com/user/private-repo -t ghp_token
+        $ GITHUB_TOKEN=ghp_token gitingest https://github.com/user/private-repo
+
+    Include submodules:
+        $ gitingest https://github.com/user/repo --include-submodules
+
+    """
+    asyncio.run(_async_main(**cli_kwargs))
+
+
+async def _async_main(
+    source: str,
+    *,
+    max_size: int = MAX_FILE_SIZE,
+    exclude_pattern: tuple[str, ...] | None = None,
+    include_pattern: tuple[str, ...] | None = None,
+    branch: str | None = None,
+    include_gitignored: bool = False,
+    include_submodules: bool = False,
+    token: str | None = None,
+    output: str | None = None,
+) -> None:
+    """Analyze a directory or repository and create a text dump of its contents.
+
+    This command scans the specified ``source`` (a local directory or Git repo),
+    applies custom include and exclude patterns, and generates a text summary of
+    the analysis.  The summary is written to an output file or printed to ``stdout``.
+
+    Parameters
+    ----------
+    source : str
+        A directory path or a Git repository URL.
+    max_size : int
+        Maximum file size in bytes to ingest (default: 10 MB).
+    exclude_pattern : tuple[str, ...] | None
+        Glob patterns for pruning the file set.
+    include_pattern : tuple[str, ...] | None
+        Glob patterns for including files in the output.
+    branch : str | None
+        Git branch to ingest. If ``None``, the repository's default branch is used.
+    include_gitignored : bool
+        If ``True``, also ingest files matched by ``.gitignore`` or ``.gitingestignore`` (default: ``False``).
+    include_submodules : bool
+        If ``True``, recursively include all Git submodules within the repository (default: ``False``).
+    token : str | None
+        GitHub personal access token (PAT) for accessing private repositories.
+        Can also be set via the ``GITHUB_TOKEN`` environment variable.
+    output : str | None
+        The path where the output file will be written (default: ``digest.txt`` in current directory).
+        Use ``"-"`` to write to ``stdout``.
+
+    Raises
+    ------
+    click.Abort
+        Raised if an error occurs during execution and the command must be aborted.
+
+    """
+    try:
+        # Normalise pattern containers (the ingest layer expects sets)
+        exclude_patterns = set(exclude_pattern) if exclude_pattern else set()
+        include_patterns = set(include_pattern) if include_pattern else set()
+
+        output_target = output if output is not None else OUTPUT_FILE_NAME
+
+        if output_target == "-":
+            click.echo("Analyzing source, preparing output for stdout...", err=True)
+        else:
+            click.echo(f"Analyzing source, output will be written to '{output_target}'...", err=True)
+
+        summary, _, _ = await ingest_async(
+            source,
+            max_file_size=max_size,
+            include_patterns=include_patterns,
+            exclude_patterns=exclude_patterns,
+            branch=branch,
+            include_gitignored=include_gitignored,
+            include_submodules=include_submodules,
+            token=token,
+            output=output_target,
+        )
+    except Exception as exc:
+        # Convert any exception into Click.Abort so that exit status is non-zero
+        click.echo(f"Error: {exc}", err=True)
+        raise click.Abort from exc
+
+    if output_target == "-":  # stdout
+        click.echo("\n--- Summary ---", err=True)
+        click.echo(summary, err=True)
+        click.echo("--- End Summary ---", err=True)
+        click.echo("Analysis complete! Output sent to stdout.", err=True)
+    else:  # file
+        click.echo(f"Analysis complete! Output written to: {output_target}")
+        click.echo("\nSummary:")
+        click.echo(summary)
+
+
+if __name__ == "__main__":
+    main()

gitingest/clone.py

@@ -0,0 +1,176 @@
+"""Module containing functions for cloning a Git repository to a local path."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import git
+
+from gitingest.config import DEFAULT_TIMEOUT
+from gitingest.utils.git_utils import (
+    check_repo_exists,
+    checkout_partial_clone,
+    create_git_repo,
+    ensure_git_installed,
+    git_auth_context,
+    is_github_host,
+    resolve_commit,
+)
+from gitingest.utils.logging_config import get_logger
+from gitingest.utils.os_utils import ensure_directory_exists_or_create
+from gitingest.utils.timeout_wrapper import async_timeout
+
+if TYPE_CHECKING:
+    from gitingest.schemas import CloneConfig
+
+# Initialize logger for this module
+logger = get_logger(__name__)
+
+
+@async_timeout(DEFAULT_TIMEOUT)
+async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
+    """Clone a repository to a local path based on the provided configuration.
+
+    This function handles the process of cloning a Git repository to the local file system.
+    It can clone a specific branch, tag, or commit if provided, and it raises exceptions if
+    any errors occur during the cloning process.
+
+    Parameters
+    ----------
+    config : CloneConfig
+        The configuration for cloning the repository.
+    token : str | None
+        GitHub personal access token (PAT) for accessing private repositories.
+
+    Raises
+    ------
+    ValueError
+        If the repository is not found, if the provided URL is invalid, or if the token format is invalid.
+    RuntimeError
+        If Git operations fail during the cloning process.
+
+    """
+    # Extract and validate query parameters
+    url: str = config.url
+    local_path: str = config.local_path
+    partial_clone: bool = config.subpath != "/"
+
+    logger.info(
+        "Starting git clone operation",
+        extra={
+            "url": url,
+            "local_path": local_path,
+            "partial_clone": partial_clone,
+            "subpath": config.subpath,
+            "branch": config.branch,
+            "tag": config.tag,
+            "commit": config.commit,
+            "include_submodules": config.include_submodules,
+        },
+    )
+
+    logger.debug("Ensuring git is installed")
+    await ensure_git_installed()
+
+    logger.debug("Creating local directory", extra={"parent_path": str(Path(local_path).parent)})
+    await ensure_directory_exists_or_create(Path(local_path).parent)
+
+    logger.debug("Checking if repository exists", extra={"url": url})
+    if not await check_repo_exists(url, token=token):
+        logger.error("Repository not found", extra={"url": url})
+        msg = "Repository not found. Make sure it is public or that you have provided a valid token."
+        raise ValueError(msg)
+
+    logger.debug("Resolving commit reference")
+    commit = await resolve_commit(config, token=token)
+    logger.debug("Resolved commit", extra={"commit": commit})
+
+    # Clone the repository using GitPython with proper authentication
+    logger.info("Executing git clone operation", extra={"url": "<redacted>", "local_path": local_path})
+    try:
+        clone_kwargs = {
+            "single_branch": True,
+            "no_checkout": True,
+            "depth": 1,
+        }
+
+        with git_auth_context(url, token) as (git_cmd, auth_url):
+            if partial_clone:
+                # For partial clones, use git.Git() with filter and sparse options
+                cmd_args = ["--single-branch", "--no-checkout", "--depth=1"]
+                cmd_args.extend(["--filter=blob:none", "--sparse"])
+                cmd_args.extend([auth_url, local_path])
+                git_cmd.clone(*cmd_args)
+            elif token and is_github_host(url):
+                # For authenticated GitHub repos, use git_cmd with auth URL
+                cmd_args = ["--single-branch", "--no-checkout", "--depth=1", auth_url, local_path]
+                git_cmd.clone(*cmd_args)
+            else:
+                # For non-authenticated repos, use the standard GitPython method
+                git.Repo.clone_from(url, local_path, **clone_kwargs)
+
+        logger.info("Git clone completed successfully")
+    except git.GitCommandError as exc:
+        msg = f"Git clone failed: {exc}"
+        raise RuntimeError(msg) from exc
+
+    # Checkout the subpath if it is a partial clone
+    if partial_clone:
+        logger.info("Setting up partial clone for subpath", extra={"subpath": config.subpath})
+        await checkout_partial_clone(config, token=token)
+        logger.debug("Partial clone setup completed")
+
+    # Perform post-clone operations
+    await _perform_post_clone_operations(config, local_path, url, token, commit)
+
+    logger.info("Git clone operation completed successfully", extra={"local_path": local_path})
+
+
+async def _perform_post_clone_operations(
+    config: CloneConfig,
+    local_path: str,
+    url: str,
+    token: str | None,
+    commit: str,
+) -> None:
+    """Perform post-clone operations like fetching, checkout, and submodule updates.
+
+    Parameters
+    ----------
+    config : CloneConfig
+        The configuration for cloning the repository.
+    local_path : str
+        The local path where the repository was cloned.
+    url : str
+        The repository URL.
+    token : str | None
+        GitHub personal access token (PAT) for accessing private repositories.
+    commit : str
+        The commit SHA to checkout.
+
+    Raises
+    ------
+    RuntimeError
+        If any Git operation fails.
+
+    """
+    try:
+        repo = create_git_repo(local_path, url, token)
+
+        # Ensure the commit is locally available
+        logger.debug("Fetching specific commit", extra={"commit": commit})
+        repo.git.fetch("--depth=1", "origin", commit)
+
+        # Write the work-tree at that commit
+        logger.info("Checking out commit", extra={"commit": commit})
+        repo.git.checkout(commit)
+
+        # Update submodules
+        if config.include_submodules:
+            logger.info("Updating submodules")
+            repo.git.submodule("update", "--init", "--recursive", "--depth=1")
+            logger.debug("Submodules updated successfully")
+    except git.GitCommandError as exc:
+        msg = f"Git operation failed: {exc}"
+        raise RuntimeError(msg) from exc

gitingest/config.py

@@ -0,0 +1,14 @@
+"""Configuration file for the project."""
+
+import tempfile
+from pathlib import Path
+
+MAX_FILE_SIZE = 10 * 1024 * 1024  # Maximum size of a single file to process (10 MB)
+MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
+MAX_FILES = 10_000  # Maximum number of files to process
+MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # Maximum size of output file (500 MB)
+DEFAULT_TIMEOUT = 60  # seconds
+
+OUTPUT_FILE_NAME = "digest.txt"
+
+TMP_BASE_PATH = Path(tempfile.gettempdir()) / "gitingest"