fluidattacks_core_git 12.0.0__py3-none-any.whl

fluidattacks_core/git/show_file.py

@@ -0,0 +1,408 @@
+import asyncio
+import io
+import logging
+import os
+import re
+import shutil
+import tarfile
+import uuid
+from pathlib import Path
+
+import aiofiles
+
+from fluidattacks_core.git.ssh_utils import (
+    parse_ssh_url,
+    ssh_git_env_context,
+)
+from fluidattacks_core.git.utils import format_url, get_https_git_config_args, run_git
+
+LOGGER = logging.getLogger(__name__)
+
+COMMIT_SHA_PATTERN = re.compile(r"^[0-9a-fA-F]{7,40}\Z")
+
+
+def _is_commit_sha(ref: str) -> bool:
+    return bool(COMMIT_SHA_PATTERN.fullmatch(ref))
+
+
+def _ref_for_fetch(ref: str) -> str:
+    if ref.startswith("origin/"):
+        return ref.removeprefix("origin/")
+    return ref
+
+
+def _show_spec_for_ref(path: str) -> str:
+    return f"FETCH_HEAD:{path}"
+
+
+async def _run_git(
+    env: dict[str, str],
+    config_args: list[str],
+    args: list[str],
+    *,
+    cwd: str | None = None,
+) -> tuple[bytes, bytes, int]:
+    try:
+        return await run_git(*config_args, *args, env=env, cwd=cwd, timeout=30.0)
+    except asyncio.exceptions.TimeoutError:
+        return b"", b"git command timed out", -1
+    except OSError as exc:
+        return b"", str(exc).encode(), -1
+
+
+class _GitResult:
+    __slots__ = ("_error", "_stdout")
+
+    def __init__(
+        self,
+        error: str | None = None,
+        stdout: bytes | None = None,
+    ) -> None:
+        self._error = error
+        self._stdout = stdout
+
+    @property
+    def is_ok(self) -> bool:
+        return self._error is None
+
+    @property
+    def error(self) -> str | None:
+        return self._error
+
+    @property
+    def stdout(self) -> bytes | None:
+        return self._stdout
+
+    @classmethod
+    async def from_git(
+        cls,
+        env: dict[str, str],
+        config_args: list[str],
+        args: list[str],
+        *,
+        cwd: str | None = None,
+    ) -> "_GitResult":
+        stdout, stderr, code = await _run_git(env, config_args, args, cwd=cwd)
+        if code != 0:
+            return cls(stderr.decode("utf-8"))
+        return cls(stdout=stdout)
+
+    async def and_then(
+        self,
+        env: dict[str, str],
+        config_args: list[str],
+        args: list[str],
+        *,
+        cwd: str | None = None,
+    ) -> "_GitResult":
+        if not self.is_ok:
+            return self
+        return await _GitResult.from_git(env, config_args, args, cwd=cwd)
+
+
+async def _run_git_sequence(
+    env: dict[str, str],
+    config_args: list[str],
+    steps: list[tuple[list[str], str | None]],
+) -> _GitResult:
+    result = await _GitResult.from_git(env, config_args, steps[0][0], cwd=steps[0][1])
+    for args, cwd in steps[1:]:
+        result = await result.and_then(env, config_args, args, cwd=cwd)
+    return result
+
+
+async def _show_via_fetch_show(  # noqa: PLR0913
+    env: dict[str, str],
+    config_args: list[str],
+    repo_url: str,
+    ref_fetch: str,
+    show_spec: str,
+    work_dir: str,
+) -> tuple[bytes | None, str | None]:
+    fetch_args = ["fetch"]
+    if not _is_commit_sha(ref_fetch):
+        fetch_args.extend(["--depth", "1"])
+    fetch_args.extend(["origin", ref_fetch])
+
+    steps: list[tuple[list[str], str | None]] = [
+        (["init"], work_dir),
+        (["remote", "add", "origin", repo_url], work_dir),
+        (fetch_args, work_dir),
+        (["show", show_spec], work_dir),
+    ]
+    result = await _run_git_sequence(env, config_args, steps)
+    if not result.is_ok:
+        return None, result.error
+    return result.stdout, None
+
+
+async def _show_via_archive_remote(
+    env: dict[str, str],
+    config_args: list[str],
+    repo_url: str,
+    ref_fetch: str,
+    path: str,
+) -> tuple[bytes | None, str | None]:
+    stdout, stderr, code = await _run_git(
+        env,
+        config_args,
+        ["archive", f"--remote={repo_url}", ref_fetch, "--", path],
+        cwd=None,
+    )
+    if code != 0:
+        return None, stderr.decode("utf-8")
+    if len(stdout) == 0:
+        return None, "git archive produced empty output"
+    try:
+        with tarfile.open(fileobj=io.BytesIO(stdout), mode="r:") as tar:
+            for member in tar:
+                if (
+                    member.isfile()
+                    and (extracted := tar.extractfile(member))
+                    and extracted is not None
+                ):
+                    return extracted.read(), None
+            return None, "no file in archive"
+    except (tarfile.TarError, OSError) as exc:
+        return None, str(exc)
+
+
+def _path_contained_in(base: Path, candidate: Path) -> bool:
+    try:
+        return candidate.resolve().is_relative_to(base.resolve())
+    except (ValueError, OSError):
+        return False
+
+
+def _sparse_checkout_directory(path: str) -> str:
+    """Directory to pass to sparse-checkout set in cone mode.
+
+    Cone mode only accepts directories and includes all files under them.
+    For a file path like src/main.py we must set the parent directory (src)
+    so the file is actually checked out.
+    """
+    parent = Path(path).parent
+    return "." if not parent.parts else str(parent)
+
+
+async def _show_via_sparse_checkout(  # noqa: PLR0913
+    env: dict[str, str],
+    config_args: list[str],
+    repo_url: str,
+    ref_fetch: str,
+    path: str,
+    temp_dir: str,
+) -> tuple[bytes | None, str | None]:
+    sparse_dir = f"{temp_dir}/{uuid.uuid4()}"
+    sparse_dir_path = _sparse_checkout_directory(path)
+
+    fetch_args = ["fetch"]
+    if not _is_commit_sha(ref_fetch):
+        fetch_args.extend(["--depth", "1"])
+    fetch_args.extend(["origin", ref_fetch])
+
+    steps: list[tuple[list[str], str | None]] = [
+        (
+            [
+                "clone",
+                "--no-checkout",
+                "--depth",
+                "1",
+                repo_url,
+                Path(sparse_dir).name,
+            ],
+            temp_dir,
+        ),
+        (fetch_args, sparse_dir),
+        (["sparse-checkout", "init", "--cone"], sparse_dir),
+        (["sparse-checkout", "set", sparse_dir_path], sparse_dir),
+        (["checkout", "FETCH_HEAD"], sparse_dir),
+    ]
+    try:
+        result = await _run_git_sequence(env, config_args, steps)
+        if not result.is_ok:
+            return None, result.error
+
+        base = Path(sparse_dir)
+        file_path = base / path
+        if not _path_contained_in(base, file_path):
+            return None, "path escapes repository"
+        try:
+            async with aiofiles.open(file_path, "rb") as f:
+                return await f.read(), None
+        except OSError as exc:
+            return None, str(exc)
+    finally:
+        shutil.rmtree(sparse_dir, ignore_errors=True)
+
+
+async def _try_show_strategies(  # noqa: PLR0913
+    env: dict[str, str],
+    config_args: list[str],
+    repo_url: str,
+    ref_fetch: str,
+    show_spec: str,
+    path: str,
+    work_dir: str,
+    temp_dir: str,
+) -> tuple[bytes | None, str | None]:
+    content, err = await _show_via_fetch_show(
+        env,
+        config_args,
+        repo_url,
+        ref_fetch,
+        show_spec,
+        work_dir,
+    )
+    if content is not None:
+        return content, None
+    last_error = err or "fetch+show failed"
+    LOGGER.debug(
+        "show_file fallback: fetch+show failed, trying archive --remote",
+        extra={"extra": {"error": last_error}},
+    )
+
+    content, err = await _show_via_archive_remote(
+        env,
+        config_args,
+        repo_url,
+        ref_fetch,
+        path,
+    )
+    if content is not None:
+        return content, None
+    last_error = err or "archive --remote failed"
+    LOGGER.debug(
+        "show_file fallback: archive --remote failed, trying sparse-checkout",
+        extra={"extra": {"error": last_error}},
+    )
+
+    content, err = await _show_via_sparse_checkout(
+        env,
+        config_args,
+        repo_url,
+        ref_fetch,
+        path,
+        temp_dir,
+    )
+    if content is not None:
+        return content, None
+    return None, err or "sparse-checkout failed"
+
+
+async def show_file_at_ref_ssh(
+    repo_url: str,
+    ref: str,
+    path: str,
+    temp_dir: str,
+    *,
+    credential_key: str,
+) -> tuple[bytes | None, str | None]:
+    ref_fetch = _ref_for_fetch(ref)
+    show_spec = _show_spec_for_ref(path)
+    work_dir = f"{temp_dir}/{uuid.uuid4()}"
+    Path(work_dir).mkdir(parents=True, exist_ok=True)  # noqa: ASYNC240
+    repo_url_parsed = parse_ssh_url(repo_url)
+    config_args: list[str] = []
+
+    try:
+        with ssh_git_env_context(temp_dir, credential_key) as env:
+            return await _try_show_strategies(
+                env,
+                config_args,
+                repo_url_parsed,
+                ref_fetch,
+                show_spec,
+                path,
+                work_dir,
+                temp_dir,
+            )
+    finally:
+        shutil.rmtree(work_dir, ignore_errors=True)
+
+
+async def show_file_at_ref_https(  # noqa: PLR0913
+    repo_url: str,
+    ref: str,
+    path: str,
+    temp_dir: str,
+    *,
+    user: str | None = None,
+    password: str | None = None,
+    token: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    follow_redirects: bool = False,
+) -> tuple[bytes | None, str | None]:
+    ref_fetch = _ref_for_fetch(ref)
+    show_spec = _show_spec_for_ref(path)
+    work_dir = f"{temp_dir}/{uuid.uuid4()}"
+    Path(work_dir).mkdir(parents=True, exist_ok=True)  # noqa: ASYNC240
+
+    url = format_url(
+        repo_url=repo_url,
+        user=user,
+        password=password,
+        token=token,
+        provider=provider,
+        is_pat=is_pat,
+    )
+    config_args = get_https_git_config_args(
+        follow_redirects=follow_redirects,
+        is_pat=is_pat,
+        token=token,
+        disable_credential_prompt=token is not None or password is not None,
+    )
+    env = os.environ.copy()
+    if token is not None or password is not None:
+        env["GIT_TERMINAL_PROMPT"] = "0"
+
+    try:
+        return await _try_show_strategies(
+            env,
+            config_args,
+            url,
+            ref_fetch,
+            show_spec,
+            path,
+            work_dir,
+            temp_dir,
+        )
+    finally:
+        shutil.rmtree(work_dir, ignore_errors=True)
+
+
+async def show_file_at_ref(  # noqa: PLR0913
+    repo_url: str,
+    ref: str,
+    path: str,
+    temp_dir: str,
+    *,
+    credential_key: str | None = None,
+    user: str | None = None,
+    password: str | None = None,
+    token: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    follow_redirects: bool = False,
+) -> tuple[bytes | None, str | None]:
+    if credential_key:
+        return await show_file_at_ref_ssh(
+            repo_url,
+            ref,
+            path,
+            temp_dir,
+            credential_key=credential_key,
+        )
+    return await show_file_at_ref_https(
+        repo_url,
+        ref,
+        path,
+        temp_dir,
+        user=user,
+        password=password,
+        token=token,
+        provider=provider,
+        is_pat=is_pat,
+        follow_redirects=follow_redirects,
+    )

fluidattacks_core/git/ssh_utils.py

@@ -0,0 +1,214 @@
+import asyncio
+import base64
+import ipaddress
+import os
+import tempfile
+import uuid
+from collections.abc import Generator
+from contextlib import contextmanager, suppress
+from pathlib import Path
+from urllib.parse import urlparse
+
+from fluidattacks_core.http.validations import (
+    HTTPValidationError,
+    validate_local_request,
+)
+
+from fluidattacks_core.git.utils import run_git
+
+
+class SSHHostValidationError(Exception):
+    pass
+
+
+def _extract_hostname_from_ssh_url(url: str) -> str | None:
+    if "://" in url:
+        parsed = urlparse(url)
+        if parsed.hostname:
+            return parsed.hostname
+    if "@" in url and ":" in url:
+        host_part = url.split("@", maxsplit=1)[-1]
+        return host_part.split(":", maxsplit=1)[0]
+    return None
+
+
+async def validate_ssh_host(
+    url: str,
+    *,
+    allow_local_network: bool = False,
+    allow_localhost: bool = False,
+) -> None:
+    hostname = _extract_hostname_from_ssh_url(parse_ssh_url(url))
+    if not hostname:
+        return
+
+    loop = asyncio.get_running_loop()
+    infos = await loop.getaddrinfo(hostname, None)
+
+    seen: set[str] = set()
+    ips: list[ipaddress.IPv4Address | ipaddress.IPv6Address] = []
+    for _family, _type, _proto, _canonname, sockaddr in infos:
+        addr = sockaddr[0]
+        if addr not in seen:
+            seen.add(addr)
+            ips.append(ipaddress.ip_address(addr))
+
+    if not ips:
+        msg = f"No IP addresses resolved for hostname: {hostname}"
+        raise SSHHostValidationError(msg)
+
+    try:
+        validate_local_request(
+            ips,
+            allow_local_network=allow_local_network,
+            allow_localhost=allow_localhost,
+        )
+    except HTTPValidationError as exc:
+        msg = f"SSH host validation failed for {hostname}: {exc}"
+        raise SSHHostValidationError(msg) from exc
+
+
+def _add_ssh_scheme_to_url(url: str) -> str:
+    scheme: str = "ssh://"
+    if url.startswith(scheme):
+        return url
+    return scheme + url
+
+
+def _url_has_port(url: str) -> bool:
+    parsed_url = urlparse(url)
+    try:
+        if parsed_url.port:
+            return True
+    except ValueError:
+        # Port could not be cast to integer value, or
+        # Port out of range 0-65535
+        return False
+    else:
+        return False
+
+
+def _set_default_ssh_port(url_with_scheme: str) -> str:
+    """Add a default port placeholder to a URL that lacks an explicit port.
+
+    This function modifies URLs that use the SSH protocol format for Git
+    repositories. It adds a placeholder for the default port
+    (represented by ':/')  after the hostname.
+
+    Args:
+        url_with_scheme (str): The input URL, expected to be in the format
+        "ssh://git@hostname:path/to/repo.git"
+
+    Returns:
+        str: The modified URL with the default port placeholder added,
+        in the format "ssh://git@hostname:/path/to/repo.git"
+
+    Examples:
+        "ssh://git@gitlab.com:fluidattacks/demo.git" becomes
+        "ssh://git@gitlab.com:/fluidattacks/demo.git"
+
+    Note:
+        This function modifies the URL only if all the following
+        conditions are met:
+        1. The URL starts with 'ssh://'.
+        2. The URL does not already contain a port.
+        3. The URL contains exactly two colons after the 'ssh://' scheme.
+        URLs not meeting these criteria are returned unchanged.
+
+    """
+    has_ssh_scheme = url_with_scheme.startswith("ssh://")
+
+    # formatting is skipped if no ssh scheme or URL contains a port
+    if not has_ssh_scheme or _url_has_port(url_with_scheme):
+        return url_with_scheme
+
+    url_parts = url_with_scheme.split(":", 2)
+    if len(url_parts) < 3:
+        return url_with_scheme
+
+    return f"{url_parts[0]}:{url_parts[1]}:/{url_parts[2]}"
+
+
+def parse_ssh_url(url: str) -> str:
+    if "source.developers.google" in url or url.startswith("ssh://FLUID"):
+        return url
+
+    url_with_scheme = _add_ssh_scheme_to_url(url)
+
+    # url misses an explicit ssh port
+    return _set_default_ssh_port(url_with_scheme)
+
+
+def get_ssh_git_env(ssh_key_path: str | Path) -> dict[str, str]:
+    return {
+        **os.environ.copy(),
+        "GIT_SSH_COMMAND": (
+            f"ssh -i {ssh_key_path}"
+            " -o UserKnownHostsFile=/dev/null"
+            " -o StrictHostKeyChecking=no"
+            " -o IdentitiesOnly=yes"
+            " -o HostkeyAlgorithms=+ssh-rsa"
+            " -o PubkeyAcceptedAlgorithms=+ssh-rsa"
+        ),
+    }
+
+
+def create_ssh_key_file(temp_dir: str, credential_key: str) -> str:
+    ssh_file_name: str = os.path.join(temp_dir, str(uuid.uuid4()))  # noqa: PTH118
+    with open(  # noqa: PTH123
+        os.open(ssh_file_name, os.O_CREAT | os.O_WRONLY, 0o400),
+        "w",
+        encoding="utf-8",
+    ) as ssh_file:
+        ssh_file.write(base64.b64decode(credential_key).decode())
+    return ssh_file_name
+
+
+@contextmanager
+def ssh_git_env_context(
+    temp_dir: str, credential_key: str
+) -> Generator[dict[str, str], None, None]:
+    ssh_key_path = create_ssh_key_file(temp_dir, credential_key)
+    try:
+        yield get_ssh_git_env(ssh_key_path)
+    finally:
+        with suppress(OSError):
+            os.remove(ssh_key_path)  # noqa: PTH107
+
+
+async def ssh_ls_remote(
+    repo_url: str,
+    credential_key: str,
+    branch: str,
+) -> tuple[str | None, str | None]:
+    raw_root_url = parse_ssh_url(repo_url)
+    with (
+        tempfile.TemporaryDirectory() as temp_dir,
+        ssh_git_env_context(temp_dir, credential_key) as env,
+    ):
+        try:
+            stdout, stderr, returncode = await run_git(
+                "ls-remote",
+                "--",
+                raw_root_url,
+                branch,
+                env=env,
+                timeout=20,
+            )
+        except asyncio.exceptions.TimeoutError:
+            return None, "git ls-remote time out"
+        if returncode == 0:
+            return stdout.decode().split("\t")[0], None
+        return None, stderr.decode("utf-8")
+
+
+async def call_ssh_ls_remote(
+    repo_url: str,
+    credential_key: str,
+    branch: str,
+) -> tuple[str | None, str | None]:
+    return await ssh_ls_remote(
+        repo_url=repo_url,
+        credential_key=credential_key,
+        branch=branch,
+    )