fluidattacks_core_git 12.0.0__py3-none-any.whl

fluidattacks_core/git/codecommit_utils.py

@@ -0,0 +1,129 @@
+import asyncio
+import logging
+import os
+import re
+import uuid
+
+import boto3
+from botocore.exceptions import (
+    ClientError,
+)
+
+from fluidattacks_core.git.utils import run_git
+
+LOGGER = logging.getLogger(__name__)
+
+
+def extract_region(url: str) -> str:
+    pattern = r"codecommit::([a-z0-9-]+)://"
+    match = re.search(pattern, url)
+    if match:
+        return match.group(1)
+    return "us-east-1"
+
+
+async def assume_role_and_execute_git_ls_remote(
+    arn: str,
+    repo_url: str,
+    branch: str,
+    org_external_id: str,
+    *,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    try:
+        sts_client = boto3.client("sts")
+        assumed_role = sts_client.assume_role(
+            RoleArn=arn,
+            RoleSessionName=f"session-{uuid.uuid4()}",
+            ExternalId=org_external_id,
+        )
+        credentials = assumed_role["Credentials"]
+
+        return await codecommit_ls_remote(
+            env={
+                "AWS_ACCESS_KEY_ID": credentials["AccessKeyId"],
+                "AWS_SECRET_ACCESS_KEY": credentials["SecretAccessKey"],
+                "AWS_SESSION_TOKEN": credentials["SessionToken"],
+                "AWS_DEFAULT_REGION": extract_region(repo_url),
+            },
+            branch=branch,
+            repo_url=repo_url,
+            follow_redirects=follow_redirects,
+        )
+
+    except ClientError as exc:
+        err_message = "Error executing ls-remote with codecommit"
+        LOGGER.exception(
+            err_message,
+            extra={
+                "extra": {
+                    "repo_url": repo_url,
+                    "arn": arn,
+                    "org_external_id": org_external_id,
+                    "exc": exc,
+                },
+            },
+        )
+
+        return None, err_message
+
+
+async def _execute_git_command(
+    *,
+    branch: str,
+    env: dict[str, str],
+    url: str,
+    follow_redirects: bool = False,
+) -> tuple[bytes, bytes, int]:
+    return await run_git(
+        "-c",
+        "http.sslVerify=false",
+        "-c",
+        f"http.followRedirects={follow_redirects}",
+        "ls-remote",
+        "--",
+        url,
+        branch,
+        env={**os.environ.copy(), **env},
+        timeout=20,
+    )
+
+
+async def codecommit_ls_remote(
+    *,
+    branch: str,
+    env: dict[str, str],
+    repo_url: str,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    try:
+        stdout, stderr, return_code = await _execute_git_command(
+            branch=branch,
+            env=env,
+            follow_redirects=follow_redirects,
+            url=repo_url,
+        )
+    except asyncio.exceptions.TimeoutError:
+        return None, "git ls-remote time out"
+
+    if return_code == 0:
+        return stdout.decode().split("\t")[0], None
+
+    return None, stderr.decode("utf-8")
+
+
+async def call_codecommit_ls_remote(
+    repo_url: str,
+    arn: str,
+    branch: str,
+    org_external_id: str,
+    *,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    return await assume_role_and_execute_git_ls_remote(
+        repo_url=repo_url,
+        arn=arn,
+        branch=branch,
+        org_external_id=org_external_id,
+        follow_redirects=follow_redirects,
+    )

fluidattacks_core/git/constants.py

@@ -0,0 +1,7 @@
+DEFAULT_DOWNLOAD_BUFFER_SIZE = 64 * 1024  # 64KB
+
+MULTIPART_THRESHOLD = 5 * 1024 * 1024 * 1024
+MULTIPART_PART_SIZE = 50 * 1024 * 1024
+MAX_RETRIES_PER_PART = 3
+MAX_S3_PARTS = 10_000
+UPLOAD_CONCURRENCY = 8

fluidattacks_core/git/delete_files.py

@@ -0,0 +1,47 @@
+import logging
+import os
+from contextlib import suppress
+from pathlib import Path
+
+from pathspec import PathSpec
+from pathspec import util as pathspec_util
+
+LOGGER = logging.getLogger(__name__)
+
+
+def remove_empty_directories(repo_path: str) -> None:
+    for root, dirs, _ in os.walk(repo_path, topdown=False):
+        for name in dirs:
+            path = os.path.join(root, name)  # noqa: PTH118
+            try:
+                if not os.listdir(path):  # noqa: PTH208
+                    Path(path).rmdir()
+            except FileNotFoundError:
+                LOGGER.exception(
+                    "Error removing empty directory", extra={"extra": {"dir_path": path}}
+                )
+                continue
+
+
+def delete_out_of_scope_files(git_ignore: list[str], repo_path: str) -> None:
+    # Compute what files should be deleted according to the scope rules
+    spec = PathSpec.from_lines("gitwildmatch", git_ignore)
+    try:
+        matches = list(spec.match_tree(repo_path, follow_links=False))
+    except pathspec_util.RecursionError:
+        LOGGER.exception(
+            "RecursionError while matching tree (symlink cycle detected)",
+            extra={"extra": {"repo_path": repo_path}},
+        )
+        return
+
+    for match in matches:
+        if match.startswith(".git/"):
+            continue
+
+        file_path = os.path.join(repo_path, match)  # noqa: PTH118
+        if Path(file_path).is_file():
+            with suppress(FileNotFoundError):
+                Path(file_path).unlink()
+
+    remove_empty_directories(repo_path)

fluidattacks_core/git/download_file.py

@@ -0,0 +1,47 @@
+import logging
+
+import aiofiles
+import aiohttp
+import anyio
+
+from .constants import DEFAULT_DOWNLOAD_BUFFER_SIZE
+
+LOGGER = logging.getLogger(__name__)
+
+
+async def _write_response_to_file(
+    response: aiohttp.ClientResponse,
+    destination_path: str,
+    download_buffer_size: int,
+) -> bool:
+    async with aiofiles.open(destination_path, "wb") as file:
+        while True:
+            try:
+                chunk = await response.content.read(download_buffer_size)
+            except TimeoutError:
+                LOGGER.exception("Read timeout for path %s", destination_path)
+                return False
+            if not chunk:
+                break
+            await file.write(chunk)
+    return await anyio.Path(destination_path).exists()
+
+
+async def download_file(
+    *,
+    url: str,
+    destination_path: str,
+    download_buffer_size: int = DEFAULT_DOWNLOAD_BUFFER_SIZE,
+) -> bool:
+    timeout = aiohttp.ClientTimeout(total=60 * 60, connect=30)
+    async with aiohttp.ClientSession(timeout=timeout) as session:  # noqa: SIM117
+        async with session.get(url) as response:
+            if response.status != 200:
+                LOGGER.error(
+                    "Failed to download file: HTTP %s, for path %s",
+                    response.status,
+                    destination_path,
+                )
+                return False
+
+            return await _write_response_to_file(response, destination_path, download_buffer_size)

fluidattacks_core/git/download_repo.py

@@ -0,0 +1,148 @@
+import logging
+import os
+import shutil
+import tarfile
+import tempfile
+from pathlib import Path
+
+import anyio
+from git import GitError
+from git.cmd import Git
+from git.repo import Repo
+
+from .constants import DEFAULT_DOWNLOAD_BUFFER_SIZE
+from .delete_files import delete_out_of_scope_files
+from .download_file import download_file
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _is_member_safe(
+    member: tarfile.TarInfo,
+) -> bool:
+    return not (
+        member.issym() or member.islnk() or Path(member.name).is_absolute() or "../" in member.name
+    )
+
+
+def _safe_extract_tar(tar_handler: tarfile.TarFile, file_path: Path) -> bool:
+    for member in tar_handler.getmembers():
+        if not _is_member_safe(member):
+            LOGGER.error("Unsafe path detected: %s", member.name)
+            continue
+        try:
+            tar_handler.extract(member, path=file_path, numeric_owner=True)
+        except tarfile.ExtractError:
+            LOGGER.exception("Error extracting %s", member.name)
+
+    return True
+
+
+def remove_symlinks_in_directory(directory: str) -> None:
+    for root, dirs, files in os.walk(directory):
+        for name in (*dirs, *files):
+            file_path = os.path.join(root, name)  # noqa: PTH118
+            if Path(file_path).is_symlink():
+                Path(file_path).unlink(missing_ok=True)
+
+
+def _reset_repo_sync(repo_path: str) -> bool:
+    try:
+        Path.cwd()
+    except OSError:
+        LOGGER.exception("Failed to get the working directory: %s", repo_path)
+        os.chdir(repo_path)
+
+    try:
+        Git().execute(
+            [
+                "git",
+                "config",
+                "--global",
+                "--add",
+                "safe.directory",
+                "*",
+            ],
+        )
+    except GitError:
+        LOGGER.exception("Failed to add safe directory %s", repo_path)
+
+    try:
+        repo = Repo(repo_path)
+        repo.git.reset("--hard", "HEAD")
+    except GitError:
+        LOGGER.exception("Expand repositories has failed for repository %s", repo_path)
+
+        return False
+
+    if repo.working_dir:
+        remove_symlinks_in_directory(str(repo.working_dir))
+
+    return True
+
+
+async def reset_repo(repo_path: str) -> bool:
+    return await anyio.to_thread.run_sync(_reset_repo_sync, repo_path)
+
+
+def _extract_and_replace_sync(file_path: Path, tmp_path: Path, destination_path: Path) -> bool:
+    with tarfile.open(file_path, "r:gz") as tar_handler:
+        _safe_extract_tar(tar_handler, tmp_path)
+
+    extracted_dirs = [d for d in tmp_path.iterdir() if d.is_dir()]
+    if not extracted_dirs:
+        LOGGER.error("No directory found in the extracted archive: %s", destination_path)
+        return False
+
+    if len(extracted_dirs) > 1:
+        LOGGER.warning(
+            "Multiple directories found in archive, using first one: %s",
+            destination_path,
+        )
+    extracted_dir = extracted_dirs[0]
+
+    if destination_path.exists():
+        shutil.rmtree(destination_path)
+
+    shutil.move(extracted_dir, destination_path)
+    return True
+
+
+async def download_repo_from_s3(
+    download_url: str,
+    destination_path: Path,
+    git_ignore: list[str] | None = None,
+    *,
+    download_buffer_size: int = DEFAULT_DOWNLOAD_BUFFER_SIZE,
+) -> bool:
+    await anyio.Path(destination_path.parent).mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="fluidattacks_", ignore_cleanup_errors=True) as tmpdir:
+        tmp_path = Path(tmpdir)
+        file_path = tmp_path / "repo.tar.gz"
+        result = await download_file(
+            url=download_url,
+            destination_path=str(file_path.absolute()),
+            download_buffer_size=download_buffer_size,
+        )
+        if not result:
+            LOGGER.error("Failed to download repository from %s", download_url)
+            return False
+
+        try:
+            if not await anyio.to_thread.run_sync(
+                _extract_and_replace_sync, file_path, tmp_path, destination_path
+            ):
+                return False
+        except OSError:
+            LOGGER.exception(
+                "Error downloading repository", extra={"extra": {"path": destination_path}}
+            )
+            return False
+
+    if not await reset_repo(str(destination_path)):
+        shutil.rmtree(destination_path, ignore_errors=True)
+        return False
+
+    delete_out_of_scope_files(git_ignore or [], str(destination_path))
+
+    return True

fluidattacks_core/git/https_utils.py

@@ -0,0 +1,230 @@
+import asyncio
+import base64
+import logging
+from contextlib import suppress
+
+import aiohttp
+from fluidattacks_core.http.client import request
+from fluidattacks_core.http.validations import HTTPValidationError
+from urllib3.exceptions import LocationParseError
+from urllib3.util import Url, parse_url
+
+from fluidattacks_core.git.classes import InvalidParameter
+from fluidattacks_core.git.utils import format_url, get_https_git_config_args, run_git
+
+LOGGER = logging.getLogger(__name__)
+
+
+def format_redirected_url(
+    original_url: Url,
+    redirect_url: Url,
+) -> str:
+    return (
+        redirect_url._replace(
+            query=None,
+            path=(redirect_url.path or "").removesuffix("info/refs"),
+        ).url
+        if original_url.host == redirect_url.host
+        else original_url.url
+    )
+
+
+async def get_redirected_url(
+    *,
+    url: str,
+    user: str | None,
+    password: str | None,
+    token: str | None,
+    is_pat: bool,
+) -> str:
+    try:
+        uri = parse_url(url)._replace(auth=None).url
+    except LocationParseError:
+        uri = url
+    if user is not None and password is not None:
+        return await _get_redirected_url(
+            uri,
+            authorization="Basic " + base64.b64encode(f"{user}:{password}".encode()).decode(),
+        )
+    if token is not None and is_pat:
+        return await _get_redirected_url(
+            uri,
+            authorization=("Basic " + base64.b64encode(f":{token}".encode()).decode()),
+        )
+    if token is not None and not is_pat:
+        return await _get_redirected_url(uri, authorization=f"Bearer {token}")
+    if url.startswith("http"):
+        return await _get_redirected_url(url)
+    raise InvalidParameter
+
+
+async def _get_redirected_url(
+    url: str,
+    authorization: str | None = None,
+) -> str:
+    try:
+        return await _get_url(url, authorization=authorization)
+    except (aiohttp.ClientError, TimeoutError) as exc:
+        LOGGER.warning(
+            "Failed to get redirected-url",
+            extra={"extra": {"url": url, "exc": exc}},
+        )
+        raise
+    except (ValueError, HTTPValidationError) as exc:
+        LOGGER.warning(
+            "Failed validation to get redirected-url",
+            extra={"extra": {"url": url, "exc": exc}},
+        )
+        raise
+
+
+async def _get_url(
+    original_url: str,
+    *,
+    redirect_url: str = "",
+    max_retries: int = 5,
+    authorization: str | None = None,
+) -> str:
+    try:
+        original = parse_url(original_url.removesuffix("/"))
+        url = parse_url(redirect_url.removesuffix("/")) if redirect_url else original
+    except LocationParseError as exc:
+        msg = f"Invalid URL {redirect_url}"
+        raise HTTPValidationError(msg) from exc
+
+    if max_retries < 1:
+        return format_redirected_url(original, url)
+
+    # https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
+    url = (
+        url._replace(path=(url.path or "") + "/info/refs")
+        if not (url.path or "").endswith("info/refs")
+        else url
+    )
+    url = url._replace(query="service=git-upload-pack")
+
+    result = await request(
+        url.url,
+        method="GET",
+        headers={
+            "Host": url.host or "",
+            "Accept": "*/*",
+            "Accept-Encoding": "deflate, gzip",
+            "Pragma": "no-cache",
+            "Accept-Language": "*",
+            "User-Agent": "FluidAttacksAPIClient/1.0",
+            **({"Authorization": authorization} if authorization else {}),
+        },
+        timeout=20,
+    )
+    if result.status == 200:
+        return format_redirected_url(original, url)
+    if result.status > 300 and result.status < 400 and "Location" in result.headers:
+        with suppress(LocationParseError):
+            _url = parse_url(result.headers["Location"])
+            return await _get_url(
+                original_url,
+                redirect_url=result.headers["Location"],
+                max_retries=max_retries - 1,
+                authorization=authorization if _url.host == url.host else None,
+            )
+
+        return await _get_url(
+            original_url,
+            redirect_url=result.headers["Location"],
+            max_retries=max_retries - 1,
+        )
+
+    return format_redirected_url(original, url)
+
+
+async def _execute_git_command(
+    *,
+    url: str,
+    branch: str,
+    is_pat: bool,
+    token: str | None = None,
+    follow_redirects: bool = False,
+) -> tuple[bytes, bytes, int]:
+    git_config_args = get_https_git_config_args(
+        follow_redirects=follow_redirects,
+        is_pat=is_pat,
+        token=token,
+    )
+    return await run_git(*git_config_args, "ls-remote", "--", url, branch, timeout=20)
+
+
+async def https_ls_remote(  # noqa: PLR0913
+    *,
+    repo_url: str,
+    branch: str,
+    user: str | None = None,
+    password: str | None = None,
+    token: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    url = format_url(
+        repo_url=repo_url,
+        user=user,
+        password=password,
+        token=token,
+        provider=provider,
+        is_pat=is_pat,
+    )
+    try:
+        stdout, stderr, return_code = await _execute_git_command(
+            url=url,
+            branch=branch,
+            is_pat=is_pat,
+            token=token,
+            follow_redirects=follow_redirects,
+        )
+    except asyncio.exceptions.TimeoutError:
+        return None, "git ls-remote time out"
+
+    if return_code == 0:
+        return stdout.decode().split("\t")[0], None
+
+    return None, stderr.decode("utf-8")
+
+
+async def call_https_ls_remote(  # noqa: PLR0913
+    *,
+    repo_url: str,
+    user: str | None,
+    password: str | None,
+    token: str | None,
+    branch: str,
+    provider: str | None,
+    is_pat: bool,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    if user is not None and password is not None:
+        return await https_ls_remote(
+            repo_url=repo_url,
+            user=user,
+            password=password,
+            branch=branch,
+            follow_redirects=follow_redirects,
+        )
+
+    if token is not None:
+        return await https_ls_remote(
+            repo_url=repo_url,
+            token=token,
+            branch=branch,
+            provider=provider or "",
+            is_pat=is_pat,
+            follow_redirects=follow_redirects,
+        )
+
+    if repo_url.startswith("http"):
+        return await https_ls_remote(
+            repo_url=repo_url,
+            branch=branch,
+            follow_redirects=follow_redirects,
+        )
+
+    raise InvalidParameter

fluidattacks_core/git/remote.py

@@ -0,0 +1,45 @@
+from .codecommit_utils import call_codecommit_ls_remote
+from .https_utils import call_https_ls_remote
+from .ssh_utils import call_ssh_ls_remote
+
+
+async def ls_remote(  # noqa: PLR0913
+    repo_url: str,
+    repo_branch: str,
+    *,
+    credential_key: str | None = None,
+    user: str | None = None,
+    password: str | None = None,
+    token: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    arn: str | None = None,
+    org_external_id: str | None = None,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    if credential_key is not None:
+        return await call_ssh_ls_remote(
+            repo_url,
+            credential_key,
+            repo_branch,
+        )
+
+    if arn is not None and org_external_id is not None:
+        return await call_codecommit_ls_remote(
+            repo_url,
+            arn,
+            repo_branch,
+            org_external_id=org_external_id,
+            follow_redirects=follow_redirects,
+        )
+
+    return await call_https_ls_remote(
+        repo_url=repo_url,
+        user=user,
+        password=password,
+        token=token,
+        branch=repo_branch,
+        provider=provider,
+        is_pat=is_pat,
+        follow_redirects=follow_redirects,
+    )