fluidattacks_core_git 12.0.0__py3-none-any.whl

fluidattacks_core/git/__init__.py

@@ -0,0 +1,382 @@
+import asyncio
+import logging
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from subprocess import (  # nosec
+    SubprocessError,
+)
+
+from git.exc import GitError
+from git.repo import Repo
+
+from .classes import (
+    CommitInfo,
+    InvalidParameter,
+    RebaseResult,
+)
+from .clone import (
+    call_codecommit_clone,
+    https_clone,
+    ssh_clone,
+)
+from .download_repo import (
+    download_repo_from_s3,
+    remove_symlinks_in_directory,
+    reset_repo,
+)
+from .https_utils import (
+    https_ls_remote,
+)
+from .remote import (
+    ls_remote,
+)
+from .show_file import (
+    show_file_at_ref,
+    show_file_at_ref_https,
+    show_file_at_ref_ssh,
+)
+from .ssh_utils import (
+    SSHHostValidationError,
+    ssh_ls_remote,
+)
+from .upload_repo import (
+    MultipartCallbacks,
+    UploadPartResult,
+    upload_repo_to_s3,
+)
+from .utils import run_git
+
+LOGGER = logging.getLogger(__name__)
+
+__all__ = [
+    "CommitInfo",
+    "InvalidParameter",
+    "MultipartCallbacks",
+    "RebaseResult",
+    "SSHHostValidationError",
+    "UploadPartResult",
+    "clone",
+    "disable_quotepath",
+    "download_repo_from_s3",
+    "get_head_commit",
+    "get_last_commit_info_new",
+    "get_line_author",
+    "get_modified_filenames",
+    "https_clone",
+    "https_ls_remote",
+    "is_commit_in_branch",
+    "ls_remote",
+    "rebase",
+    "remove_symlinks_in_directory",
+    "reset_repo",
+    "show_file_at_ref",
+    "show_file_at_ref_https",
+    "show_file_at_ref_ssh",
+    "ssh_clone",
+    "ssh_ls_remote",
+    "upload_repo_to_s3",
+]
+
+
+async def disable_quotepath(git_path: str) -> None:
+    try:
+        _, stderr, returncode = await run_git(
+            f"--git-dir={git_path}",
+            "config",
+            "core.quotepath",
+            "off",
+        )
+    except asyncio.exceptions.TimeoutError:
+        LOGGER.exception(
+            "Timed out disabling git quotepath",
+            extra={"extra": {"git_path": git_path}},
+        )
+        return
+
+    if returncode != 0:
+        LOGGER.error(
+            "Failed to disable git quotepath",
+            extra={"extra": {"git_path": git_path, "stderr": stderr.decode()}},
+        )
+
+
+async def get_last_commit_info_new(
+    repo_path: str,
+    filename: str,
+) -> CommitInfo | None:
+    proc = await asyncio.create_subprocess_exec(
+        "git",
+        "log",
+        "--max-count",
+        "1",
+        "--format=%H%n%ce%n%cI",
+        "--",
+        filename,
+        stderr=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        cwd=repo_path,
+    )
+    stdout, stderr = await proc.communicate()
+    git_log = stdout.decode().splitlines()
+
+    if stderr or proc.returncode != 0 or not git_log:
+        return None
+
+    return CommitInfo(
+        hash=git_log[0],
+        author=git_log[1],
+        modified_date=datetime.fromisoformat(git_log[2]),
+    )
+
+
+async def get_line_author(
+    repo_path: str,
+    filename: str,
+    line: int,
+    rev: str = "HEAD",
+) -> CommitInfo | None:
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "git",
+            "blame",
+            "-L",
+            f"{line!s},+1",
+            "-l",
+            "-p",
+            "-M",
+            "-C",
+            "-C",
+            rev,
+            "--",
+            filename,
+            stderr=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            cwd=repo_path,
+        )
+        stdout, stderr = await proc.communicate()
+        cmd_output = stdout.decode("utf-8", "ignore")
+    except (
+        OSError,
+        SubprocessError,
+        UnicodeDecodeError,
+    ):
+        LOGGER.exception(
+            "An error occurred while getting the line author",
+            extra={
+                "extra": {
+                    "repo_path": repo_path,
+                    "filename": filename,
+                    "line": str(line),
+                },
+            },
+        )
+
+        return None
+
+    if stderr or proc.returncode != 0 or not cmd_output:
+        return None
+
+    commit_hash = cmd_output.splitlines()[0].split(" ")[0]
+    mail_search = re.search(r"author-mail <(.*?)>", cmd_output)
+    author_email = mail_search.group(1) if mail_search else ""
+    time_search = re.search(r"committer-time (\d*)", cmd_output)
+    committer_time = time_search.group(1) if time_search else "0"
+    commit_date = datetime.fromtimestamp(float(committer_time), UTC)
+
+    return CommitInfo(
+        hash=commit_hash,
+        author=author_email,
+        modified_date=commit_date,
+    )
+
+
+async def get_modified_filenames(repo_path: str, commit_sha: str) -> list[str]:
+    proc = await asyncio.create_subprocess_exec(
+        "git",
+        "diff",
+        "--name-only",
+        f"{commit_sha}..HEAD",
+        stderr=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        cwd=repo_path,
+    )
+    stdout, stderr = await proc.communicate()
+    if stderr or proc.returncode != 0:
+        return []
+
+    return stdout.decode().splitlines()
+
+
+async def is_commit_in_branch(
+    repo_path: str,
+    branch: str,
+    commit_sha: str,
+) -> bool:
+    proc = await asyncio.create_subprocess_exec(
+        "git",
+        "branch",
+        "--contains",
+        f"{commit_sha}",
+        stderr=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        cwd=repo_path,
+    )
+    stdout, stderr = await proc.communicate()
+    if stderr or proc.returncode != 0:
+        return False
+
+    return branch in stdout.decode()
+
+
+def rebase(  # noqa: PLR0913
+    repo: Repo,
+    *,
+    path: str,
+    line: int,
+    rev_a: str,
+    rev_b: str,
+    ignore_errors: bool = True,
+) -> RebaseResult | None:
+    try:
+        result: list[str] = (
+            repo.git(c="core.quotepath=off")
+            .blame(
+                f"{rev_a}..{rev_b}",
+                "--",
+                path,
+                L=f"{line},+1",
+                l=True,
+                p=True,
+                show_number=True,
+                reverse=True,
+                show_name=True,
+                M=True,
+                C=True,
+            )
+            .splitlines()
+        )
+    except GitError:
+        if ignore_errors:
+            LOGGER.exception("A git error occurred while rebasing")
+            return None
+
+        raise
+
+    new_rev = result[0].split(" ")[0]
+    new_line = int(result[0].split(" ")[1])
+    new_path = next(
+        (row.split(" ", maxsplit=1)[1] for row in result if row.startswith("filename ")),
+        path,
+    )
+    # With core.quotepath=off, git only wraps the path in double quotes when it
+    # C-style-escapes "unusual" characters (", \, control chars). Bare Unicode
+    # paths (CJK, Cyrillic, ...) come through as proper Python strings and must
+    # skip the unicode-escape pipeline, which would fail .encode("latin-1") on
+    # codepoints above U+00FF.
+    if new_path.startswith('"'):
+        try:
+            decoded_bytes = new_path.encode("latin-1").decode("unicode-escape").encode("latin-1")
+        except (UnicodeDecodeError, UnicodeEncodeError):
+            if ignore_errors:
+                LOGGER.exception(
+                    "Error decoding the new path",
+                    extra={
+                        "extra": {
+                            "path": path,
+                            "new_path": new_path,
+                        },
+                    },
+                )
+                return None
+
+            raise
+
+        try:
+            new_path = decoded_bytes.decode("utf-8").strip('"')
+        except UnicodeDecodeError:
+            # Filesystem locales may emit non-UTF-8 (e.g. Latin-1) path bytes
+            new_path = decoded_bytes.decode("latin-1").strip('"')
+
+    return RebaseResult(path=new_path, line=new_line, rev=new_rev)
+
+
+def get_head_commit(path_to_repo: Path, branch: str) -> str | None:
+    try:
+        return (
+            Repo(path_to_repo.resolve(), search_parent_directories=True).heads[branch].object.hexsha
+        )
+    except (GitError, AttributeError, IndexError):
+        return None
+
+
+async def clone(  # noqa: PLR0913
+    repo_url: str,
+    repo_branch: str,
+    *,
+    temp_dir: str,
+    credential_key: str | None = None,
+    user: str | None = None,
+    password: str | None = None,
+    token: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    arn: str | None = None,
+    org_external_id: str | None = None,
+    follow_redirects: bool = False,
+    mirror: bool = False,
+) -> tuple[str | None, str | None]:
+    if credential_key:
+        return await ssh_clone(
+            branch=repo_branch,
+            credential_key=credential_key,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            mirror=mirror,
+        )
+    if user is not None and password is not None:
+        return await https_clone(
+            branch=repo_branch,
+            password=password,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            token=None,
+            user=user,
+            follow_redirects=follow_redirects,
+            mirror=mirror,
+        )
+    if token is not None:
+        return await https_clone(
+            branch=repo_branch,
+            password=None,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            token=token,
+            user=None,
+            provider=provider,
+            is_pat=is_pat,
+            follow_redirects=follow_redirects,
+            mirror=mirror,
+        )
+    if arn is not None and org_external_id is not None:
+        return await call_codecommit_clone(
+            branch=repo_branch,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            arn=arn,
+            org_external_id=org_external_id,
+            follow_redirects=follow_redirects,
+            mirror=mirror,
+        )
+
+    if repo_url.startswith("http"):
+        # it can be a public repository
+        return await https_clone(
+            branch=repo_branch,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            follow_redirects=follow_redirects,
+            mirror=mirror,
+        )
+
+    raise InvalidParameter

fluidattacks_core/git/classes.py

@@ -0,0 +1,29 @@
+from datetime import (
+    datetime,
+)
+from typing import (
+    NamedTuple,
+)
+
+
+class CommitInfo(NamedTuple):
+    hash: str
+    author: str
+    modified_date: datetime
+
+
+class RebaseResult(NamedTuple):
+    path: str
+    line: int
+    rev: str
+
+
+class InvalidParameter(Exception):  # noqa: N818
+    """Exception to control empty required parameters."""
+
+    def __init__(self, field: str = "") -> None:
+        if field:
+            msg = f"Exception - Field {field} is invalid"
+        else:
+            msg = "Exception - Error value is not valid"
+        super().__init__(msg)

fluidattacks_core/git/clone.py

@@ -0,0 +1,243 @@
+import asyncio
+import json
+import logging
+import os
+import uuid
+
+import boto3
+from botocore.exceptions import ClientError
+
+from .codecommit_utils import (
+    extract_region,
+)
+from .ssh_utils import (
+    parse_ssh_url,
+    ssh_git_env_context,
+)
+from .utils import (
+    format_url,
+    get_https_git_config_args,
+)
+
+LOGGER = logging.getLogger(__name__)
+MSG = "Repo cloning failed"
+
+
+async def ssh_clone(
+    *,
+    branch: str,
+    credential_key: str,
+    repo_url: str,
+    temp_dir: str,
+    mirror: bool = False,
+) -> tuple[str | None, str | None]:
+    parsed_repo_url = parse_ssh_url(repo_url)
+    folder_to_clone_root = f"{temp_dir}/{uuid.uuid4()}"
+    with ssh_git_env_context(temp_dir, credential_key) as env:
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                "git",
+                "-c",
+                "core.symlinks=false",
+                "clone",
+                *(
+                    ["--mirror"]
+                    if mirror
+                    else [
+                        "--branch",
+                        branch,
+                        "--single-branch",
+                    ]
+                ),
+                "--",
+                parsed_repo_url,
+                folder_to_clone_root,
+                stderr=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                env=env,
+                cwd=temp_dir,
+            )
+            _, stderr = await proc.communicate()
+        except OSError as ex:
+            LOGGER.exception(MSG, extra={"extra": {"branch": branch, "repo": repo_url}})
+            return None, str(ex)
+
+    if mirror and proc.returncode == 0:
+        with open(f"{folder_to_clone_root}/.info.json", "w") as f:  # noqa: ASYNC230,PTH123
+            json.dump({"fluid_branch": branch, "repo": repo_url}, f)
+    if proc.returncode == 0:
+        return (folder_to_clone_root, None)
+
+    LOGGER.error(MSG, extra={"extra": {"message": stderr.decode()}})
+
+    return (None, stderr.decode("utf-8"))
+
+
+async def https_clone(  # noqa: PLR0913
+    *,
+    branch: str,
+    repo_url: str,
+    temp_dir: str,
+    password: str | None = None,
+    token: str | None = None,
+    user: str | None = None,
+    provider: str | None = None,
+    is_pat: bool = False,
+    follow_redirects: bool = False,
+    mirror: bool = False,
+) -> tuple[str | None, str | None]:
+    url = format_url(
+        repo_url=repo_url,
+        user=user,
+        password=password,
+        token=token,
+        provider=provider,
+        is_pat=is_pat,
+    )
+    config_args = get_https_git_config_args(
+        follow_redirects=follow_redirects,
+        is_pat=is_pat,
+        token=token,
+    )
+    folder_to_clone_root = f"{temp_dir}/{uuid.uuid4()}"
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "git",
+            *config_args,
+            "clone",
+            *(
+                ["--mirror"]
+                if mirror
+                else [
+                    "--branch",
+                    branch,
+                    "--single-branch",
+                ]
+            ),
+            "--",
+            url,
+            folder_to_clone_root,
+            stderr=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            cwd=temp_dir,
+        )
+        _, stderr = await proc.communicate()
+    except OSError as ex:
+        LOGGER.exception(MSG, extra={"extra": {"branch": branch, "repo": repo_url}})
+
+        return None, str(ex)
+
+    if mirror and proc.returncode == 0:
+        with open(f"{folder_to_clone_root}/.info.json", "w") as f:  # noqa: ASYNC230,PTH123
+            json.dump({"fluid_branch": branch, "repo": repo_url}, f)
+
+    if proc.returncode == 0:
+        return (folder_to_clone_root, None)
+
+    LOGGER.error(MSG, extra={"extra": {"message": stderr.decode()}})
+
+    return (None, stderr.decode("utf-8"))
+
+
+async def codecommit_clone(  # noqa: PLR0913
+    *,
+    env: dict[str, str],
+    branch: str,
+    repo_url: str,
+    temp_dir: str,
+    mirror: bool = False,
+    follow_redirects: bool = False,
+) -> tuple[str | None, str | None]:
+    folder_to_clone_root = f"{temp_dir}/{uuid.uuid4()}"
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "git",
+            "-c",
+            "core.symlinks=false",
+            "-c",
+            "http.sslVerify=false",
+            "-c",
+            f"http.followRedirects={follow_redirects}",
+            "clone",
+            *(
+                ["--mirror"]
+                if mirror
+                else [
+                    "--branch",
+                    branch,
+                    "--single-branch",
+                ]
+            ),
+            "--",
+            repo_url,
+            folder_to_clone_root,
+            cwd=temp_dir,
+            env={**os.environ.copy(), **env},
+            stderr=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+        )
+        _, stderr = await proc.communicate()
+    except OSError as ex:
+        LOGGER.exception(MSG, extra={"extra": {"branch": branch, "repo": repo_url}})
+
+        return None, str(ex)
+
+    if mirror and proc.returncode == 0:
+        with open(f"{folder_to_clone_root}/.info.json", "w") as f:  # noqa: ASYNC230, PTH123
+            json.dump({"fluid_branch": branch, "repo": repo_url}, f)
+
+    if proc.returncode == 0:
+        return (folder_to_clone_root, None)
+
+    LOGGER.error(MSG, extra={"extra": {"message": stderr.decode()}})
+
+    return (None, stderr.decode("utf-8"))
+
+
+async def call_codecommit_clone(  # noqa: PLR0913
+    *,
+    branch: str,
+    repo_url: str,
+    temp_dir: str,
+    arn: str,
+    org_external_id: str,
+    follow_redirects: bool = False,
+    mirror: bool = False,
+) -> tuple[str | None, str | None]:
+    try:
+        sts_client = boto3.client("sts")
+        assumed_role = sts_client.assume_role(
+            RoleArn=arn,
+            RoleSessionName=f"session-{uuid.uuid4()}",
+            ExternalId=org_external_id,
+        )
+        credentials = assumed_role["Credentials"]
+
+        return await codecommit_clone(
+            env={
+                "AWS_ACCESS_KEY_ID": credentials["AccessKeyId"],
+                "AWS_SECRET_ACCESS_KEY": credentials["SecretAccessKey"],
+                "AWS_SESSION_TOKEN": credentials["SessionToken"],
+                "AWS_DEFAULT_REGION": extract_region(repo_url),
+            },
+            branch=branch,
+            repo_url=repo_url,
+            temp_dir=temp_dir,
+            follow_redirects=follow_redirects,
+            mirror=mirror,
+        )
+
+    except ClientError as exc:
+        LOGGER.exception(
+            MSG,
+            extra={
+                "extra": {
+                    "repo_url": repo_url,
+                    "arn": arn,
+                    "org_external_id": org_external_id,
+                    "exc": exc,
+                },
+            },
+        )
+
+        return None, str(exc)