codecov-cli 11.0.0__py3-none-any.whl

codecov_cli/helpers/versioning_systems.py

@@ -0,0 +1,201 @@
+from itertools import chain
+import logging
+import re
+import subprocess
+import typing as t
+from pathlib import Path
+from shutil import which
+
+from codecov_cli.fallbacks import FallbackFieldEnum
+from codecov_cli.helpers.folder_searcher import search_files
+from codecov_cli.helpers.git import parse_git_service, parse_slug
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger("codecovcli")
+
+IGNORE_DIRS = [
+    "*.egg-info",
+    ".DS_Store",
+    ".circleci",
+    ".env",
+    ".envs",
+    ".git",
+    ".gitignore",
+    ".mypy_cache",
+    ".nvmrc",
+    ".nyc_output",
+    ".ruff_cache",
+    ".venv",
+    ".venvns",
+    ".virtualenv",
+    ".virtualenvs",
+    "__pycache__",
+    "bower_components",
+    "build/lib/",
+    "jspm_packages",
+    "node_modules",
+    "vendor",
+    "virtualenv",
+    "virtualenvs",
+]
+
+IGNORE_PATHS = [
+    "*.gif",
+    "*.jpeg",
+    "*.jpg",
+    "*.md",
+    "*.png",
+    "shunit2*",
+]
+
+
+class VersioningSystemInterface(ABC):
+    def __repr__(self) -> str:
+        return str(type(self))
+
+    @abstractmethod
+    def get_fallback_value(self, fallback_field: FallbackFieldEnum) -> t.Optional[str]:
+        pass
+
+    @abstractmethod
+    def get_network_root(self) -> t.Optional[Path]:
+        pass
+
+    @abstractmethod
+    def list_relevant_files(
+        self, directory: t.Optional[Path] = None, recurse_submodules: bool = False
+    ) -> t.Optional[t.List[str]]:
+        pass
+
+
+def get_versioning_system() -> t.Optional[VersioningSystemInterface]:
+    for klass in [GitVersioningSystem, NoVersioningSystem]:
+        if klass.is_available():
+            logger.debug(f"versioning system found: {klass}")
+            return klass()
+
+
+class GitVersioningSystem(VersioningSystemInterface):
+    @classmethod
+    def is_available(cls):
+        if which("git") is not None:
+            p = subprocess.run(
+                ["git", "rev-parse", "--show-toplevel"], capture_output=True
+            )
+            if p.stdout:
+                return True
+        return False
+
+    def get_fallback_value(self, fallback_field: FallbackFieldEnum):
+        if fallback_field == FallbackFieldEnum.commit_sha:
+            # here we will get the commit SHA of the latest commit
+            # that is NOT a merge commit
+            p = subprocess.run(
+                # List current commit parent's SHA
+                ["git", "rev-parse", "HEAD^@"],
+                capture_output=True,
+            )
+            parents_hash = p.stdout.decode().strip().splitlines()
+            if len(parents_hash) == 2:
+                # IFF the current commit is a merge commit it will have 2 parents
+                # We return the 2nd one - The commit that came from the branch merged into ours
+                return parents_hash[1]
+            # At this point we know the current commit is not a merge commit
+            # so we get it's SHA and return that
+            p = subprocess.run(["git", "log", "-1", "--format=%H"], capture_output=True)
+            if p.stdout:
+                return p.stdout.decode().strip()
+
+        if fallback_field == FallbackFieldEnum.branch:
+            p = subprocess.run(
+                ["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True
+            )
+            if p.stdout:
+                branch_name = p.stdout.decode().strip()
+                # branch_name will be 'HEAD' if we are in 'detached HEAD' state
+                return branch_name if branch_name != "HEAD" else None
+
+        if fallback_field == FallbackFieldEnum.slug:
+            # if there are multiple remotes, we will prioritize using the one called 'origin' if it exists, else we will use the first one in 'git remote' list
+
+            p = subprocess.run(["git", "remote"], capture_output=True)
+
+            if not p.stdout:
+                return None
+
+            remotes = p.stdout.decode().strip().splitlines()
+
+            remote_name = "origin" if "origin" in remotes else remotes[0]
+
+            p = subprocess.run(
+                ["git", "ls-remote", "--get-url", remote_name], capture_output=True
+            )
+            if not p.stdout:
+                return None
+
+            remote_url = p.stdout.decode().strip()
+
+            return parse_slug(remote_url)
+
+        if fallback_field == FallbackFieldEnum.git_service:
+            # if there are multiple remotes, we will prioritize using the one called 'origin' if it exists, else we will use the first one in 'git remote' list
+
+            p = subprocess.run(["git", "remote"], capture_output=True)
+            if not p.stdout:
+                return None
+
+            remotes = p.stdout.decode().strip().splitlines()
+            remote_name = "origin" if "origin" in remotes else remotes[0]
+            p = subprocess.run(
+                ["git", "ls-remote", "--get-url", remote_name], capture_output=True
+            )
+            if not p.stdout:
+                return None
+
+            remote_url = p.stdout.decode().strip()
+            return parse_git_service(remote_url)
+
+        return None
+
+    def get_network_root(self):
+        p = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True)
+        if p.stdout:
+            return Path(p.stdout.decode().rstrip())
+        return None
+
+    def list_relevant_files(
+        self, directory: t.Optional[Path] = None, recurse_submodules: bool = False
+    ) -> t.List[str]:
+        dir_to_use = directory or self.get_network_root()
+        if dir_to_use is None:
+            raise ValueError("Can't determine root folder")
+
+        cmd = ["git", "-C", str(dir_to_use), "ls-files", "-z"]
+        if recurse_submodules:
+            cmd.append("--recurse-submodules")
+        res = subprocess.run(cmd, capture_output=True)
+        return res.stdout.decode().split("\0")
+
+
+class NoVersioningSystem(VersioningSystemInterface):
+    @classmethod
+    def is_available(cls):
+        return True
+
+    def get_network_root(self):
+        return Path.cwd()
+
+    def get_fallback_value(self, fallback_field: FallbackFieldEnum):
+        return None
+
+    def list_relevant_files(
+        self, directory: t.Optional[Path] = None, recurse_submodules: bool = False
+    ) -> t.List[str]:
+        dir_to_use = directory or self.get_network_root()
+        if dir_to_use is None:
+            raise ValueError("Can't determine root folder")
+
+        files = search_files(
+            dir_to_use, folders_to_ignore=[], filename_include_regex=re.compile("")
+        )
+        return [f.relative_to(dir_to_use).as_posix() for f in files]

codecov_cli/main.py

@@ -0,0 +1,99 @@
+import logging
+import pathlib
+import typing
+
+import click
+
+from codecov_cli import __version__
+from codecov_cli.opentelemetry import init_telem
+from codecov_cli.commands.base_picking import pr_base_picking
+from codecov_cli.commands.commit import create_commit
+from codecov_cli.commands.create_report_result import create_report_results
+from codecov_cli.commands.empty_upload import empty_upload
+from codecov_cli.commands.get_report_results import get_report_results
+from codecov_cli.commands.labelanalysis import label_analysis
+from codecov_cli.commands.process_test_results import process_test_results
+from codecov_cli.commands.report import create_report
+from codecov_cli.commands.send_notifications import send_notifications
+from codecov_cli.commands.staticanalysis import static_analysis
+from codecov_cli.commands.upload import do_upload
+from codecov_cli.commands.upload_coverage import upload_coverage
+from codecov_cli.commands.upload_process import upload_process
+from codecov_cli.helpers.ci_adapters import get_ci_adapter, get_ci_providers_list
+from codecov_cli.helpers.config import load_cli_config
+from codecov_cli.helpers.logging_utils import configure_logger
+from codecov_cli.helpers.versioning_systems import get_versioning_system
+
+logger = logging.getLogger("codecovcli")
+
+
+@click.group()
+@click.option(
+    "--auto-load-params-from",
+    type=click.Choice(
+        [provider.get_service_name() for provider in get_ci_providers_list()],
+        case_sensitive=False,
+    ),
+)
+@click.option(
+    "--codecov-yml-path",
+    type=click.Path(path_type=pathlib.Path),
+    default=None,
+)
+@click.option(
+    "--enterprise-url", "--url", "-u", help="Change the upload host (Enterprise use)"
+)
+@click.option("-v", "--verbose", "verbose", help="Use verbose logging", is_flag=True)
+@click.option(
+    "--disable-telem", help="Disable sending telemetry data to Codecov", is_flag=True
+)
+@click.pass_context
+@click.version_option(__version__, prog_name="codecovcli")
+def cli(
+    ctx: click.Context,
+    auto_load_params_from: typing.Optional[str],
+    codecov_yml_path: pathlib.Path,
+    enterprise_url: str,
+    verbose: bool = False,
+    disable_telem: bool = False,
+):
+    ctx.obj["cli_args"] = ctx.params
+    ctx.obj["cli_args"]["version"] = f"cli-{__version__}"
+    configure_logger(logger, log_level=(logging.DEBUG if verbose else logging.INFO))
+    ctx.help_option_names = ["-h", "--help"]
+    ctx.obj["ci_adapter"] = get_ci_adapter(auto_load_params_from)
+    ctx.obj["versioning_system"] = get_versioning_system()
+    ctx.obj["codecov_yaml"] = load_cli_config(codecov_yml_path)
+    if ctx.obj["codecov_yaml"] is None:
+        logger.debug("No codecov_yaml found")
+    elif (token := ctx.obj["codecov_yaml"].get("codecov", {}).get("token")) is not None:
+        ctx.default_map = {ctx.invoked_subcommand: {"token": token}}
+    ctx.obj["enterprise_url"] = enterprise_url
+    ctx.obj["disable_telem"] = disable_telem
+
+    init_telem(ctx.obj)
+
+
+cli.add_command(do_upload)
+cli.add_command(create_commit)
+cli.add_command(create_report)
+cli.add_command(pr_base_picking)
+cli.add_command(empty_upload)
+cli.add_command(upload_coverage)
+cli.add_command(upload_process)
+cli.add_command(send_notifications)
+cli.add_command(process_test_results)
+
+# deprecated commands:
+cli.add_command(create_report_results)
+cli.add_command(get_report_results)
+cli.add_command(label_analysis)
+cli.add_command(static_analysis)
+
+
+def run():
+    cli(obj={})
+
+
+if __name__ == "__main__":
+    run()

codecov_cli/opentelemetry.py

@@ -0,0 +1,26 @@
+import os
+import time
+
+import sentry_sdk
+
+from codecov_cli import __version__
+
+
+def init_telem(ctx):
+    if ctx["disable_telem"]:
+        return
+    if ctx["enterprise_url"]:  # dont run on dedicated cloud
+        return
+    if os.getenv("CODECOV_ENV", "production") == "test":
+        return
+
+    sentry_sdk.init(
+        dsn="https://0bea75c61745c221a6ef1ac1709b1f4d@o26192.ingest.us.sentry.io/4508615876083713",
+        enable_tracing=True,
+        environment=os.getenv("CODECOV_ENV", "production"),
+        release=f"cli@{__version__}",
+    )
+
+
+def close_telem():
+    sentry_sdk.flush()

codecov_cli/plugins/__init__.py

@@ -0,0 +1,92 @@
+import logging
+import typing
+from importlib import import_module
+
+import click
+
+from codecov_cli.plugins.compress_pycoverage_contexts import CompressPycoverageContexts
+from codecov_cli.plugins.gcov import GcovPlugin
+from codecov_cli.plugins.pycoverage import Pycoverage
+from codecov_cli.plugins.xcode import XcodePlugin
+
+logger = logging.getLogger("codecovcli")
+
+
+class NoopPlugin(object):
+    def run_preparation(self, collector):
+        pass
+
+
+def select_preparation_plugins(
+    cli_config: typing.Dict, plugin_names: typing.List[str], plugin_config: typing.Dict
+):
+    plugins = [_get_plugin(cli_config, p, plugin_config) for p in plugin_names]
+    logger.debug(
+        "Selected preparation plugins",
+        extra=dict(
+            extra_log_attributes=dict(
+                selected_plugins=list(map(type, plugins)),
+                cli_config=cli_config,
+            )
+        ),
+    )
+    return plugins
+
+
+def _load_plugin_from_yaml(plugin_dict: typing.Dict):
+    try:
+        module_obj = import_module(plugin_dict["module"])
+        class_obj = getattr(module_obj, plugin_dict["class"])
+    except ModuleNotFoundError:
+        click.secho(
+            f"Unable to dynamically load module {plugin_dict['module']}",
+            err=True,
+        )
+        return NoopPlugin()
+    except AttributeError:
+        click.secho(
+            f"Unable to dynamically load class {plugin_dict['class']} from module {plugin_dict['module']}",
+            err=True,
+        )
+        return NoopPlugin()
+    try:
+        params = plugin_dict.get("params", None)
+        if params:
+            return class_obj(**plugin_dict["params"])
+        else:
+            return class_obj()
+
+    except TypeError:
+        click.secho(
+            f"Unable to instantiate {class_obj} with provided parameters {plugin_dict.get('params', '')}",
+            err=True,
+        )
+        return NoopPlugin()
+
+
+def _get_plugin(cli_config, plugin_name, plugin_config):
+    if plugin_name == "noop":
+        return NoopPlugin()
+    if plugin_name == "gcov":
+        return GcovPlugin(
+            plugin_config.get("project_root", None),
+            plugin_config.get("folders_to_ignore", None),
+            plugin_config.get("gcov_executable", "gcov"),
+            plugin_config.get("gcov_include", None),
+            plugin_config.get("gcov_ignore", None),
+            plugin_config.get("gcov_args", None),
+        )
+    if plugin_name == "pycoverage":
+        config = cli_config.get("plugins", {}).get("pycoverage", {})
+        return Pycoverage(config)
+    if plugin_name == "xcode":
+        return XcodePlugin(
+            plugin_config.get("swift_project", None),
+        )
+    if plugin_name == "compress-pycoverage":
+        config = cli_config.get("plugins", {}).get("compress-pycoverage", {})
+        return CompressPycoverageContexts(config)
+    if cli_config and plugin_name in cli_config.get("plugins", {}):
+        return _load_plugin_from_yaml(cli_config["plugins"][plugin_name])
+    click.secho(f"Unable to find plugin {plugin_name}", fg="magenta", err=True)
+    return NoopPlugin()

codecov_cli/plugins/compress_pycoverage_contexts.py

@@ -0,0 +1,141 @@
+import json
+import logging
+import pathlib
+from decimal import Decimal
+from typing import Any, List
+
+import ijson
+import sentry_sdk
+
+from codecov_cli.plugins.types import PreparationPluginReturn
+
+logger = logging.getLogger("codecovcli")
+
+
+class Encoder(json.JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if isinstance(o, Decimal):
+            return str(o)
+        return super().default(o)
+
+
+class CompressPycoverageContextsConfig(dict):
+    @property
+    def file_to_compress(self) -> pathlib.Path:
+        """
+        The report file to compress.
+        file_to_compress: Union[str, pathlib.Path] [default coverage.json]
+        """
+        return pathlib.Path(self.get("file_to_compress", "coverage.json"))
+
+    @property
+    def delete_uncompressed(self) -> bool:
+        """
+        Flag indicating to delete the original file after compressing.
+        Recommended to avoid uploading the uncompressed file.
+        delete_uncompressed: bool [default True]
+        """
+        return self.get("delete_uncompressed", True)
+
+
+class CompressPycoverageContexts(object):
+    def __init__(self, config: dict = None) -> None:
+        if config is None:
+            config = {}
+        self.config = CompressPycoverageContextsConfig(config)
+        self.file_to_compress = self.config.file_to_compress
+        self.file_to_write = pathlib.Path(
+            str(self.file_to_compress).replace(".json", "") + ".codecov.json"
+        )
+
+    def run_preparation(self, collector) -> PreparationPluginReturn:
+        with sentry_sdk.start_span(name="compress_pycoverage"):
+            if not self.file_to_compress.exists():
+                logger.warning(
+                    f"File to compress {self.file_to_compress} not found. Aborting"
+                )
+                return PreparationPluginReturn(
+                    success=False,
+                    messages=[f"File to compress {self.file_to_compress} not found."],
+                )
+            if not self.file_to_compress.is_file():
+                logger.warning(
+                    f"File to compress {self.file_to_compress} is not a file. Aborting"
+                )
+                return PreparationPluginReturn(
+                    success=False,
+                    messages=[f"File to compress {self.file_to_compress} is not a file."],
+                )
+            # Create in and out streams
+            fd_in = open(self.file_to_compress, "rb")
+            fd_out = open(self.file_to_write, "w")
+            # Compress the file
+            fd_out.write("{")
+            self._copy_meta(fd_in, fd_out)
+            files_in_report = ijson.kvitems(fd_in, "files")
+            self._compress_files(files_in_report, fd_out)
+            fd_out.write("}")
+            # Close streams
+            fd_in.close()
+            fd_out.close()
+            logger.info(f"Compressed report written to {self.file_to_write}")
+            # Delete original file if needed
+            if self.config.delete_uncompressed:
+                logger.info(f"Deleting file {self.file_to_compress}")
+                self.file_to_compress.unlink()
+            return PreparationPluginReturn(success=True, messages=[])
+
+    def _compress_files(self, files_in_report, fd_out) -> None:
+        """
+        Compress the 'files' entry in the coverage data.
+        This is done by creating a labels table [str -> int] mapping labels to an index.
+        This index then substitutes the label itself in the contexts
+        """
+        labels_table = {}
+        nxt_idx = 0
+
+        fd_out.write('"files":{')
+        for file_name, file_coverage_details in files_in_report:
+            self._copy_file_details(file_name, file_coverage_details, fd_out)
+            fd_out.write('"contexts": {')
+            contexts = file_coverage_details["contexts"]
+            for line_number, labels in contexts.items():
+                fd_out.write(f'"{line_number}":')
+                new_labels = []
+                for label in labels:
+                    stripped_label = label.split("|")[0]  # removes '|run' from label
+                    if stripped_label not in labels_table:
+                        labels_table[stripped_label] = nxt_idx
+                        nxt_idx += 1
+                    new_labels.append(labels_table[stripped_label])
+                fd_out.write(json.dumps(new_labels))
+                # fd_out.write(self._bitmask_label_indexes(new_labels))
+                fd_out.write(",")
+            if len(contexts):  # Avoid removing '{' if contexts == {}
+                # Because there will be an extra ',' after the last line
+                fd_out.seek(fd_out.tell() - 1)
+            # One curly brace for the 'contexts', one for the file_name
+            fd_out.write("}},")
+        # Because there will be an extra ',' after the last file_name
+        fd_out.seek(fd_out.tell() - 1)
+        fd_out.write("},")
+        # Save the inverted index of labels table in the report
+        # So when we are processing the result we have int -> label
+        fd_out.write(
+            f'"labels_table": {json.dumps({value: key for key, value in labels_table.items()})}'
+        )
+
+    def _copy_file_details(self, file_name, file_details, fd_out) -> None:
+        fd_out.write(f'"{file_name}":{{')
+        fd_out.write(f'"executed_lines": {file_details["executed_lines"]},')
+        fd_out.write(f'"summary": {json.dumps(file_details["summary"], cls=Encoder)},')
+        fd_out.write(f'"missing_lines": {file_details["missing_lines"]},')
+        fd_out.write(f'"excluded_lines": {file_details["excluded_lines"]},')
+
+    def _copy_meta(self, fd_in, fd_out) -> None:
+        meta = ijson.kvitems(fd_in, "")
+        for key, value in meta:
+            if key == "files":
+                continue
+            fd_out.write(f'"{key}": {json.dumps(value, cls=Encoder)},')
+        fd_in.seek(0)

codecov_cli/plugins/gcov.py

@@ -0,0 +1,69 @@
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import typing
+
+import sentry_sdk
+
+from codecov_cli.helpers.folder_searcher import globs_to_regex, search_files
+from codecov_cli.plugins.types import PreparationPluginReturn
+
+logger = logging.getLogger("codecovcli")
+
+
+class GcovPlugin(object):
+    def __init__(
+        self,
+        project_root: typing.Optional[pathlib.Path] = None,
+        folders_to_ignore: typing.Optional[typing.List[str]] = None,
+        executable: typing.Optional[str] = "gcov",
+        patterns_to_include: typing.Optional[typing.List[str]] = None,
+        patterns_to_ignore: typing.Optional[typing.List[str]] = None,
+        extra_arguments: typing.Optional[typing.List[str]] = None,
+    ):
+        self.executable = executable or "gcov"
+        self.extra_arguments = extra_arguments or []
+        self.folders_to_ignore = folders_to_ignore or []
+        self.patterns_to_ignore = patterns_to_ignore or []
+        self.patterns_to_include = patterns_to_include or []
+        self.project_root = project_root or pathlib.Path(os.getcwd())
+
+    def run_preparation(self, collector) -> PreparationPluginReturn:
+        with sentry_sdk.start_span(name="gcov"):
+            logger.debug(
+                f"Running {self.executable} plugin...",
+            )
+
+            if shutil.which(self.executable) is None:
+                logger.warning(f"{self.executable} is not installed or can't be found.")
+                return
+
+            filename_include_regex = globs_to_regex(["*.gcno", *self.patterns_to_include])
+            filename_exclude_regex = globs_to_regex(self.patterns_to_ignore)
+
+            matched_paths = [
+                str(path)
+                for path in search_files(
+                    self.project_root,
+                    self.folders_to_ignore,
+                    filename_include_regex=filename_include_regex,
+                    filename_exclude_regex=filename_exclude_regex,
+                )
+            ]
+
+            if not matched_paths:
+                logger.warning(f"No {self.executable} data found.")
+                return
+
+            logger.warning(f"Running {self.executable} on the following list of files:")
+            for path in matched_paths:
+                logger.warning(path)
+
+            s = subprocess.run(
+                [self.executable, "-pb", *self.extra_arguments, *matched_paths],
+                cwd=self.project_root,
+                capture_output=True,
+            )
+            return PreparationPluginReturn(success=True, messages=[s.stdout])