codecov-cli 11.0.0__py3-none-any.whl

codecov_cli/services/upload/file_finder.py

@@ -0,0 +1,320 @@
+import logging
+import os
+from pathlib import Path
+from typing import Iterable, List, Optional, Pattern
+
+import sentry_sdk
+
+from codecov_cli.helpers.folder_searcher import globs_to_regex, search_files
+from codecov_cli.helpers.upload_type import ReportType
+from codecov_cli.types import UploadCollectionResultFile
+
+logger = logging.getLogger("codecovcli")
+
+
+coverage_files_patterns = [
+    "*.clover",
+    "*.codecov.*",
+    "*.gcov",
+    "*.lcov",
+    "*.lst",
+    "*coverage*.*",
+    "*Jacoco*.xml",
+    "clover.xml",
+    "cobertura.xml",
+    "codecov-result.json",
+    "codecov.*",
+    "cover.out",
+    "coverage-final.json",
+    "excoveralls.json",
+    "gcov.info",
+    "jacoco*.xml",
+    "lcov.dat",
+    "pylcov.dat",
+    "lcov.info",
+    "luacov.report.out",
+    "naxsi.info",
+    "nosetests.xml",
+    "report.xml",
+    "test_cov.xml",
+]
+
+test_results_files_patterns = [
+    "*junit*.xml",
+    "*test*.xml",
+    # the actual JUnit (Java) prefixes the tests with "TEST-"
+    "*TEST-*.xml",
+]
+
+coverage_files_excluded_patterns = [
+    "*.*js",
+    "*.SHA256SUM",
+    "*.am",
+    "*.bash",
+    "*.bat",
+    "*.bw",
+    "*.cfg",
+    "*.class",
+    "*.cmake",
+    "*.conf",
+    "*.coverage",
+    "*.cp",
+    "*.cpp",
+    "*.crt",
+    "*.csg",
+    "*.css",
+    "*.csv",
+    "*.dart",
+    "*.data",
+    "*.db",
+    "*.dox",
+    "*.ec",
+    "*.egg",
+    "*.egg-info",
+    "*.el",
+    "*.env",
+    "*.erb",
+    "*.err",
+    "*.exe",
+    "*.feature",
+    "*.ftl",
+    "*.gif",
+    "*.go",
+    "*.gradle",
+    "*.gz",
+    "*.h",
+    "*.html",
+    "*.in",
+    "*.jade",
+    "*.jar*",
+    "*.jpeg",
+    "*.jpg",
+    "*.js",
+    "*.less",
+    "*.library",
+    "*.log",
+    "*.m4",
+    "*.mak*",
+    "*.map",
+    "*.md",
+    "*.module",
+    "*.mp4",
+    "*.o",
+    "*.p12",
+    "*.pem",
+    "*.png",
+    "*.pom*",
+    "*.profdata",
+    "*.proto",
+    "*.prototxt",
+    "*.ps1",
+    "*.pth",
+    "*.py",
+    "*.pyc",
+    "*.pyo",
+    "*.rake",
+    "*.rb",
+    "*.rsp",
+    "*.rst",
+    "*.ru",
+    "*.sbt",
+    "*.scss",
+    "*.serialized",
+    "*.sh",
+    "*.sha256sum",
+    "*.snapshot",
+    "*.sql",
+    "*.svg",
+    "*.tar.tz",
+    "*.template",
+    "*.ts",
+    "*.whl",
+    "*.xcconfig",
+    "*.xcoverage.*",
+    "*.yaml",
+    "*.yml",
+    "*.zip",
+    "*/classycle/report.xml",
+    "*codecov.yml",
+    "*~",
+    ".*coveragerc",
+    ".coverage*",
+    ".ds_store",
+    ".git*",
+    ".nvmrc",
+    "codecov.SHA256SUM",
+    "codecov.SHA256SUM.sig",
+    "codecov.yaml",
+    "coverage-summary.json",
+    "createdFiles.lst",
+    "fullLocaleNames.lst",
+    "include.lst",
+    "inputFiles.lst",
+    "phpunit-code-coverage.xml",
+    "phpunit-coverage.xml",
+    "remapInstanbul.coverage*.json",
+    "scoverage.measurements.*",
+    "test-result-*-codecoverage.json",
+    "test_*_coverage.txt",
+    "testrunner-coverage*",
+]
+
+test_results_files_excluded_patterns = (
+    coverage_files_patterns + coverage_files_excluded_patterns
+)
+
+
+default_folders_to_ignore = [
+    "vendor",
+    "bower_components",
+    ".circleci",
+    "conftest_*.c.gcov",
+    ".egg-info*",
+    ".env",
+    ".envs",
+    ".git",
+    ".go",
+    ".hg",
+    ".map",
+    ".marker",
+    ".tox",
+    ".venv",
+    ".venvs",
+    ".virtualenv",
+    ".virtualenvs",
+    ".yarn",
+    ".yarn-cache",
+    "__pycache__",
+    "env",
+    "envs",
+    "htmlcov",
+    "js/generated/coverage",
+    "node_modules",
+    "venv",
+    "venvs",
+    "virtualenv",
+    "virtualenvs",
+    "jspm_packages",
+    ".nyc_output",
+]
+
+
+class FileFinder(object):
+    def __init__(
+        self,
+        search_root: Optional[Path] = None,
+        folders_to_ignore: Optional[List[Path]] = None,
+        explicitly_listed_files: Optional[List[Path]] = None,
+        disable_search: bool = False,
+        report_type: ReportType = ReportType.COVERAGE,
+    ):
+        self.search_root = search_root or Path(os.getcwd())
+        self.folders_to_ignore = (
+            [f.as_posix() for f in folders_to_ignore] if folders_to_ignore else []
+        )
+        self.explicitly_listed_files = explicitly_listed_files or []
+        self.disable_search = disable_search
+        self.report_type: ReportType = report_type
+
+    def find_files(self) -> List[UploadCollectionResultFile]:
+        with sentry_sdk.start_span(name="find_files"):
+            if self.report_type == ReportType.COVERAGE:
+                files_excluded_patterns = coverage_files_excluded_patterns
+                files_patterns = coverage_files_patterns
+            elif self.report_type == ReportType.TEST_RESULTS:
+                files_excluded_patterns = test_results_files_excluded_patterns
+                files_patterns = test_results_files_patterns
+            regex_patterns_to_exclude = globs_to_regex(files_excluded_patterns)
+            assert regex_patterns_to_exclude  # this is never `None`
+            files_paths: Iterable[Path] = []
+            user_files_paths = []
+            if self.explicitly_listed_files:
+                user_files_paths = self.get_user_specified_files(
+                    regex_patterns_to_exclude
+                )
+            if not self.disable_search:
+                regex_patterns_to_include = globs_to_regex(files_patterns)
+                assert regex_patterns_to_include  # this is never `None`
+                files_paths = search_files(
+                    self.search_root,
+                    default_folders_to_ignore + self.folders_to_ignore,
+                    filename_include_regex=regex_patterns_to_include,
+                    filename_exclude_regex=regex_patterns_to_exclude,
+                )
+            result_files = [UploadCollectionResultFile(path) for path in files_paths]
+            user_result_files = [
+                UploadCollectionResultFile(path)
+                for path in user_files_paths
+                if user_files_paths
+            ]
+
+            user_result_files = []
+            for path in user_files_paths:
+                if os.path.isfile(path):
+                    user_result_files.append(UploadCollectionResultFile(path))
+                else:
+                    logger.warning(
+                        f'File "{path}" could not be found or does not exist. Please enter in the full path or from the search root "{self.search_root}"',
+                    )
+
+            return list(set(result_files + user_result_files))
+
+    def get_user_specified_files(self, regex_patterns_to_exclude: Pattern):
+        user_filenames_to_include = []
+        files_excluded_but_user_includes = []
+        for file in self.explicitly_listed_files:
+            user_filenames_to_include.append(file.name)
+            if regex_patterns_to_exclude.match(file.name):
+                files_excluded_but_user_includes.append(file.as_posix())
+        if files_excluded_but_user_includes:
+            logger.warning(
+                "Some files being explicitly added are found in the list of excluded files for upload. We are still going to search for the explicitly added files.",
+                extra=dict(
+                    extra_log_attributes=dict(files=files_excluded_but_user_includes)
+                ),
+            )
+        regex_patterns_to_include = globs_to_regex(user_filenames_to_include)
+        multipart_include_regex = globs_to_regex(
+            [path.resolve().as_posix() for path in self.explicitly_listed_files]
+        )
+        user_files_paths = list(
+            search_files(
+                self.search_root,
+                self.folders_to_ignore,
+                filename_include_regex=regex_patterns_to_include,
+                multipart_include_regex=multipart_include_regex,
+            )
+        )
+        not_found_files = []
+        user_files_paths_resolved = [path.resolve() for path in user_files_paths]
+        for filepath in self.explicitly_listed_files:
+            if filepath.resolve() not in user_files_paths_resolved:
+                ## The file given might be linked or in a parent dir, check to see if it exists
+                if filepath.exists():
+                    user_files_paths.append(filepath)
+                else:
+                    not_found_files.append(filepath)
+
+        if not_found_files:
+            logger.warning(
+                "Some files were not found",
+                extra=dict(extra_log_attributes=dict(not_found_files=not_found_files)),
+            )
+
+        return user_files_paths
+
+
+def select_file_finder(
+    root_folder_to_search,
+    folders_to_ignore,
+    explicitly_listed_files,
+    disable_search,
+    report_type: ReportType = ReportType.COVERAGE,
+):
+    return FileFinder(
+        root_folder_to_search,
+        folders_to_ignore,
+        explicitly_listed_files,
+        disable_search,
+        report_type,
+    )

codecov_cli/services/upload/legacy_upload_sender.py

@@ -0,0 +1,132 @@
+import logging
+import typing
+from dataclasses import dataclass
+
+import sentry_sdk
+
+from codecov_cli import __version__ as codecov_cli_version
+from codecov_cli.helpers.config import LEGACY_CODECOV_API_URL
+from codecov_cli.helpers.request import send_post_request, send_put_request
+from codecov_cli.types import UploadCollectionResult, UploadCollectionResultFile
+
+logger = logging.getLogger("codecovcli")
+
+
+@dataclass
+class UploadSendingResultWarning(object):
+    __slots__ = ("message",)
+    message: str
+
+
+@dataclass
+class UploadSendingError(object):
+    __slots__ = ("code", "params", "description")
+    code: str
+    params: typing.Dict
+    description: str
+
+
+@dataclass
+class UploadSendingResult(object):
+    __slots__ = ("error", "warnings")
+    error: typing.Optional[UploadSendingError]
+    warnings: typing.List[UploadSendingResultWarning]
+
+
+class LegacyUploadSender(object):
+    def send_upload_data(
+        self,
+        upload_data: UploadCollectionResult,
+        commit_sha: str,
+        token: str,
+        env_vars: typing.Dict[str, str],
+        name: typing.Optional[str] = None,
+        branch: typing.Optional[str] = None,
+        slug: typing.Optional[str] = None,
+        pull_request_number: typing.Optional[str] = None,
+        build_code: typing.Optional[str] = None,
+        build_url: typing.Optional[str] = None,
+        job_code: typing.Optional[str] = None,
+        flags: typing.List[str] = None,
+        ci_service: typing.Optional[str] = None,
+        enterprise_url: typing.Optional[str] = None,
+        args: dict = None,
+        **kwargs,
+    ) -> UploadSendingResult:
+        with sentry_sdk.start_span(name="upload_legacy"):
+            params = {
+                "package": f"codecov-cli/{codecov_cli_version}",
+                "commit": commit_sha,
+                "build": build_code,
+                "build_url": build_url,
+                "branch": branch,
+                "name": name,
+                "slug": slug,
+                "service": ci_service,
+                "flags": flags,
+                "pr": pull_request_number,
+                "job": job_code,
+            }
+
+            if token:
+                headers = {"X-Upload-Token": token}
+            else:
+                logger.warning("Token is empty.")
+                headers = {"X-Upload-Token": ""}
+
+            data = {
+                "cli_args": args,
+            }
+
+            upload_url = enterprise_url or LEGACY_CODECOV_API_URL
+            resp = send_post_request(
+                f"{upload_url}/upload/v4", data=data, headers=headers, params=params
+            )
+            if resp.status_code >= 400:
+                return resp
+            result_url, put_url = resp.text.split("\n")
+
+            reports_payload = self._generate_payload(upload_data, env_vars)
+            resp = send_put_request(put_url, data=reports_payload)
+            return resp
+
+    def _generate_payload(
+        self, upload_data: UploadCollectionResult, env_vars: typing.Dict[str, str]
+    ) -> bytes:
+        env_vars_section = self._generate_env_vars_section(env_vars)
+        network_section = self._generate_network_section(upload_data)
+        coverage_files_section = self._generate_coverage_files_section(upload_data)
+
+        return b"".join([env_vars_section, network_section, coverage_files_section])
+
+    def _generate_env_vars_section(self, env_vars) -> bytes:
+        filtered_env_vars = {
+            key: value for key, value in env_vars.items() if value is not None
+        }
+
+        if not filtered_env_vars:
+            return b""
+
+        env_vars_section = "".join(
+            f"{env_var}={value}\n" for env_var, value in filtered_env_vars.items()
+        )
+        return env_vars_section.encode() + b"<<<<<< ENV\n"
+
+    def _generate_network_section(self, upload_data: UploadCollectionResult) -> bytes:
+        network_files = upload_data.network
+
+        if not network_files:
+            return b""
+
+        network_files_section = "".join(file + "\n" for file in network_files)
+        return network_files_section.encode() + b"<<<<<< network\n"
+
+    def _generate_coverage_files_section(self, upload_data: UploadCollectionResult):
+        return b"".join(self._format_coverage_file(file) for file in upload_data.files)
+
+    def _format_coverage_file(self, file: UploadCollectionResultFile) -> bytes:
+        header = b"# path=" + file.get_filename().encode() + b"\n"
+        file_content = file.get_content() + b"\n"
+        file_end = b"<<<<<< EOF\n"
+
+        return header + file_content + file_end

codecov_cli/services/upload/network_finder.py

@@ -0,0 +1,49 @@
+import pathlib
+import typing
+
+from codecov_cli.helpers.versioning_systems import VersioningSystemInterface
+
+
+class NetworkFinder(object):
+    def __init__(
+        self,
+        versioning_system: VersioningSystemInterface,
+        recurse_submodules: bool,
+        network_filter: typing.Optional[str],
+        network_prefix: typing.Optional[str],
+        network_root_folder: pathlib.Path,
+    ):
+        self.versioning_system = versioning_system
+        self.recurse_submodules = recurse_submodules
+        self.network_filter = network_filter
+        self.network_prefix = network_prefix
+        self.network_root_folder = network_root_folder
+
+    def find_files(self, ignore_filters=False) -> typing.List[str]:
+        files = self.versioning_system.list_relevant_files(
+            self.network_root_folder, self.recurse_submodules
+        )
+
+        if files and not ignore_filters:
+            if self.network_filter:
+                files = [file for file in files if file.startswith(self.network_filter)]
+            if self.network_prefix:
+                files = [self.network_prefix + file for file in files]
+
+        return files
+
+
+def select_network_finder(
+    versioning_system: VersioningSystemInterface,
+    recurse_submodules: bool,
+    network_filter: typing.Optional[str],
+    network_prefix: typing.Optional[str],
+    network_root_folder: pathlib.Path,
+):
+    return NetworkFinder(
+        versioning_system,
+        recurse_submodules,
+        network_filter,
+        network_prefix,
+        network_root_folder,
+    )

codecov_cli/services/upload/upload_collector.py

@@ -0,0 +1,198 @@
+import logging
+import pathlib
+import re
+import typing
+import uuid
+from collections import namedtuple
+from fnmatch import fnmatch
+
+import click
+import sentry_sdk
+
+from codecov_cli.helpers.upload_type import ReportType
+from codecov_cli.services.upload.file_finder import FileFinder
+from codecov_cli.services.upload.network_finder import NetworkFinder
+from codecov_cli.types import (
+    PreparationPluginInterface,
+    UploadCollectionResult,
+    UploadCollectionResultFileFixer,
+)
+
+logger = logging.getLogger("codecovcli")
+
+fix_patterns_to_apply = namedtuple(
+    "fix_patterns_to_apply", ["without_reason", "with_reason", "eof"]
+)
+
+
+class UploadCollector(object):
+    def __init__(
+        self,
+        preparation_plugins: typing.List[PreparationPluginInterface],
+        network_finder: NetworkFinder,
+        file_finder: FileFinder,
+        plugin_config: dict,
+        disable_file_fixes: bool = False,
+    ):
+        self.preparation_plugins = preparation_plugins
+        self.network_finder = network_finder
+        self.file_finder = file_finder
+        self.disable_file_fixes = disable_file_fixes
+        self.plugin_config = plugin_config
+
+    def _produce_file_fixes(
+        self, files: typing.List[str]
+    ) -> typing.List[UploadCollectionResultFileFixer]:
+        if not files or self.disable_file_fixes:
+            return []
+        # patterns that we don't need to specify a reason for
+        empty_line_regex = re.compile(r"^\s*$")
+        comment_regex = re.compile(r"^\s*\/\/.*$")
+        bracket_regex = re.compile(r"^\s*[\{\}]\s*(\/\/.*)?$")
+        list_regex = re.compile(r"^\s*[\]\[]\s*(\/\/.*)?$")
+        parenthesis_regex = re.compile(r"^\s*[\(\)]\s*(\/\/.*)?$")
+        go_function_regex = re.compile(r"^\s*func\s*[\{]\s*(\/\/.*)?$")
+        php_end_bracket_regex = re.compile(r"^\s*\);\s*(\/\/.*)?$")
+
+        # patterns to specify a reason for
+        comment_block_regex = re.compile(r"^\s*(\/\*|\*\/)\s*$")
+        lcov_excel_regex = re.compile(r"\/\/ LCOV_EXCL")
+
+        kt_patterns_to_apply = fix_patterns_to_apply(
+            [bracket_regex, parenthesis_regex], [comment_block_regex], True
+        )
+        go_patterns_to_apply = fix_patterns_to_apply(
+            [empty_line_regex, comment_regex, bracket_regex, go_function_regex],
+            [comment_block_regex],
+            False,
+        )
+        dart_patterns_to_apply = fix_patterns_to_apply(
+            [bracket_regex],
+            [],
+            False,
+        )
+        php_patterns_to_apply = fix_patterns_to_apply(
+            [bracket_regex, list_regex, php_end_bracket_regex],
+            [],
+            False,
+        )
+        cpp_swift_vala_patterns_to_apply = fix_patterns_to_apply(
+            [empty_line_regex, bracket_regex],
+            [lcov_excel_regex],
+            False,
+        )
+
+        file_regex_patterns = {
+            "*.kt": kt_patterns_to_apply,
+            "*.go": go_patterns_to_apply,
+            "*.dart": dart_patterns_to_apply,
+            "*.php": php_patterns_to_apply,
+            "*.c": cpp_swift_vala_patterns_to_apply,
+            "*.cpp": cpp_swift_vala_patterns_to_apply,
+            "*.cxx": cpp_swift_vala_patterns_to_apply,
+            "*.h": cpp_swift_vala_patterns_to_apply,
+            "*.hpp": cpp_swift_vala_patterns_to_apply,
+            "*.m": cpp_swift_vala_patterns_to_apply,
+            "*.swift": cpp_swift_vala_patterns_to_apply,
+            "*.vala": cpp_swift_vala_patterns_to_apply,
+        }
+
+        result = []
+        for filename in files:
+            for glob, fix_patterns in file_regex_patterns.items():
+                if fnmatch(filename, glob):
+                    result.append(self._get_file_fixes(filename, fix_patterns))
+                    break
+
+        return result
+
+    def _get_file_fixes(
+        self, filename: str, fix_patterns_to_apply: fix_patterns_to_apply
+    ) -> UploadCollectionResultFileFixer:
+        path = pathlib.Path(filename)
+        fixed_lines_without_reason = set()
+        fixed_lines_with_reason = set()
+        eof = None
+
+        try:
+            with open(filename, "r", encoding="utf-8") as f:
+                # If lineno is unset that means that the
+                # file is empty thus the eof should be 0
+                # so lineno will be set to -1 here
+                lineno = -1
+                # overwrite lineno in this for loop
+                # if f is empty, lineno stays at -1
+                for lineno, line_content in enumerate(f):
+                    if any(
+                        pattern.match(line_content)
+                        for pattern in fix_patterns_to_apply.with_reason
+                    ):
+                        fixed_lines_with_reason.add((lineno + 1, line_content))
+                    elif any(
+                        pattern.match(line_content)
+                        for pattern in fix_patterns_to_apply.without_reason
+                    ):
+                        fixed_lines_without_reason.add(lineno + 1)
+                if fix_patterns_to_apply.eof:
+                    eof = lineno + 1
+        except UnicodeDecodeError as err:
+            logger.warning(
+                f"There was an issue decoding: {filename}, file fixes were not applied to this file.",
+                extra=dict(
+                    encoding=err.encoding,
+                    reason=err.reason,
+                ),
+            )
+        except IsADirectoryError:
+            logger.info(f"Skipping {filename}, found a directory not a file")
+
+        return UploadCollectionResultFileFixer(
+            path, fixed_lines_without_reason, fixed_lines_with_reason, eof
+        )
+
+    def generate_upload_data(
+        self, report_type: ReportType = ReportType.COVERAGE
+    ) -> UploadCollectionResult:
+        with sentry_sdk.start_span(name="upload_collector"):
+            for prep in self.preparation_plugins:
+                logger.debug(f"Running preparation plugin: {type(prep)}")
+                prep.run_preparation(self)
+            logger.debug("Collecting relevant files")
+            with sentry_sdk.start_span(name="file_collector"):
+                network = self.network_finder.find_files()
+                unfiltered_network = self.network_finder.find_files(True)
+                report_files = self.file_finder.find_files()
+            logger.info(
+                f"Found {len(report_files)} {report_type.value} files to report"
+            )
+            logger.debug(
+                f"Found {len(network)} network files to report, ({len(unfiltered_network)} without filtering)"
+            )
+            if not report_files:
+                if report_type == ReportType.TEST_RESULTS:
+                    error_message = "No JUnit XML reports found. Please review our documentation (https://docs.codecov.com/docs/test-result-ingestion-beta) to generate and upload the file."
+                    logger.error(error_message)
+                    return UploadCollectionResult(
+                        network=network,
+                        files=[],
+                        file_fixes=[],
+                    )
+                else:
+                    error_message = "No coverage reports found. Please make sure you're generating reports successfully."
+                raise click.ClickException(
+                    click.style(
+                        error_message,
+                        fg="red",
+                    )
+                )
+            for file in report_files:
+                logger.info(f"> {file}")
+            return UploadCollectionResult(
+                network=network,
+                files=report_files,
+                file_fixes=(
+                    self._produce_file_fixes(unfiltered_network)
+                    if report_type == ReportType.COVERAGE
+                    else []
+                ),
+            )