megalinter 10.2.0.post0.dev20261114094955__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megalinter/Linter.py +2587 -0
- megalinter/MegaLinter.py +1659 -0
- megalinter/Reporter.py +45 -0
- megalinter/__about__.py +1 -0
- megalinter/__init__.py +21 -0
- megalinter/_core-requirements.lock +459 -0
- megalinter/_templates/.ansible-lint +5 -0
- megalinter/_templates/.arm-ttk.psd1 +32 -0
- megalinter/_templates/.bandit.yml +302 -0
- megalinter/_templates/.cfnlintrc.yml +2 -0
- megalinter/_templates/.checkov.yml +6 -0
- megalinter/_templates/.chktexrc +802 -0
- megalinter/_templates/.clj-kondo/config.edn +2 -0
- megalinter/_templates/.coffee-lint.json +135 -0
- megalinter/_templates/.devskim.json +3 -0
- megalinter/_templates/.flake8 +3 -0
- megalinter/_templates/.gherkin-lintrc +25 -0
- megalinter/_templates/.golangci.yml +29 -0
- megalinter/_templates/.groovylintrc.json +65 -0
- megalinter/_templates/.grype.yaml +151 -0
- megalinter/_templates/.hadolint.yaml +6 -0
- megalinter/_templates/.htmlhintrc +25 -0
- megalinter/_templates/.isort.cfg +8 -0
- megalinter/_templates/.jscpd.json +28 -0
- megalinter/_templates/.luacheckrc +1 -0
- megalinter/_templates/.markdown-link-check.json +5 -0
- megalinter/_templates/.markdownlint.json +16 -0
- megalinter/_templates/.mypy.ini +4 -0
- megalinter/_templates/.npmpackagejsonlintrc.json +3 -0
- megalinter/_templates/.perlcriticrc +2 -0
- megalinter/_templates/.php-cs-fixer.dist.php +17 -0
- megalinter/_templates/.phplint.yml +6 -0
- megalinter/_templates/.protolintrc.yml +7 -0
- megalinter/_templates/.pylintrc +470 -0
- megalinter/_templates/.ruby-lint.yml +8 -0
- megalinter/_templates/.ruff.toml +7 -0
- megalinter/_templates/.scalafix.conf +14 -0
- megalinter/_templates/.secretlintrc.json +7 -0
- megalinter/_templates/.snakefmt.toml +0 -0
- megalinter/_templates/.spectral.yaml +11 -0
- megalinter/_templates/.sql-config.json +3 -0
- megalinter/_templates/.sqlfluff +16 -0
- megalinter/_templates/.stylelintrc.json +3 -0
- megalinter/_templates/.tflint.hcl +33 -0
- megalinter/_templates/.trufflehogignore +1 -0
- megalinter/_templates/.yamllint.yml +19 -0
- megalinter/_templates/README.md +6 -0
- megalinter/_templates/analysis_options.yml +57 -0
- megalinter/_templates/code-analyzer-flow.yml +22 -0
- megalinter/_templates/code-analyzer.yml +75 -0
- megalinter/_templates/java-pmd-ruleset.xml +21 -0
- megalinter/_templates/mega-linter.yml +195 -0
- megalinter/_templates/native/.ruby-lint.yml +10 -0
- megalinter/_templates/native/phpstan.neon.dist +4 -0
- megalinter/_templates/phpcs.xml +6 -0
- megalinter/_templates/phpstan.neon.dist +6 -0
- megalinter/_templates/psalm.xml +9 -0
- megalinter/_templates/pyproject.toml +2 -0
- megalinter/_templates/sun_checks.xml +197 -0
- megalinter/alpaca.py +36 -0
- megalinter/api_providers/ApiProvider.py +136 -0
- megalinter/api_providers/ApiProviderDatadog.py +163 -0
- megalinter/api_providers/ApiProviderElastic.py +88 -0
- megalinter/api_providers/ApiProviderGrafana.py +222 -0
- megalinter/api_providers/ApiProviderNewRelic.py +154 -0
- megalinter/api_providers/__init__.py +25 -0
- megalinter/ci_providers/CiProvider.py +94 -0
- megalinter/ci_providers/CiProviderAzurePipelines.py +182 -0
- megalinter/ci_providers/CiProviderBitbucket.py +61 -0
- megalinter/ci_providers/CiProviderGithubActions.py +121 -0
- megalinter/ci_providers/CiProviderGitlab.py +109 -0
- megalinter/ci_providers/CiProviderJenkins.py +313 -0
- megalinter/ci_providers/__init__.py +54 -0
- megalinter/cli.py +279 -0
- megalinter/config.py +476 -0
- megalinter/constants.py +138 -0
- megalinter/descriptors/action.megalinter-descriptor.yml +181 -0
- megalinter/descriptors/additional/betterleaks-default.toml +1851 -0
- megalinter/descriptors/all_flavors.json +1190 -0
- megalinter/descriptors/ansible.megalinter-descriptor.yml +127 -0
- megalinter/descriptors/api.megalinter-descriptor.yml +132 -0
- megalinter/descriptors/arm.megalinter-descriptor.yml +117 -0
- megalinter/descriptors/bash.megalinter-descriptor.yml +316 -0
- megalinter/descriptors/bicep.megalinter-descriptor.yml +122 -0
- megalinter/descriptors/c.megalinter-descriptor.yml +80 -0
- megalinter/descriptors/clojure.megalinter-descriptor.yml +237 -0
- megalinter/descriptors/cloudformation.megalinter-descriptor.yml +127 -0
- megalinter/descriptors/coffee.megalinter-descriptor.yml +73 -0
- megalinter/descriptors/copypaste.megalinter-descriptor.yml +88 -0
- megalinter/descriptors/cpp.megalinter-descriptor.yml +69 -0
- megalinter/descriptors/csharp.megalinter-descriptor.yml +235 -0
- megalinter/descriptors/css.megalinter-descriptor.yml +163 -0
- megalinter/descriptors/dart.megalinter-descriptor.yml +161 -0
- megalinter/descriptors/dockerfile.megalinter-descriptor.yml +131 -0
- megalinter/descriptors/editorconfig.megalinter-descriptor.yml +83 -0
- megalinter/descriptors/env.megalinter-descriptor.yml +100 -0
- megalinter/descriptors/gherkin.megalinter-descriptor.yml +78 -0
- megalinter/descriptors/go.megalinter-descriptor.yml +290 -0
- megalinter/descriptors/graphql.megalinter-descriptor.yml +91 -0
- megalinter/descriptors/groovy.megalinter-descriptor.yml +108 -0
- megalinter/descriptors/html.megalinter-descriptor.yml +155 -0
- megalinter/descriptors/java.megalinter-descriptor.yml +227 -0
- megalinter/descriptors/javascript.megalinter-descriptor.yml +383 -0
- megalinter/descriptors/json.megalinter-descriptor.yml +222 -0
- megalinter/descriptors/jsx.megalinter-descriptor.yml +149 -0
- megalinter/descriptors/kotlin.megalinter-descriptor.yml +219 -0
- megalinter/descriptors/kubernetes.megalinter-descriptor.yml +331 -0
- megalinter/descriptors/latex.megalinter-descriptor.yml +68 -0
- megalinter/descriptors/lua.megalinter-descriptor.yml +231 -0
- megalinter/descriptors/markdown.megalinter-descriptor.yml +241 -0
- megalinter/descriptors/native-locks/ACTION_ACTIONLINT.json +79 -0
- megalinter/descriptors/native-locks/ACTION_ZIZMOR.json +10 -0
- megalinter/descriptors/native-locks/ANSIBLE_ANSIBLE_LINT.json +11 -0
- megalinter/descriptors/native-locks/API_SPECTRAL.json +3250 -0
- megalinter/descriptors/native-locks/ARM_ARM_TTK.json +79 -0
- megalinter/descriptors/native-locks/BASH_EXEC.json +7 -0
- megalinter/descriptors/native-locks/BASH_SHELLCHECK.json +84 -0
- megalinter/descriptors/native-locks/BASH_SHFMT.json +84 -0
- megalinter/descriptors/native-locks/BICEP_BICEP_LINTER.json +84 -0
- megalinter/descriptors/native-locks/CLOJURE_CLJSTYLE.json +51 -0
- megalinter/descriptors/native-locks/CLOJURE_CLJ_KONDO.json +79 -0
- megalinter/descriptors/native-locks/CLOUDFORMATION_CFN_LINT.json +10 -0
- megalinter/descriptors/native-locks/COPYPASTE_JSCPD.json +174 -0
- megalinter/descriptors/native-locks/CPP_CLANG_FORMAT.json +10 -0
- megalinter/descriptors/native-locks/CPP_CPPCHECK.json +8 -0
- megalinter/descriptors/native-locks/CPP_CPPLINT.json +10 -0
- megalinter/descriptors/native-locks/CSHARP_CSHARPIER.json +84 -0
- megalinter/descriptors/native-locks/CSHARP_DOTNET_FORMAT.json +7 -0
- megalinter/descriptors/native-locks/CSHARP_ROSLYNATOR.json +84 -0
- megalinter/descriptors/native-locks/CSS_BIOME.json +233 -0
- megalinter/descriptors/native-locks/CSS_STYLELINT.json +1630 -0
- megalinter/descriptors/native-locks/C_CLANG_FORMAT.json +10 -0
- megalinter/descriptors/native-locks/C_CPPCHECK.json +8 -0
- megalinter/descriptors/native-locks/C_CPPLINT.json +10 -0
- megalinter/descriptors/native-locks/DART_DARTANALYZER.json +7 -0
- megalinter/descriptors/native-locks/DOCKERFILE_HADOLINT.json +84 -0
- megalinter/descriptors/native-locks/EDITORCONFIG_EDITORCONFIG_CHECKER.json +79 -0
- megalinter/descriptors/native-locks/ENV_DOTENV_LINTER.json +79 -0
- megalinter/descriptors/native-locks/GHERKIN_GHERKIN_LINT.json +405 -0
- megalinter/descriptors/native-locks/GO_GOLANGCI_LINT.json +85 -0
- megalinter/descriptors/native-locks/GO_REVIVE.json +79 -0
- megalinter/descriptors/native-locks/GRAPHQL_BIOME.json +233 -0
- megalinter/descriptors/native-locks/GROOVY_NPM_GROOVY_LINT.json +302 -0
- megalinter/descriptors/native-locks/HTML_DJLINT.json +10 -0
- megalinter/descriptors/native-locks/HTML_HTMLHINT.json +330 -0
- megalinter/descriptors/native-locks/JAVASCRIPT_BIOME.json +233 -0
- megalinter/descriptors/native-locks/JAVASCRIPT_ES.json +2899 -0
- megalinter/descriptors/native-locks/JAVASCRIPT_PRETTIER.json +48 -0
- megalinter/descriptors/native-locks/JAVASCRIPT_STANDARD.json +3741 -0
- megalinter/descriptors/native-locks/JAVA_CHECKSTYLE.json +85 -0
- megalinter/descriptors/native-locks/JAVA_PMD.json +84 -0
- megalinter/descriptors/native-locks/JSON_BIOME.json +233 -0
- megalinter/descriptors/native-locks/JSON_JSONLINT.json +477 -0
- megalinter/descriptors/native-locks/JSON_NPM_PACKAGE_JSON_LINT.json +1282 -0
- megalinter/descriptors/native-locks/JSON_PRETTIER.json +48 -0
- megalinter/descriptors/native-locks/JSON_V8R.json +1306 -0
- megalinter/descriptors/native-locks/JSX_BIOME.json +233 -0
- megalinter/descriptors/native-locks/JSX_ESLINT.json +2320 -0
- megalinter/descriptors/native-locks/KOTLIN_DETEKT.json +84 -0
- megalinter/descriptors/native-locks/KOTLIN_KTLINT.json +84 -0
- megalinter/descriptors/native-locks/KUBERNETES_HELM.json +85 -0
- megalinter/descriptors/native-locks/KUBERNETES_KUBECONFORM.json +80 -0
- megalinter/descriptors/native-locks/KUBERNETES_KUBESCAPE.json +79 -0
- megalinter/descriptors/native-locks/LATEX_CHKTEX.json +7 -0
- megalinter/descriptors/native-locks/LUA_LUACHECK.json +39 -0
- megalinter/descriptors/native-locks/LUA_STYLUA.json +79 -0
- megalinter/descriptors/native-locks/MARKDOWN_MARKDOWNLINT.json +1138 -0
- megalinter/descriptors/native-locks/MARKDOWN_MARKDOWN_TABLE_FORMATTER.json +45 -0
- megalinter/descriptors/native-locks/MARKDOWN_RUMDL.json +10 -0
- megalinter/descriptors/native-locks/PERL_PERLCRITIC.json +7 -0
- megalinter/descriptors/native-locks/PHP_PHPCS.json +84 -0
- megalinter/descriptors/native-locks/PHP_PHPCSFIXER.json +84 -0
- megalinter/descriptors/native-locks/PHP_PHPLINT.json +84 -0
- megalinter/descriptors/native-locks/PHP_PHPSTAN.json +84 -0
- megalinter/descriptors/native-locks/PHP_PSALM.json +85 -0
- megalinter/descriptors/native-locks/POWERSHELL_POWERSHELL.json +79 -0
- megalinter/descriptors/native-locks/POWERSHELL_POWERSHELL_FORMATTER.json +79 -0
- megalinter/descriptors/native-locks/PROTOBUF_PROTOLINT.json +79 -0
- megalinter/descriptors/native-locks/PYTHON_BANDIT.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_BLACK.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_FLAKE8.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_ISORT.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_MYPY.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_NBQA_MYPY.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_PYLINT.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_PYRIGHT.json +63 -0
- megalinter/descriptors/native-locks/PYTHON_RUFF.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_RUFF_FORMAT.json +10 -0
- megalinter/descriptors/native-locks/PYTHON_TY.json +10 -0
- megalinter/descriptors/native-locks/RAKU_RAKU.json +8 -0
- megalinter/descriptors/native-locks/REPOSITORY_BETTERLEAKS.json +79 -0
- megalinter/descriptors/native-locks/REPOSITORY_CHECKOV.json +10 -0
- megalinter/descriptors/native-locks/REPOSITORY_DEVSKIM.json +84 -0
- megalinter/descriptors/native-locks/REPOSITORY_DUSTILOCK.json +15 -0
- megalinter/descriptors/native-locks/REPOSITORY_GIT_DIFF.json +7 -0
- megalinter/descriptors/native-locks/REPOSITORY_GRYPE.json +79 -0
- megalinter/descriptors/native-locks/REPOSITORY_KINGFISHER.json +79 -0
- megalinter/descriptors/native-locks/REPOSITORY_LS_LINT.json +84 -0
- megalinter/descriptors/native-locks/REPOSITORY_OSV_SCANNER.json +85 -0
- megalinter/descriptors/native-locks/REPOSITORY_SECRETLINT.json +1060 -0
- megalinter/descriptors/native-locks/REPOSITORY_SEMGREP.json +10 -0
- megalinter/descriptors/native-locks/REPOSITORY_SYFT.json +79 -0
- megalinter/descriptors/native-locks/REPOSITORY_TRIVY.json +84 -0
- megalinter/descriptors/native-locks/REPOSITORY_TRIVY_SBOM.json +84 -0
- megalinter/descriptors/native-locks/REPOSITORY_TRUFFLEHOG.json +84 -0
- megalinter/descriptors/native-locks/ROBOTFRAMEWORK_ROBOCOP.json +10 -0
- megalinter/descriptors/native-locks/RST_RSTCHECK.json +10 -0
- megalinter/descriptors/native-locks/RST_RSTFMT.json +10 -0
- megalinter/descriptors/native-locks/RST_RST_LINT.json +10 -0
- megalinter/descriptors/native-locks/RUBY_RUBOCOP.json +12 -0
- megalinter/descriptors/native-locks/RUST_CLIPPY.json +7 -0
- megalinter/descriptors/native-locks/R_LINTR.json +8 -0
- megalinter/descriptors/native-locks/SALESFORCE_CODE_ANALYZER_APEX.json +27048 -0
- megalinter/descriptors/native-locks/SALESFORCE_CODE_ANALYZER_APEXGURU.json +27048 -0
- megalinter/descriptors/native-locks/SALESFORCE_CODE_ANALYZER_AURA.json +27048 -0
- megalinter/descriptors/native-locks/SALESFORCE_CODE_ANALYZER_FLOW.json +27048 -0
- megalinter/descriptors/native-locks/SALESFORCE_CODE_ANALYZER_LWC.json +27048 -0
- megalinter/descriptors/native-locks/SCALA_SCALAFIX.json +8 -0
- megalinter/descriptors/native-locks/SNAKEMAKE_LINT.json +10 -0
- megalinter/descriptors/native-locks/SNAKEMAKE_SNAKEFMT.json +10 -0
- megalinter/descriptors/native-locks/SPELL_CODESPELL.json +10 -0
- megalinter/descriptors/native-locks/SPELL_CSPELL.json +1070 -0
- megalinter/descriptors/native-locks/SPELL_LYCHEE.json +84 -0
- megalinter/descriptors/native-locks/SPELL_PROSELINT.json +10 -0
- megalinter/descriptors/native-locks/SPELL_VALE.json +79 -0
- megalinter/descriptors/native-locks/SQL_SQLFLUFF.json +10 -0
- megalinter/descriptors/native-locks/SWIFT_SWIFTLINT.json +8 -0
- megalinter/descriptors/native-locks/TEKTON_TEKTON_LINT.json +1063 -0
- megalinter/descriptors/native-locks/TERRAFORM_TERRAFORM_FMT.json +84 -0
- megalinter/descriptors/native-locks/TERRAFORM_TERRAGRUNT.json +84 -0
- megalinter/descriptors/native-locks/TERRAFORM_TFLINT.json +80 -0
- megalinter/descriptors/native-locks/TERRAFORM_TOFU_FMT.json +79 -0
- megalinter/descriptors/native-locks/TERRAFORM_TOFU_VALIDATE.json +79 -0
- megalinter/descriptors/native-locks/TSX_BIOME.json +233 -0
- megalinter/descriptors/native-locks/TSX_ESLINT.json +2848 -0
- megalinter/descriptors/native-locks/TYPESCRIPT_BIOME.json +248 -0
- megalinter/descriptors/native-locks/TYPESCRIPT_ES.json +3455 -0
- megalinter/descriptors/native-locks/TYPESCRIPT_PRETTIER.json +63 -0
- megalinter/descriptors/native-locks/TYPESCRIPT_STANDARD.json +4286 -0
- megalinter/descriptors/native-locks/VBDOTNET_DOTNET_FORMAT.json +7 -0
- megalinter/descriptors/native-locks/XML_XMLLINT.json +7 -0
- megalinter/descriptors/native-locks/YAML_PRETTIER.json +48 -0
- megalinter/descriptors/native-locks/YAML_V8R.json +1306 -0
- megalinter/descriptors/native-locks/YAML_YAMLLINT.json +10 -0
- megalinter/descriptors/native-locks/runtimes/dart.json +83 -0
- megalinter/descriptors/native-locks/runtimes/dotnet.json +78 -0
- megalinter/descriptors/native-locks/runtimes/go.json +83 -0
- megalinter/descriptors/native-locks/runtimes/java.json +83 -0
- megalinter/descriptors/native-locks/runtimes/node.json +88 -0
- megalinter/descriptors/native-locks/runtimes/perl.json +22 -0
- megalinter/descriptors/native-locks/runtimes/php.json +22 -0
- megalinter/descriptors/native-locks/runtimes/powershell.json +78 -0
- megalinter/descriptors/native-runtimes.yml +372 -0
- megalinter/descriptors/perl.megalinter-descriptor.yml +110 -0
- megalinter/descriptors/php.megalinter-descriptor.yml +574 -0
- megalinter/descriptors/powershell.megalinter-descriptor.yml +175 -0
- megalinter/descriptors/protobuf.megalinter-descriptor.yml +99 -0
- megalinter/descriptors/python.megalinter-descriptor.yml +1212 -0
- megalinter/descriptors/r.megalinter-descriptor.yml +95 -0
- megalinter/descriptors/raku.megalinter-descriptor.yml +81 -0
- megalinter/descriptors/repository.megalinter-descriptor.yml +1720 -0
- megalinter/descriptors/robotframework.megalinter-descriptor.yml +101 -0
- megalinter/descriptors/rst.megalinter-descriptor.yml +201 -0
- megalinter/descriptors/ruby.megalinter-descriptor.yml +133 -0
- megalinter/descriptors/rust.megalinter-descriptor.yml +65 -0
- megalinter/descriptors/salesforce.megalinter-descriptor.yml +675 -0
- megalinter/descriptors/scala.megalinter-descriptor.yml +116 -0
- megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +43473 -0
- megalinter/descriptors/schemas/megalinter-custom-flavor.jsonschema.json +194 -0
- megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json +2160 -0
- megalinter/descriptors/schemas/megalinter-linter.jsonschema.json +7 -0
- megalinter/descriptors/schemas/megalinter-native-runtimes.jsonschema.json +356 -0
- megalinter/descriptors/shared/biome.megalinter-linter.yml +83 -0
- megalinter/descriptors/shared/clang-format.megalinter-linter.yml +57 -0
- megalinter/descriptors/shared/cppcheck.megalinter-linter.yml +49 -0
- megalinter/descriptors/shared/cpplint.megalinter-linter.yml +39 -0
- megalinter/descriptors/shared/dotnet-format.megalinter-linter.yml +46 -0
- megalinter/descriptors/shared/eslint.megalinter-linter.yml +80 -0
- megalinter/descriptors/shared/prettier.megalinter-linter.yml +78 -0
- megalinter/descriptors/shared/v8r.megalinter-linter.yml +64 -0
- megalinter/descriptors/snakemake.megalinter-descriptor.yml +145 -0
- megalinter/descriptors/spell.megalinter-descriptor.yml +506 -0
- megalinter/descriptors/sql.megalinter-descriptor.yml +114 -0
- megalinter/descriptors/swift.megalinter-descriptor.yml +133 -0
- megalinter/descriptors/tekton.megalinter-descriptor.yml +75 -0
- megalinter/descriptors/terraform.megalinter-descriptor.yml +595 -0
- megalinter/descriptors/tsx.megalinter-descriptor.yml +194 -0
- megalinter/descriptors/typescript.megalinter-descriptor.yml +375 -0
- megalinter/descriptors/vbdotnet.megalinter-descriptor.yml +41 -0
- megalinter/descriptors/xml.megalinter-descriptor.yml +72 -0
- megalinter/descriptors/yaml.megalinter-descriptor.yml +157 -0
- megalinter/flavor_factory.py +195 -0
- megalinter/linter_factory.py +219 -0
- megalinter/linters/ArmLinter.py +80 -0
- megalinter/linters/BashBashExecLinter.py +18 -0
- megalinter/linters/BetterleaksLinter.py +164 -0
- megalinter/linters/BicepLinter.py +16 -0
- megalinter/linters/BiomeLinter.py +76 -0
- megalinter/linters/CSpellLinter.py +155 -0
- megalinter/linters/CheckovLinter.py +162 -0
- megalinter/linters/ClippyLinter.py +37 -0
- megalinter/linters/CljKondoLinter.py +41 -0
- megalinter/linters/CppLintLinter.py +26 -0
- megalinter/linters/DetektLinter.py +14 -0
- megalinter/linters/DevskimLinter.py +14 -0
- megalinter/linters/DotnetFormatLinter.py +21 -0
- megalinter/linters/DustilockLinter.py +105 -0
- megalinter/linters/EslintLinter.py +161 -0
- megalinter/linters/GherkinLintLinter.py +38 -0
- megalinter/linters/GrypeLinter.py +22 -0
- megalinter/linters/JavaPmdLinter.py +60 -0
- megalinter/linters/JavaScriptStandardLinter.py +15 -0
- megalinter/linters/JsCpdLinter.py +99 -0
- megalinter/linters/KingfisherLinter.py +14 -0
- megalinter/linters/KubeconformLinter.py +18 -0
- megalinter/linters/KubescapeLinter.py +16 -0
- megalinter/linters/LsLintLinter.py +39 -0
- megalinter/linters/LuacheckLinter.py +32 -0
- megalinter/linters/LycheeLinter.py +70 -0
- megalinter/linters/OsvScannerLinter.py +32 -0
- megalinter/linters/PhpCsFixerLinter.py +47 -0
- megalinter/linters/PhpstanLinter.py +45 -0
- megalinter/linters/PowershellLinter.py +126 -0
- megalinter/linters/ProselintLinter.py +14 -0
- megalinter/linters/ProtolintLinter.py +17 -0
- megalinter/linters/PyrightLinter.py +30 -0
- megalinter/linters/RLinter.py +99 -0
- megalinter/linters/RakuLinter.py +51 -0
- megalinter/linters/ReviveLinter.py +14 -0
- megalinter/linters/RoslynatorLinter.py +37 -0
- megalinter/linters/RubocopLinter.py +46 -0
- megalinter/linters/RuffFormatLinter.py +12 -0
- megalinter/linters/SalesforceCodeAnalyzerApexGuruLinter.py +86 -0
- megalinter/linters/SalesforceCodeAnalyzerLinter.py +28 -0
- megalinter/linters/SecretLintLinter.py +61 -0
- megalinter/linters/SemgrepLinter.py +97 -0
- megalinter/linters/ShellcheckLinter.py +36 -0
- megalinter/linters/SqlFluffLinter.py +29 -0
- megalinter/linters/StyLuaLinter.py +42 -0
- megalinter/linters/StyleLintLinter.py +49 -0
- megalinter/linters/SwiftLintLinter.py +43 -0
- megalinter/linters/SyftLinter.py +89 -0
- megalinter/linters/TfLintLinter.py +70 -0
- megalinter/linters/TofuValidateLinter.py +65 -0
- megalinter/linters/TrivyLinter.py +50 -0
- megalinter/linters/TrivySbomLinter.py +83 -0
- megalinter/linters/TruffleHogLinter.py +91 -0
- megalinter/linters/TypeScriptStandardLinter.py +15 -0
- megalinter/linters/V8rLinter.py +70 -0
- megalinter/linters/ValeLinter.py +12 -0
- megalinter/linters/XmlLintLinter.py +35 -0
- megalinter/linters/YamllintLinter.py +93 -0
- megalinter/linters/ZizmorLinter.py +37 -0
- megalinter/linters/__init__.py +0 -0
- megalinter/llm_advisor.py +235 -0
- megalinter/llm_provider/README.md +149 -0
- megalinter/llm_provider/__init__.py +37 -0
- megalinter/llm_provider/llm_provider.py +282 -0
- megalinter/llm_provider/llm_provider_anthropic.py +131 -0
- megalinter/llm_provider/llm_provider_deepseek.py +45 -0
- megalinter/llm_provider/llm_provider_factory.py +94 -0
- megalinter/llm_provider/llm_provider_google.py +117 -0
- megalinter/llm_provider/llm_provider_grok.py +41 -0
- megalinter/llm_provider/llm_provider_huggingface.py +99 -0
- megalinter/llm_provider/llm_provider_mistral.py +45 -0
- megalinter/llm_provider/llm_provider_ollama.py +120 -0
- megalinter/llm_provider/llm_provider_openai.py +54 -0
- megalinter/llm_provider/llm_provider_openai_compatible.py +140 -0
- megalinter/llm_provider/llm_provider_orcarouter.py +43 -0
- megalinter/logger.py +231 -0
- megalinter/native/__init__.py +0 -0
- megalinter/native/catalog.py +29 -0
- megalinter/native/context.py +135 -0
- megalinter/native/download.py +86 -0
- megalinter/native/extract.py +108 -0
- megalinter/native/installer.py +214 -0
- megalinter/native/lock.py +72 -0
- megalinter/native/methods.py +431 -0
- megalinter/native/resolve.py +147 -0
- megalinter/native/runtimes.py +272 -0
- megalinter/native/selection.py +56 -0
- megalinter/native/spec.py +123 -0
- megalinter/native/state.py +61 -0
- megalinter/native/toolchain.py +158 -0
- megalinter/plugin_factory.py +136 -0
- megalinter/pre_post_factory.py +288 -0
- megalinter/prerun_report.py +266 -0
- megalinter/removed_linters.py +233 -0
- megalinter/reporters/ApiReporter.py +260 -0
- megalinter/reporters/AzureCommentReporter.py +203 -0
- megalinter/reporters/BitbucketCommentReporter.py +177 -0
- megalinter/reporters/ConfigReporter.py +168 -0
- megalinter/reporters/ConsoleLinterReporter.py +195 -0
- megalinter/reporters/ConsoleReporter.py +181 -0
- megalinter/reporters/EmailReporter.py +150 -0
- megalinter/reporters/FileIoReporter.py +86 -0
- megalinter/reporters/GithubCommentReporter.py +163 -0
- megalinter/reporters/GithubStatusReporter.py +107 -0
- megalinter/reporters/GitlabCommentReporter.py +223 -0
- megalinter/reporters/JsonReporter.py +127 -0
- megalinter/reporters/MarkdownSummaryReporter.py +54 -0
- megalinter/reporters/RedisLinterReporter.py +78 -0
- megalinter/reporters/RedisReporter.py +78 -0
- megalinter/reporters/SarifReporter.py +140 -0
- megalinter/reporters/TapReporter.py +89 -0
- megalinter/reporters/TextReporter.py +132 -0
- megalinter/reporters/UpdatedSourcesReporter.py +129 -0
- megalinter/reporters/WebHookLinterReporter.py +52 -0
- megalinter/reporters/WebHookReporter.py +54 -0
- megalinter/reporters/__init__.py +0 -0
- megalinter/run.py +45 -0
- megalinter/utils.py +1265 -0
- megalinter/utils_reporter.py +950 -0
- megalinter/utils_sarif.py +253 -0
- megalinter/utilstest.py +865 -0
- megalinter-10.2.0.post0.dev20261114094955.dist-info/METADATA +2499 -0
- megalinter-10.2.0.post0.dev20261114094955.dist-info/RECORD +420 -0
- megalinter-10.2.0.post0.dev20261114094955.dist-info/WHEEL +4 -0
- megalinter-10.2.0.post0.dev20261114094955.dist-info/entry_points.txt +2 -0
- megalinter-10.2.0.post0.dev20261114094955.dist-info/licenses/LICENSE +661 -0
megalinter/Linter.py
ADDED
|
@@ -0,0 +1,2587 @@
|
|
|
1
|
+
#!/usr/bin/env
|
|
2
|
+
# flake8: noqa: E203
|
|
3
|
+
"""
|
|
4
|
+
Template class for custom linters: any linter class in /linters folder must inherit from this class
|
|
5
|
+
The following list of items can/must be overridden on custom linter local class:
|
|
6
|
+
- field descriptor_id (required) ex: "JAVASCRIPT"
|
|
7
|
+
- field name (optional) ex: "JAVASCRIPT_ES". If not set, language value is used
|
|
8
|
+
- field linter_name (required) ex: "eslint"
|
|
9
|
+
- field linter_url (required) ex: "https://eslint.org/"
|
|
10
|
+
- field test_folder (optional) ex: "docker". If not set, language.lowercase() value is used
|
|
11
|
+
- field config_file_name (optional) ex: ".eslintrc.yml". If not set, no default config file will be searched
|
|
12
|
+
- field file_extensions (optional) ex: [".js"]. At least file_extension or file_names_regex must be set
|
|
13
|
+
- field file_names_regex (optional) ex: ["Dockerfile(-.+)?"]. At least file_extension or file_names_regex must be set
|
|
14
|
+
- method build_lint_command (optional) : Return CLI command to lint a file with the related linter
|
|
15
|
+
Default: linter_name + (if config_file(-c + config_file)) + config_file
|
|
16
|
+
- method build_version_command (optional): Returns CLI command to get the related linter version.
|
|
17
|
+
Default: linter_name --version
|
|
18
|
+
- method build_extract_version_regex (optional): Returns RegEx to extract version from version command output
|
|
19
|
+
Default: "\\d+(\\.\\d+)+"
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import errno
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import shutil
|
|
29
|
+
import signal
|
|
30
|
+
import subprocess
|
|
31
|
+
import sys
|
|
32
|
+
import urllib.error
|
|
33
|
+
import urllib.request
|
|
34
|
+
from time import perf_counter
|
|
35
|
+
|
|
36
|
+
import yaml
|
|
37
|
+
from megalinter import config, pre_post_factory, utils, utils_reporter, utils_sarif
|
|
38
|
+
from megalinter.constants import DEFAULT_DOCKER_WORKSPACE_DIR
|
|
39
|
+
from megalinter.llm_advisor import LLMAdvisor
|
|
40
|
+
from megalinter.native import toolchain as native_toolchain
|
|
41
|
+
from megalinter.native.context import find_git_bash, is_native_mode
|
|
42
|
+
|
|
43
|
+
# Default maximum duration of a single linter command. Far above any legitimate
|
|
44
|
+
# single-linter runtime (the slowest project scanners take minutes), so false
|
|
45
|
+
# kills stay rare while a stalled linter (e.g. I/O stall on a bind-mounted
|
|
46
|
+
# workspace) fails in bounded time instead of hanging the whole run.
|
|
47
|
+
# Override per linter with <LINTER_KEY>_TIMEOUT_SECONDS, globally with
|
|
48
|
+
# LINTER_TIMEOUT_SECONDS, and disable with value 0
|
|
49
|
+
DEFAULT_LINTER_TIMEOUT_SECONDS = 300
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Linux caps a single argv entry at MAX_ARG_STRLEN (128 KiB). Half of it is
|
|
53
|
+
# kept as the budget for generated exclusion values, leaving room for the rest
|
|
54
|
+
# of the command line
|
|
55
|
+
MAX_PROJECT_EXCLUDE_ARG_BYTES = 64 * 1024
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Linter:
|
|
59
|
+
TEMPLATES_DIR = "/action/lib/.automation/"
|
|
60
|
+
# Native mode (MegaLinter without Docker): toolchain of the installed linter
|
|
61
|
+
native_toolchain = None
|
|
62
|
+
|
|
63
|
+
# Constructor: Initialize Linter instance with name and config variables
|
|
64
|
+
def __init__(self, params=None, linter_config=None):
|
|
65
|
+
self.linter_version_cache = None
|
|
66
|
+
self.linter_help_cache = None
|
|
67
|
+
self.processing_order = 0
|
|
68
|
+
self.master = None
|
|
69
|
+
self.request_id = None
|
|
70
|
+
# Definition fields & default values: can be overridden at custom linter class level or in YML descriptors
|
|
71
|
+
# Ex: JAVASCRIPT
|
|
72
|
+
self.descriptor_id = (
|
|
73
|
+
"Field 'descriptor_id' must be overridden at custom linter class level"
|
|
74
|
+
)
|
|
75
|
+
# If you have several linters for the same language,override with a different name.Ex: JAVASCRIPT_ES
|
|
76
|
+
self.name = None
|
|
77
|
+
self.disabled = False
|
|
78
|
+
self.disabled_reason = None
|
|
79
|
+
self.is_formatter = False
|
|
80
|
+
self.is_sbom = False
|
|
81
|
+
self.linter_name = "Field 'linter_name' must be overridden at custom linter class level" # Ex: eslint
|
|
82
|
+
self.linter_speed = 3
|
|
83
|
+
self.can_output_sarif = False
|
|
84
|
+
self.output_sarif = False
|
|
85
|
+
# ex: https://eslint.org/
|
|
86
|
+
self.linter_url = (
|
|
87
|
+
"Field 'linter_url' must be overridden at custom linter class level"
|
|
88
|
+
)
|
|
89
|
+
self.linter_icon_png_url = None
|
|
90
|
+
self.test_folder = None # Override only if different from language.lowercase()
|
|
91
|
+
self.test_format_fix_file_extensions = []
|
|
92
|
+
self.test_format_fix_regex_exclude = None
|
|
93
|
+
self.activation_rules = []
|
|
94
|
+
self.activation_skip_reason = None
|
|
95
|
+
self.test_variables = {}
|
|
96
|
+
# Array of strings defining file extensions. Ex: ['.js','.cjs', '']
|
|
97
|
+
self.file_extensions = []
|
|
98
|
+
# Array of file name regular expressions. Ex: [Dockerfile(-.+)?]
|
|
99
|
+
self.file_names_regex = []
|
|
100
|
+
# Default name of the configuration file to use with the linter. Ex: '.eslintrc.js'
|
|
101
|
+
self.config_file_name = None
|
|
102
|
+
self.final_config_file = None
|
|
103
|
+
# Ignore file name and arg
|
|
104
|
+
self.ignore_file_name = None
|
|
105
|
+
self.cli_lint_ignore_arg_name = None
|
|
106
|
+
self.final_ignore_file = None
|
|
107
|
+
# Other
|
|
108
|
+
self.files_separator = None
|
|
109
|
+
self.files_sub_directory = None
|
|
110
|
+
self.file_contains_regex = []
|
|
111
|
+
self.file_contains_regex_extensions = []
|
|
112
|
+
self.file_names_not_ends_with = []
|
|
113
|
+
self.active_only_if_file_found = []
|
|
114
|
+
self.lint_all_files = False
|
|
115
|
+
self.lint_all_other_linters_files = False
|
|
116
|
+
self.is_plugin = False
|
|
117
|
+
self.pre_commands = None
|
|
118
|
+
self.post_commands = None
|
|
119
|
+
self.unsecured_env_variables = []
|
|
120
|
+
self.ignore_for_flavor_suggestions = False
|
|
121
|
+
# Maximum lint command duration in seconds, from <LINTER_KEY>_TIMEOUT_SECONDS
|
|
122
|
+
# or LINTER_TIMEOUT_SECONDS config variables. None = no timeout (default)
|
|
123
|
+
self.timeout_seconds = None
|
|
124
|
+
self.timeout_config_var = None
|
|
125
|
+
|
|
126
|
+
self.cli_lint_mode = "file"
|
|
127
|
+
self.supported_cli_lint_modes = ["file"]
|
|
128
|
+
self.cli_executable = []
|
|
129
|
+
self.cli_executable_fix = []
|
|
130
|
+
self.cli_executable_version = []
|
|
131
|
+
self.cli_executable_help = []
|
|
132
|
+
# Default arg name for configurations to use in linter CLI call
|
|
133
|
+
self.cli_config_arg_name = "-c"
|
|
134
|
+
self.cli_config_default_value = None
|
|
135
|
+
self.cli_config_extra_args = (
|
|
136
|
+
[]
|
|
137
|
+
) # Extra arguments to send to cli when a config file is used
|
|
138
|
+
self.cli_sarif_args = []
|
|
139
|
+
self.sarif_output_file = None
|
|
140
|
+
self.sarif_default_output_file = None
|
|
141
|
+
self.no_config_if_fix = False
|
|
142
|
+
self.cli_lint_extra_args = [] # Extra arguments to send to cli every time
|
|
143
|
+
self.cli_command_remove_args = (
|
|
144
|
+
[]
|
|
145
|
+
) # Arguments to remove in case fix argument is sent
|
|
146
|
+
# Name of the cli argument to send in case of APPLY_FIXES required by user
|
|
147
|
+
self.cli_lint_fix_arg_name = None
|
|
148
|
+
self.cli_lint_fix_remove_args = (
|
|
149
|
+
[]
|
|
150
|
+
) # Arguments to remove in case fix argument is sent
|
|
151
|
+
self.cli_lint_user_args = (
|
|
152
|
+
[]
|
|
153
|
+
) # Arguments from config, defined in <LINTER_KEY>_ARGUMENTS variable
|
|
154
|
+
# Extra arguments to send to cli every time, just before file argument
|
|
155
|
+
self.cli_lint_extra_args_after = []
|
|
156
|
+
self.cli_lint_mode_file_extra_args_after = []
|
|
157
|
+
self.cli_lint_mode_list_of_files_extra_args_after = []
|
|
158
|
+
self.cli_lint_mode_project_extra_args_after = []
|
|
159
|
+
# Regex matched against the linter config file: when found, the config
|
|
160
|
+
# defines the paths to analyze and cli_lint_mode_project_extra_args_after
|
|
161
|
+
# is not added, as it would override them (ex: phpcs <file>)
|
|
162
|
+
self.cli_lint_mode_project_config_paths_regex = None
|
|
163
|
+
# Workspace-relative .gitignored directories, set by MegaLinter when
|
|
164
|
+
# IGNORE_GITIGNORED_FILES is active, forwarded with excluded directories.
|
|
165
|
+
# The name ones can also be sent to tools matching them at any nesting
|
|
166
|
+
# level, as no tracked directory shares their path suffix
|
|
167
|
+
self.project_gitignored_directories = []
|
|
168
|
+
self.project_gitignored_name_directories = []
|
|
169
|
+
# Native CLI argument used to forward EXCLUDED_DIRECTORIES /
|
|
170
|
+
# ADDITIONAL_EXCLUDED_DIRECTORIES to linters running in project mode
|
|
171
|
+
self.cli_lint_mode_project_exclude_arg_name = None
|
|
172
|
+
self.cli_lint_mode_project_exclude_arg_value = "{{DIR}}"
|
|
173
|
+
self.cli_lint_mode_project_exclude_separator = None
|
|
174
|
+
# Default values re-included when the linter's exclusion argument
|
|
175
|
+
# replaces its built-in defaults (ex: bandit -x, devskim -g)
|
|
176
|
+
self.cli_lint_mode_project_exclude_seed_values = []
|
|
177
|
+
# Dotted key path (or list of alternative ones) of the user config file
|
|
178
|
+
# setting that the exclusion argument REPLACES: its entries are
|
|
179
|
+
# re-emitted first so they are preserved (ex: checkov skip-path, trivy
|
|
180
|
+
# scan.skip-dirs, phplint exclude). TOML, JSON, INI and YAML configs are
|
|
181
|
+
# read, and a * segment matches any key (ex: pylint "*.ignore")
|
|
182
|
+
self.cli_lint_mode_project_exclude_config_key = None
|
|
183
|
+
# Workspace config files also searched for the config key, for linters
|
|
184
|
+
# discovering their configuration themselves (ex: djlint pyproject.toml)
|
|
185
|
+
self.cli_lint_mode_project_exclude_config_files = []
|
|
186
|
+
# User arguments that can not be combined with the forwarding: nothing
|
|
187
|
+
# is forwarded when one of them is found (ex: v8r --no-ignore)
|
|
188
|
+
self.cli_lint_mode_project_exclude_conflicting_args = []
|
|
189
|
+
# Generated ignore file forwarding: argument receiving the generated
|
|
190
|
+
# file, workspace files merged into it (first existing wins, or all of
|
|
191
|
+
# them with merge_all_seeds), and workspace files re-passed through the
|
|
192
|
+
# same argument (when the argument replaces their default discovery)
|
|
193
|
+
self.cli_lint_mode_project_exclude_ignore_file_arg_name = None
|
|
194
|
+
self.cli_lint_mode_project_exclude_ignore_file_seed_files = []
|
|
195
|
+
self.cli_lint_mode_project_exclude_ignore_file_merge_all_seeds = False
|
|
196
|
+
self.cli_lint_mode_project_exclude_ignore_file_pass_existing = []
|
|
197
|
+
self.cli_lint_errors_count = None
|
|
198
|
+
self.cli_lint_errors_regex = None
|
|
199
|
+
self.cli_lint_warnings_count = None
|
|
200
|
+
self.cli_lint_warnings_regex = None
|
|
201
|
+
self.common_linter_errors = []
|
|
202
|
+
# Default arg name for configurations to use in linter version call
|
|
203
|
+
self.cli_version_arg_name = "--version"
|
|
204
|
+
self.cli_version_extra_args = [] # Extra arguments to send to cli every time
|
|
205
|
+
self.cli_help_arg_name = "-h"
|
|
206
|
+
self.cli_help_extra_args = [] # Extra arguments to send to cli every time
|
|
207
|
+
self.cli_help_extra_commands = []
|
|
208
|
+
# If linter --help doesn't return 0 when it's in success, override. ex: 1
|
|
209
|
+
self.help_command_return_code = 0
|
|
210
|
+
self.version_extract_regex = r"\d+(\.\d+)+"
|
|
211
|
+
# If linter --version doesn't return 0 when it's in success, override. ex: 1
|
|
212
|
+
self.version_command_return_code = 0
|
|
213
|
+
|
|
214
|
+
self.log_lines_pre: list[str] = []
|
|
215
|
+
self.log_lines_post: list[str] = []
|
|
216
|
+
|
|
217
|
+
self.report_folder = ""
|
|
218
|
+
self.reporters = []
|
|
219
|
+
self.lint_command_log: list[str] = []
|
|
220
|
+
self.lint_command_env: dict[str, str] | None = None
|
|
221
|
+
self.lint_cwd_log = ""
|
|
222
|
+
self.native_toolchain = None
|
|
223
|
+
|
|
224
|
+
# Initialize parameters
|
|
225
|
+
default_params = {
|
|
226
|
+
"default_linter_activation": False,
|
|
227
|
+
"enable_descriptors": [],
|
|
228
|
+
"enable_linters": [],
|
|
229
|
+
"disable_descriptors": [],
|
|
230
|
+
"disable_linters": [],
|
|
231
|
+
"enable_errors_linters": [],
|
|
232
|
+
"disable_errors_linters": [],
|
|
233
|
+
"post_linter_status": True,
|
|
234
|
+
}
|
|
235
|
+
if params is None:
|
|
236
|
+
params = default_params
|
|
237
|
+
else:
|
|
238
|
+
params = {**default_params, **params}
|
|
239
|
+
|
|
240
|
+
# Initialize with configuration data
|
|
241
|
+
for key, value in linter_config.items():
|
|
242
|
+
self.__setattr__(key, value)
|
|
243
|
+
self.descriptor_cli_lint_mode = self.cli_lint_mode
|
|
244
|
+
if "request_id" in params:
|
|
245
|
+
self.request_id = params["request_id"]
|
|
246
|
+
elif self.master is not None:
|
|
247
|
+
self.request_id: str = self.master.request_id
|
|
248
|
+
elif "master" in params:
|
|
249
|
+
self.request_id: str = params["master"].request_id
|
|
250
|
+
else:
|
|
251
|
+
raise Exception("Missing megalinter request_id")
|
|
252
|
+
|
|
253
|
+
self.is_active = (
|
|
254
|
+
False
|
|
255
|
+
if "default_linter_activation" not in params
|
|
256
|
+
else params["default_linter_activation"]
|
|
257
|
+
)
|
|
258
|
+
# Disable errors
|
|
259
|
+
self.disable_errors_if_less_than = None
|
|
260
|
+
self.disable_errors = (
|
|
261
|
+
True
|
|
262
|
+
if self.is_formatter is True
|
|
263
|
+
and not config.get(self.request_id, "FORMATTERS_DISABLE_ERRORS", "true")
|
|
264
|
+
== "false"
|
|
265
|
+
else (
|
|
266
|
+
True
|
|
267
|
+
if config.get(self.request_id, "DISABLE_ERRORS", "false") == "true"
|
|
268
|
+
else False
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
# Name
|
|
272
|
+
if self.name is None:
|
|
273
|
+
self.name = (
|
|
274
|
+
self.descriptor_id + "_" + self.linter_name.upper().replace("-", "_")
|
|
275
|
+
)
|
|
276
|
+
# Sarif enablement
|
|
277
|
+
self.output_sarif = (
|
|
278
|
+
params["output_sarif"]
|
|
279
|
+
if "output_sarif" in params and self.can_output_sarif is True
|
|
280
|
+
else self.output_sarif
|
|
281
|
+
)
|
|
282
|
+
if self.output_sarif is True:
|
|
283
|
+
# Disable SARIF if linter not in specified linter list
|
|
284
|
+
sarif_enabled_linters = config.get_list(
|
|
285
|
+
self.request_id, "SARIF_REPORTER_LINTERS", None
|
|
286
|
+
)
|
|
287
|
+
if (
|
|
288
|
+
sarif_enabled_linters is not None
|
|
289
|
+
and self.name not in sarif_enabled_linters
|
|
290
|
+
):
|
|
291
|
+
self.output_sarif = False
|
|
292
|
+
# Use linter_name if the descriptor does not force another executable
|
|
293
|
+
if len(self.cli_executable) == 0:
|
|
294
|
+
self.cli_executable = [self.linter_name]
|
|
295
|
+
else:
|
|
296
|
+
self.cli_executable = [self.cli_executable]
|
|
297
|
+
# Override default executable
|
|
298
|
+
if config.exists(self.request_id, self.name + "_CLI_EXECUTABLE"):
|
|
299
|
+
self.cli_executable = config.get_list(
|
|
300
|
+
self.request_id, self.name + "_CLI_EXECUTABLE"
|
|
301
|
+
)
|
|
302
|
+
if len(self.cli_executable_fix) == 0:
|
|
303
|
+
self.cli_executable_fix = [*self.cli_executable]
|
|
304
|
+
else:
|
|
305
|
+
self.cli_executable_fix = [self.cli_executable_fix]
|
|
306
|
+
if len(self.cli_executable_version) == 0:
|
|
307
|
+
self.cli_executable_version = [*self.cli_executable]
|
|
308
|
+
else:
|
|
309
|
+
self.cli_executable_version = [self.cli_executable_version]
|
|
310
|
+
if len(self.cli_executable_help) == 0:
|
|
311
|
+
self.cli_executable_help = [*self.cli_executable]
|
|
312
|
+
else:
|
|
313
|
+
self.cli_executable_help = [self.cli_executable_help]
|
|
314
|
+
if self.test_folder is None:
|
|
315
|
+
self.test_folder = self.descriptor_id.lower()
|
|
316
|
+
|
|
317
|
+
# Apply linter customization via config settings:
|
|
318
|
+
self.file_extensions = config.get_list(
|
|
319
|
+
self.request_id, self.name + "_FILE_EXTENSIONS", self.file_extensions
|
|
320
|
+
)
|
|
321
|
+
self.file_names_regex = config.get_list(
|
|
322
|
+
self.request_id, self.name + "_FILE_NAMES_REGEX", self.file_names_regex
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
self.manage_activation(params)
|
|
326
|
+
|
|
327
|
+
if self.is_active is True:
|
|
328
|
+
self.show_elapsed_time = params.get("show_elapsed_time", False)
|
|
329
|
+
|
|
330
|
+
self.manage_apply_fixes(params)
|
|
331
|
+
|
|
332
|
+
# Disable lint_all_other_linters_files=true if we are in a standalone linter docker image,
|
|
333
|
+
# because there are no other linters. Lint all files instead of none when the
|
|
334
|
+
# descriptor defines no file_extensions (e.g. SPELL_CSPELL)
|
|
335
|
+
if (
|
|
336
|
+
self.lint_all_other_linters_files is True
|
|
337
|
+
and config.get(self.request_id, "SINGLE_LINTER", "") != ""
|
|
338
|
+
):
|
|
339
|
+
self.lint_all_other_linters_files = False
|
|
340
|
+
if len(self.file_extensions) == 0:
|
|
341
|
+
self.file_extensions = ["*"]
|
|
342
|
+
|
|
343
|
+
# Config items
|
|
344
|
+
self.linter_rules_path = (
|
|
345
|
+
params["linter_rules_path"] if "linter_rules_path" in params else "."
|
|
346
|
+
)
|
|
347
|
+
self.default_rules_location = (
|
|
348
|
+
params["default_rules_location"]
|
|
349
|
+
if "default_rules_location" in params
|
|
350
|
+
else "."
|
|
351
|
+
)
|
|
352
|
+
self.workspace = params["workspace"] if "workspace" in params else "."
|
|
353
|
+
self.github_workspace = (
|
|
354
|
+
params["github_workspace"] if "github_workspace" in params else "."
|
|
355
|
+
)
|
|
356
|
+
self.config_file = None
|
|
357
|
+
self.config_file_label = None
|
|
358
|
+
self.config_file_error = None
|
|
359
|
+
self.ignore_file = None
|
|
360
|
+
self.ignore_file_label = None
|
|
361
|
+
self.ignore_file_error = None
|
|
362
|
+
self.filter_regex_include = None
|
|
363
|
+
self.filter_regex_exclude_descriptor = None
|
|
364
|
+
self.filter_regex_exclude_linter = None
|
|
365
|
+
self.post_linter_status = (
|
|
366
|
+
params["post_linter_status"]
|
|
367
|
+
if "post_linter_status" in params
|
|
368
|
+
else False
|
|
369
|
+
)
|
|
370
|
+
self.github_api_url = (
|
|
371
|
+
params["github_api_url"] if "github_api_url" in params else None
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
self.report_types = (
|
|
375
|
+
params["report_types"] if "report_types" in params else []
|
|
376
|
+
)
|
|
377
|
+
self.report_folder = (
|
|
378
|
+
params["report_folder"] if "report_folder" in params else ""
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
self.load_config_vars(params)
|
|
382
|
+
|
|
383
|
+
# Manage sub-directory filter if defined
|
|
384
|
+
if self.files_sub_directory is not None:
|
|
385
|
+
self.files_sub_directory = config.get(
|
|
386
|
+
self.request_id,
|
|
387
|
+
f"{self.descriptor_id}_DIRECTORY",
|
|
388
|
+
self.files_sub_directory,
|
|
389
|
+
)
|
|
390
|
+
if self.files_sub_directory == "any":
|
|
391
|
+
logging.info(
|
|
392
|
+
f'[Activation] {self.name} skip check of directory as value set to "any"'
|
|
393
|
+
)
|
|
394
|
+
elif not os.path.isdir(
|
|
395
|
+
self.workspace + os.path.sep + self.files_sub_directory
|
|
396
|
+
):
|
|
397
|
+
self.is_active = False
|
|
398
|
+
logging.info(
|
|
399
|
+
f"[Activation] {self.name} has been set inactive, as subdirectory has not been found:"
|
|
400
|
+
f' {self.files_sub_directory} (set value "any" to always activate)'
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Some linters require a file to be existing, else they're deactivated ( ex: .editorconfig )
|
|
404
|
+
if len(self.active_only_if_file_found) > 0:
|
|
405
|
+
is_found = False
|
|
406
|
+
for file_to_check in self.active_only_if_file_found:
|
|
407
|
+
found_file = None
|
|
408
|
+
prop = None
|
|
409
|
+
if ":" in file_to_check:
|
|
410
|
+
file_to_check, prop = file_to_check.split(":")
|
|
411
|
+
if os.path.isfile(f"{self.workspace}{os.path.sep}{file_to_check}"):
|
|
412
|
+
found_file = f"{self.workspace}{os.path.sep}{file_to_check}"
|
|
413
|
+
elif os.path.isfile(
|
|
414
|
+
self.linter_rules_path + os.path.sep + file_to_check
|
|
415
|
+
):
|
|
416
|
+
found_file = (
|
|
417
|
+
self.linter_rules_path + os.path.sep + file_to_check
|
|
418
|
+
)
|
|
419
|
+
elif os.path.isfile(
|
|
420
|
+
f"{self.workspace}{os.path.sep}{self.linter_rules_path}{os.path.sep}{file_to_check}"
|
|
421
|
+
):
|
|
422
|
+
found_file = (
|
|
423
|
+
f"{self.workspace}{os.path.sep}{self.linter_rules_path}"
|
|
424
|
+
+ f"{os.path.sep}{file_to_check}"
|
|
425
|
+
)
|
|
426
|
+
# in user repo ./config folder (https://dot-config.github.io)
|
|
427
|
+
elif os.path.isfile(
|
|
428
|
+
f"{self.workspace}{os.path.sep}.config{os.path.sep}{file_to_check}"
|
|
429
|
+
):
|
|
430
|
+
found_file = (
|
|
431
|
+
f"{self.workspace}{os.path.sep}.config"
|
|
432
|
+
+ f"{os.path.sep}{file_to_check}"
|
|
433
|
+
)
|
|
434
|
+
elif os.path.isfile(
|
|
435
|
+
f"{self.workspace}{os.path.sep}{self.files_sub_directory}{os.path.sep}{file_to_check}"
|
|
436
|
+
):
|
|
437
|
+
found_file = (
|
|
438
|
+
f"{self.workspace}{os.path.sep}{self.files_sub_directory}"
|
|
439
|
+
+ f"{os.path.sep}{file_to_check}"
|
|
440
|
+
)
|
|
441
|
+
# filename case
|
|
442
|
+
if found_file is not None and prop is None:
|
|
443
|
+
is_found = True
|
|
444
|
+
break
|
|
445
|
+
# filename + prop case
|
|
446
|
+
if found_file is not None and prop is not None:
|
|
447
|
+
with open(found_file, "r", encoding="utf-8") as json_file:
|
|
448
|
+
found_file_content = json.load(json_file)
|
|
449
|
+
if prop in found_file_content:
|
|
450
|
+
is_found = True
|
|
451
|
+
break
|
|
452
|
+
if is_found is False:
|
|
453
|
+
self.is_active = False
|
|
454
|
+
logging.info(
|
|
455
|
+
f"[Activation] {self.name} has been set inactive, as none of these files has been found:"
|
|
456
|
+
f" {str(self.active_only_if_file_found)}"
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
# Load MegaLinter reporters
|
|
460
|
+
self.load_reporters()
|
|
461
|
+
|
|
462
|
+
# Runtime items
|
|
463
|
+
self.files = []
|
|
464
|
+
self.try_fix = False
|
|
465
|
+
self.status = "success"
|
|
466
|
+
self.stdout = None
|
|
467
|
+
self.stdout_human = None
|
|
468
|
+
self.return_code = 0
|
|
469
|
+
self.number_errors = 0
|
|
470
|
+
self.total_number_errors = 0
|
|
471
|
+
self.total_number_warnings = 0
|
|
472
|
+
self.number_fixed = 0
|
|
473
|
+
self.files_lint_results = []
|
|
474
|
+
self.start_perf = None
|
|
475
|
+
self.elapsed_time_s = 0
|
|
476
|
+
self.remote_config_file_to_delete = None
|
|
477
|
+
self.remote_ignore_file_to_delete = None
|
|
478
|
+
self.llm_suggestion = None # Store LLM advisor suggestions
|
|
479
|
+
|
|
480
|
+
# Enable or disable linter
|
|
481
|
+
def manage_activation(self, params):
|
|
482
|
+
strategies = {
|
|
483
|
+
"ENABLE": False,
|
|
484
|
+
"ENABLE_LINTERS": False,
|
|
485
|
+
"DISABLE": False,
|
|
486
|
+
"DISABLE_LINTERS": False,
|
|
487
|
+
"VALIDATE": False,
|
|
488
|
+
"VALIDATE_LINTERS": False,
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
# Default value is false in case ENABLE variables are used
|
|
492
|
+
# See megalinter/MegaLinter.py, manage_default_linter_activation() function
|
|
493
|
+
# for params["default_linter_activation"]
|
|
494
|
+
self.is_active = params["default_linter_activation"]
|
|
495
|
+
# When a linter is listed in both ENABLE_LINTERS and DISABLE_LINTERS,
|
|
496
|
+
# ENABLE_LINTERS wins by default. Setting ENABLE_DISABLE_LINTERS_PRIORITY
|
|
497
|
+
# to DISABLE lets DISABLE_LINTERS override ENABLE_LINTERS instead, so an
|
|
498
|
+
# inherited (e.g. EXTENDS) ENABLE_LINTERS list can be trimmed locally.
|
|
499
|
+
disable_priority = (
|
|
500
|
+
params.get("enable_disable_linters_priority", "ENABLE") == "DISABLE"
|
|
501
|
+
)
|
|
502
|
+
in_enable_linters = self.name in params["enable_linters"]
|
|
503
|
+
in_disable_linters = self.name in params["disable_linters"]
|
|
504
|
+
# Activate or not the linter
|
|
505
|
+
if in_enable_linters and not (in_disable_linters and disable_priority):
|
|
506
|
+
self.is_active = True
|
|
507
|
+
strategies["ENABLE_LINTERS"] = True
|
|
508
|
+
elif in_disable_linters:
|
|
509
|
+
self.is_active = False
|
|
510
|
+
strategies["DISABLE_LINTERS"] = True
|
|
511
|
+
elif self.descriptor_id in params["disable_descriptors"]:
|
|
512
|
+
self.is_active = False
|
|
513
|
+
strategies["DISABLE"] = True
|
|
514
|
+
elif self.descriptor_id in params["enable_descriptors"]:
|
|
515
|
+
self.is_active = True
|
|
516
|
+
strategies["ENABLE"] = True
|
|
517
|
+
elif (
|
|
518
|
+
config.exists(self.request_id, "VALIDATE_" + self.name)
|
|
519
|
+
and config.get(self.request_id, "VALIDATE_" + self.name) == "false"
|
|
520
|
+
):
|
|
521
|
+
self.is_active = False
|
|
522
|
+
strategies["VALIDATE_LINTERS"] = True
|
|
523
|
+
elif (
|
|
524
|
+
config.exists(self.request_id, "VALIDATE_" + self.descriptor_id)
|
|
525
|
+
and config.get(self.request_id, "VALIDATE_" + self.descriptor_id) == "false"
|
|
526
|
+
):
|
|
527
|
+
self.is_active = False
|
|
528
|
+
strategies["VALIDATE"] = True
|
|
529
|
+
elif (
|
|
530
|
+
config.exists(self.request_id, "VALIDATE_" + self.name)
|
|
531
|
+
and config.get(self.request_id, "VALIDATE_" + self.name) == "true"
|
|
532
|
+
):
|
|
533
|
+
self.is_active = True
|
|
534
|
+
strategies["VALIDATE_LINTERS"] = True
|
|
535
|
+
elif (
|
|
536
|
+
config.exists(self.request_id, "VALIDATE_" + self.descriptor_id)
|
|
537
|
+
and config.get(self.request_id, "VALIDATE_" + self.descriptor_id) == "true"
|
|
538
|
+
):
|
|
539
|
+
self.is_active = True
|
|
540
|
+
strategies["VALIDATE"] = True
|
|
541
|
+
# check activation rules
|
|
542
|
+
if self.is_active is True and len(self.activation_rules) > 0:
|
|
543
|
+
self.is_active, self.activation_skip_reason = utils.check_activation_rules(
|
|
544
|
+
self.activation_rules, self
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
strategiesUsed = format(
|
|
548
|
+
", ".join("{0}".format(k) for k, v in strategies.items() if v)
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
if not strategiesUsed:
|
|
552
|
+
strategiesUsed = "default"
|
|
553
|
+
|
|
554
|
+
if self.is_active:
|
|
555
|
+
logging.debug(
|
|
556
|
+
f"[Activation] + {self.name} ({self.descriptor_id}) was activated by {strategiesUsed} strategies"
|
|
557
|
+
)
|
|
558
|
+
else:
|
|
559
|
+
logging.debug(
|
|
560
|
+
f"[Activation] - {self.name} ({self.descriptor_id}) was not activated by {strategiesUsed} strategies"
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
# Manage apply fixes flag on linter
|
|
564
|
+
def manage_apply_fixes(self, params):
|
|
565
|
+
self.apply_fixes = False
|
|
566
|
+
|
|
567
|
+
param_apply_fixes = params.get("apply_fixes", "none")
|
|
568
|
+
|
|
569
|
+
# APPLY_FIXES is "all"
|
|
570
|
+
if param_apply_fixes == "all" or (
|
|
571
|
+
isinstance(param_apply_fixes, bool) and param_apply_fixes is True
|
|
572
|
+
):
|
|
573
|
+
self.apply_fixes = True
|
|
574
|
+
# APPLY_FIXES is a comma-separated list in a single string
|
|
575
|
+
elif (
|
|
576
|
+
param_apply_fixes != "none"
|
|
577
|
+
and isinstance(param_apply_fixes, str)
|
|
578
|
+
and self.name in param_apply_fixes.split(",")
|
|
579
|
+
):
|
|
580
|
+
self.apply_fixes = True
|
|
581
|
+
# APPLY_FIXES is a list of strings
|
|
582
|
+
elif (
|
|
583
|
+
param_apply_fixes != "none"
|
|
584
|
+
and isinstance(param_apply_fixes, list)
|
|
585
|
+
and (self.name in param_apply_fixes or param_apply_fixes[0] == "all")
|
|
586
|
+
):
|
|
587
|
+
self.apply_fixes = True
|
|
588
|
+
else:
|
|
589
|
+
self.apply_fixes = False
|
|
590
|
+
|
|
591
|
+
# Check that the linter is configured to be able to apply fixes
|
|
592
|
+
if self.apply_fixes is True and (
|
|
593
|
+
self.cli_lint_fix_arg_name is not None
|
|
594
|
+
or len(self.cli_lint_fix_remove_args) > 0
|
|
595
|
+
or str(self.cli_executable_fix) != str(self.cli_executable)
|
|
596
|
+
):
|
|
597
|
+
logging.debug(
|
|
598
|
+
f"[Apply Fixes] is enabled for + {self.name} ({self.descriptor_id})"
|
|
599
|
+
)
|
|
600
|
+
elif self.apply_fixes is True:
|
|
601
|
+
self.apply_fixes = False
|
|
602
|
+
logging.debug(
|
|
603
|
+
f"[Apply Fixes] cannot be enabled for {self.name} (descriptor has no fix options configured)"
|
|
604
|
+
)
|
|
605
|
+
else:
|
|
606
|
+
logging.debug(
|
|
607
|
+
f"[Apply Fixes] is disabled for + {self.name} ({self.descriptor_id})"
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
# Manage configuration variables
|
|
611
|
+
# Return True if the given cli_lint_mode is supported by this linter.
|
|
612
|
+
# When supported_cli_lint_modes is declared in the descriptor, it is the
|
|
613
|
+
# source of truth; otherwise fall back to the single descriptor lint mode.
|
|
614
|
+
def is_cli_lint_mode_supported(self, mode):
|
|
615
|
+
if len(self.supported_cli_lint_modes) > 0:
|
|
616
|
+
return mode in self.supported_cli_lint_modes
|
|
617
|
+
return self.descriptor_cli_lint_mode == mode
|
|
618
|
+
|
|
619
|
+
def load_config_vars(self, params):
|
|
620
|
+
_sentinel = object()
|
|
621
|
+
# Configuration file name: try first NAME + _FILE_NAME, then LANGUAGE + _FILE_NAME
|
|
622
|
+
# _CONFIG_FILE = _FILE_NAME (config renaming but keeping config ascending compatibility)
|
|
623
|
+
_val = config.get(self.request_id, self.name + "_CONFIG_FILE", _sentinel)
|
|
624
|
+
if _val is not _sentinel:
|
|
625
|
+
self.config_file_name = _val
|
|
626
|
+
self.update_active_if_file_found()
|
|
627
|
+
else:
|
|
628
|
+
_val = config.get(
|
|
629
|
+
self.request_id, self.descriptor_id + "_CONFIG_FILE", _sentinel
|
|
630
|
+
)
|
|
631
|
+
if _val is not _sentinel:
|
|
632
|
+
self.config_file_name = _val
|
|
633
|
+
self.update_active_if_file_found()
|
|
634
|
+
else:
|
|
635
|
+
_val = config.get(self.request_id, self.name + "_FILE_NAME", _sentinel)
|
|
636
|
+
if _val is not _sentinel:
|
|
637
|
+
self.config_file_name = _val
|
|
638
|
+
self.update_active_if_file_found()
|
|
639
|
+
else:
|
|
640
|
+
_val = config.get(
|
|
641
|
+
self.request_id, self.descriptor_id + "_FILE_NAME", _sentinel
|
|
642
|
+
)
|
|
643
|
+
if _val is not _sentinel:
|
|
644
|
+
self.config_file_name = _val
|
|
645
|
+
self.update_active_if_file_found()
|
|
646
|
+
# Ignore file name: try first NAME + _FILE_NAME, then LANGUAGE + _FILE_NAME
|
|
647
|
+
if self.cli_lint_ignore_arg_name is not None:
|
|
648
|
+
_val = config.get(self.request_id, self.name + "_IGNORE_FILE", _sentinel)
|
|
649
|
+
if _val is not _sentinel:
|
|
650
|
+
self.ignore_file_name = _val
|
|
651
|
+
else:
|
|
652
|
+
_val = config.get(
|
|
653
|
+
self.request_id, self.descriptor_id + "_IGNORE_FILE", _sentinel
|
|
654
|
+
)
|
|
655
|
+
if _val is not _sentinel:
|
|
656
|
+
self.ignore_file_name = _val
|
|
657
|
+
# Linter rules path: try first NAME + _RULE_PATH, then LANGUAGE + _RULE_PATH
|
|
658
|
+
_val = config.get(self.request_id, self.name + "_RULES_PATH", _sentinel)
|
|
659
|
+
if _val is not _sentinel:
|
|
660
|
+
self.linter_rules_path = _val
|
|
661
|
+
else:
|
|
662
|
+
_val = config.get(
|
|
663
|
+
self.request_id, self.descriptor_id + "_RULES_PATH", _sentinel
|
|
664
|
+
)
|
|
665
|
+
if _val is not _sentinel:
|
|
666
|
+
self.linter_rules_path = _val
|
|
667
|
+
# Linter config file:
|
|
668
|
+
# 0: LINTER_DEFAULT set in user config: let the linter find it, don't reference it in cli arguments
|
|
669
|
+
# 1: http rules path: fetch remove file and copy it locally (then delete it after linting)
|
|
670
|
+
# 2: repo + config_file_name
|
|
671
|
+
# 3: linter_rules_path + config_file_name
|
|
672
|
+
# 4: workspace root + linter_rules_path + config_file_name
|
|
673
|
+
# 5: workspace root + .config + config_file_name (https://dot-config.github.io)
|
|
674
|
+
# 6: mega-linter default rules path + config_file_name
|
|
675
|
+
if (
|
|
676
|
+
self.config_file_name is not None
|
|
677
|
+
and self.config_file_name != "LINTER_DEFAULT"
|
|
678
|
+
):
|
|
679
|
+
if self.linter_rules_path.startswith("http"):
|
|
680
|
+
if not self.linter_rules_path.endswith("/"):
|
|
681
|
+
self.linter_rules_path += "/"
|
|
682
|
+
remote_config_file = self.linter_rules_path + self.config_file_name
|
|
683
|
+
local_config_file = self.workspace + os.path.sep + self.config_file_name
|
|
684
|
+
existing_before = os.path.isfile(local_config_file)
|
|
685
|
+
try:
|
|
686
|
+
with (
|
|
687
|
+
urllib.request.urlopen(remote_config_file) as response,
|
|
688
|
+
open(local_config_file, "wb") as out_file,
|
|
689
|
+
):
|
|
690
|
+
shutil.copyfileobj(response, out_file)
|
|
691
|
+
self.config_file_label = remote_config_file
|
|
692
|
+
if existing_before is False:
|
|
693
|
+
self.remote_config_file_to_delete = local_config_file
|
|
694
|
+
except urllib.error.HTTPError as e:
|
|
695
|
+
self.config_file_error = (
|
|
696
|
+
f"Unable to fetch {remote_config_file}\n{str(e)}\n"
|
|
697
|
+
f" fallback to repository config or MegaLinter default config"
|
|
698
|
+
)
|
|
699
|
+
except Exception as e:
|
|
700
|
+
self.config_file_error = (
|
|
701
|
+
f"Unable to fetch {remote_config_file}\n{str(e)}\n"
|
|
702
|
+
f" fallback to repository config or MegaLinter default config"
|
|
703
|
+
)
|
|
704
|
+
# in repo root (already here or fetched by code above)
|
|
705
|
+
if os.path.isfile(self.workspace + os.path.sep + self.config_file_name):
|
|
706
|
+
self.config_file = self.workspace + os.path.sep + self.config_file_name
|
|
707
|
+
# in user repo ./github/linters folder
|
|
708
|
+
elif os.path.isfile(
|
|
709
|
+
self.linter_rules_path + os.path.sep + self.config_file_name
|
|
710
|
+
):
|
|
711
|
+
self.config_file = (
|
|
712
|
+
self.linter_rules_path + os.path.sep + self.config_file_name
|
|
713
|
+
)
|
|
714
|
+
# in workspace root
|
|
715
|
+
elif os.path.isfile(
|
|
716
|
+
self.workspace
|
|
717
|
+
+ os.path.sep
|
|
718
|
+
+ self.linter_rules_path
|
|
719
|
+
+ os.path.sep
|
|
720
|
+
+ self.config_file_name
|
|
721
|
+
):
|
|
722
|
+
self.config_file = (
|
|
723
|
+
self.workspace
|
|
724
|
+
+ os.path.sep
|
|
725
|
+
+ self.linter_rules_path
|
|
726
|
+
+ os.path.sep
|
|
727
|
+
+ self.config_file_name
|
|
728
|
+
)
|
|
729
|
+
# in user repo ./config folder (https://dot-config.github.io)
|
|
730
|
+
elif os.path.isfile(
|
|
731
|
+
self.workspace
|
|
732
|
+
+ os.path.sep
|
|
733
|
+
+ ".config"
|
|
734
|
+
+ os.path.sep
|
|
735
|
+
+ self.config_file_name
|
|
736
|
+
):
|
|
737
|
+
self.config_file = (
|
|
738
|
+
self.workspace
|
|
739
|
+
+ os.path.sep
|
|
740
|
+
+ ".config"
|
|
741
|
+
+ os.path.sep
|
|
742
|
+
+ self.config_file_name
|
|
743
|
+
)
|
|
744
|
+
# in user repo directory provided in <Linter>RULES_PATH or LINTER_RULES_PATH
|
|
745
|
+
# Native mode: a default configuration can have a variant for native
|
|
746
|
+
# installs (ex: phpstan without the SARIF formatter installed by Composer)
|
|
747
|
+
elif is_native_mode() and os.path.isfile(
|
|
748
|
+
os.path.join(
|
|
749
|
+
self.default_rules_location, "native", self.config_file_name
|
|
750
|
+
)
|
|
751
|
+
):
|
|
752
|
+
self.config_file = os.path.join(
|
|
753
|
+
self.default_rules_location, "native", self.config_file_name
|
|
754
|
+
)
|
|
755
|
+
elif os.path.isfile(
|
|
756
|
+
self.default_rules_location + os.path.sep + self.config_file_name
|
|
757
|
+
):
|
|
758
|
+
self.config_file = (
|
|
759
|
+
self.default_rules_location + os.path.sep + self.config_file_name
|
|
760
|
+
)
|
|
761
|
+
# Make config file path absolute if not located in workspace
|
|
762
|
+
if self.config_file is not None and not os.path.isfile(
|
|
763
|
+
self.workspace + os.path.sep + self.config_file
|
|
764
|
+
):
|
|
765
|
+
self.config_file = os.path.abspath(self.config_file)
|
|
766
|
+
# Set config file label if not set by remote rule
|
|
767
|
+
if self.config_file is not None and self.config_file_label is None:
|
|
768
|
+
self.config_file_label = self.config_file.replace(
|
|
769
|
+
DEFAULT_DOCKER_WORKSPACE_DIR, ""
|
|
770
|
+
).replace(self.TEMPLATES_DIR, "")
|
|
771
|
+
|
|
772
|
+
# Linter ignore file:
|
|
773
|
+
# 0: LINTER_DEFAULT set in user config: let the linter find it, don't reference it in cli arguments
|
|
774
|
+
# 1: http rules path: fetch remove file and copy it locally (then delete it after linting)
|
|
775
|
+
# 2: repo + ignore_file_name
|
|
776
|
+
# 3: linter_rules_path + ignore_file_name
|
|
777
|
+
# 4: workspace root + linter_rules_path + ignore_file_name
|
|
778
|
+
# 5: workspace root + .config + ignore_file_name (https://dot-config.github.io)
|
|
779
|
+
# 6: mega-linter default rules path + ignore_file_name
|
|
780
|
+
if (
|
|
781
|
+
self.ignore_file_name is not None
|
|
782
|
+
and self.ignore_file_name != "LINTER_DEFAULT"
|
|
783
|
+
):
|
|
784
|
+
if self.linter_rules_path.startswith("http"):
|
|
785
|
+
if not self.linter_rules_path.endswith("/"):
|
|
786
|
+
self.linter_rules_path += "/"
|
|
787
|
+
remote_ignore_file = self.linter_rules_path + self.ignore_file_name
|
|
788
|
+
local_ignore_file = self.workspace + os.path.sep + self.ignore_file_name
|
|
789
|
+
existing_before = os.path.isfile(local_ignore_file)
|
|
790
|
+
try:
|
|
791
|
+
with (
|
|
792
|
+
urllib.request.urlopen(remote_ignore_file) as response,
|
|
793
|
+
open(local_ignore_file, "wb") as out_file,
|
|
794
|
+
):
|
|
795
|
+
shutil.copyfileobj(response, out_file)
|
|
796
|
+
self.ignore_file_label = remote_ignore_file
|
|
797
|
+
if existing_before is False:
|
|
798
|
+
self.remote_ignore_file_to_delete = local_ignore_file
|
|
799
|
+
except urllib.error.HTTPError as e:
|
|
800
|
+
self.ignore_file_error = (
|
|
801
|
+
f"Unable to fetch {remote_ignore_file}\n{str(e)}\n"
|
|
802
|
+
f" fallback to repository config or MegaLinter default ignore file"
|
|
803
|
+
)
|
|
804
|
+
except Exception as e:
|
|
805
|
+
self.ignore_file_error = (
|
|
806
|
+
f"Unable to fetch {remote_ignore_file}\n{str(e)}\n"
|
|
807
|
+
f" fallback to repository config or MegaLinter default ignore file"
|
|
808
|
+
)
|
|
809
|
+
# in repo root (already here or fetched by code above)
|
|
810
|
+
if os.path.isfile(self.workspace + os.path.sep + self.ignore_file_name):
|
|
811
|
+
self.ignore_file = self.workspace + os.path.sep + self.ignore_file_name
|
|
812
|
+
# in user repo ./github/linters folder
|
|
813
|
+
elif os.path.isfile(
|
|
814
|
+
self.linter_rules_path + os.path.sep + self.ignore_file_name
|
|
815
|
+
):
|
|
816
|
+
self.ignore_file = (
|
|
817
|
+
self.linter_rules_path + os.path.sep + self.ignore_file_name
|
|
818
|
+
)
|
|
819
|
+
# in workspace root
|
|
820
|
+
elif os.path.isfile(
|
|
821
|
+
self.workspace
|
|
822
|
+
+ os.path.sep
|
|
823
|
+
+ self.linter_rules_path
|
|
824
|
+
+ os.path.sep
|
|
825
|
+
+ self.ignore_file_name
|
|
826
|
+
):
|
|
827
|
+
self.ignore_file = (
|
|
828
|
+
self.workspace
|
|
829
|
+
+ os.path.sep
|
|
830
|
+
+ self.linter_rules_path
|
|
831
|
+
+ os.path.sep
|
|
832
|
+
+ self.ignore_file_name
|
|
833
|
+
)
|
|
834
|
+
# in user repo ./config folder (https://dot-config.github.io)
|
|
835
|
+
elif os.path.isfile(
|
|
836
|
+
self.workspace
|
|
837
|
+
+ os.path.sep
|
|
838
|
+
+ ".config"
|
|
839
|
+
+ os.path.sep
|
|
840
|
+
+ self.ignore_file_name
|
|
841
|
+
):
|
|
842
|
+
self.ignore_file = (
|
|
843
|
+
self.workspace
|
|
844
|
+
+ os.path.sep
|
|
845
|
+
+ ".config"
|
|
846
|
+
+ os.path.sep
|
|
847
|
+
+ self.ignore_file_name
|
|
848
|
+
)
|
|
849
|
+
# in user repo directory provided in <Linter>RULES_PATH or LINTER_RULES_PATH
|
|
850
|
+
elif os.path.isfile(
|
|
851
|
+
self.default_rules_location + os.path.sep + self.ignore_file_name
|
|
852
|
+
):
|
|
853
|
+
self.ignore_file = (
|
|
854
|
+
self.default_rules_location + os.path.sep + self.ignore_file_name
|
|
855
|
+
)
|
|
856
|
+
# Set ignore file label if not set by remote rule
|
|
857
|
+
if self.ignore_file is not None and self.ignore_file_label is None:
|
|
858
|
+
self.ignore_file_label = self.ignore_file.replace(
|
|
859
|
+
DEFAULT_DOCKER_WORKSPACE_DIR, ""
|
|
860
|
+
).replace(self.TEMPLATES_DIR, "")
|
|
861
|
+
|
|
862
|
+
# User override of cli_lint_mode: only accept a mode the linter declares
|
|
863
|
+
# as supported (supported_cli_lint_modes), otherwise fail fast with an
|
|
864
|
+
# actionable error listing the modes that are actually available.
|
|
865
|
+
if config.exists(self.request_id, self.name + "_CLI_LINT_MODE"):
|
|
866
|
+
cli_lint_mode_config = config.get(
|
|
867
|
+
self.request_id, self.name + "_CLI_LINT_MODE"
|
|
868
|
+
)
|
|
869
|
+
if not self.is_cli_lint_mode_supported(cli_lint_mode_config):
|
|
870
|
+
supported_modes = (
|
|
871
|
+
self.supported_cli_lint_modes
|
|
872
|
+
if len(self.supported_cli_lint_modes) > 0
|
|
873
|
+
else [self.descriptor_cli_lint_mode]
|
|
874
|
+
)
|
|
875
|
+
raise ValueError(
|
|
876
|
+
f"[Configuration] {self.name}_CLI_LINT_MODE is set to "
|
|
877
|
+
f"'{cli_lint_mode_config}', which is not supported by {self.name}. "
|
|
878
|
+
f"Supported lint mode(s): {', '.join(supported_modes)}"
|
|
879
|
+
)
|
|
880
|
+
self.cli_lint_mode = cli_lint_mode_config
|
|
881
|
+
|
|
882
|
+
# Include regex :try first NAME + _FILTER_REGEX_INCLUDE, then LANGUAGE + _FILTER_REGEX_INCLUDE
|
|
883
|
+
if config.exists(self.request_id, self.name + "_FILTER_REGEX_INCLUDE"):
|
|
884
|
+
self.filter_regex_include = config.get(
|
|
885
|
+
self.request_id, self.name + "_FILTER_REGEX_INCLUDE"
|
|
886
|
+
)
|
|
887
|
+
elif config.exists(
|
|
888
|
+
self.request_id, self.descriptor_id + "_FILTER_REGEX_INCLUDE"
|
|
889
|
+
):
|
|
890
|
+
self.filter_regex_include = config.get(
|
|
891
|
+
self.request_id, self.descriptor_id + "_FILTER_REGEX_INCLUDE"
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
# User arguments from config
|
|
895
|
+
if (
|
|
896
|
+
config.get(self.request_id, self.name + "_COMMAND_REMOVE_ARGUMENTS", "")
|
|
897
|
+
!= ""
|
|
898
|
+
):
|
|
899
|
+
self.cli_command_remove_args = config.get_list_args(
|
|
900
|
+
self.request_id, self.name + "_COMMAND_REMOVE_ARGUMENTS"
|
|
901
|
+
)
|
|
902
|
+
|
|
903
|
+
# User remove arguments from config
|
|
904
|
+
if config.get(self.request_id, self.name + "_ARGUMENTS", "") != "":
|
|
905
|
+
self.cli_lint_user_args = config.get_list_args(
|
|
906
|
+
self.request_id, self.name + "_ARGUMENTS"
|
|
907
|
+
)
|
|
908
|
+
# Get PRE_COMMANDS overridden by user
|
|
909
|
+
if config.get(self.request_id, self.name + "_PRE_COMMANDS", "") != "":
|
|
910
|
+
self.pre_commands = config.get_list(
|
|
911
|
+
self.request_id, self.name + "_PRE_COMMANDS"
|
|
912
|
+
)
|
|
913
|
+
|
|
914
|
+
# Get POST_COMMANDS overridden by user
|
|
915
|
+
if config.get(self.request_id, self.name + "_POST_COMMANDS", "") != "":
|
|
916
|
+
self.post_commands = config.get_list(
|
|
917
|
+
self.request_id, self.name + "_POST_COMMANDS"
|
|
918
|
+
)
|
|
919
|
+
|
|
920
|
+
# Get secured variables allow list
|
|
921
|
+
if config.exists(self.request_id, self.name + "_UNSECURED_ENV_VARIABLES"):
|
|
922
|
+
self.unsecured_env_variables = config.get_list(
|
|
923
|
+
self.request_id, self.name + "_UNSECURED_ENV_VARIABLES"
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
# Lint command timeout: NAME + _TIMEOUT_SECONDS, then LINTER_TIMEOUT_SECONDS
|
|
927
|
+
self.load_timeout_config()
|
|
928
|
+
|
|
929
|
+
# Disable errors for this linter NAME + _DISABLE_ERRORS, then LANGUAGE + _DISABLE_ERRORS
|
|
930
|
+
if config.get(self.request_id, self.name + "_DISABLE_ERRORS_IF_LESS_THAN"):
|
|
931
|
+
self.disable_errors_if_less_than = int(
|
|
932
|
+
config.get(self.request_id, self.name + "_DISABLE_ERRORS_IF_LESS_THAN")
|
|
933
|
+
)
|
|
934
|
+
if self.disable_errors_if_less_than is not None:
|
|
935
|
+
self.disable_errors = False
|
|
936
|
+
elif self.name in params["disable_errors_linters"]:
|
|
937
|
+
self.disable_errors = True
|
|
938
|
+
elif (
|
|
939
|
+
"enable_errors_linters" in params
|
|
940
|
+
and len(params["enable_errors_linters"]) > 0
|
|
941
|
+
and self.name in params["enable_errors_linters"]
|
|
942
|
+
):
|
|
943
|
+
self.disable_errors = False
|
|
944
|
+
elif (
|
|
945
|
+
"enable_errors_linters" in params
|
|
946
|
+
and len(params["enable_errors_linters"]) > 0
|
|
947
|
+
and self.name not in params["enable_errors_linters"]
|
|
948
|
+
):
|
|
949
|
+
self.disable_errors = True
|
|
950
|
+
elif config.get(self.request_id, self.name + "_DISABLE_ERRORS", "") == "false":
|
|
951
|
+
self.disable_errors = False
|
|
952
|
+
elif config.get(self.request_id, self.name + "_DISABLE_ERRORS", "") == "true":
|
|
953
|
+
self.disable_errors = True
|
|
954
|
+
elif (
|
|
955
|
+
config.get(self.request_id, self.descriptor_id + "_DISABLE_ERRORS", "")
|
|
956
|
+
== "false"
|
|
957
|
+
):
|
|
958
|
+
self.disable_errors = False
|
|
959
|
+
elif (
|
|
960
|
+
config.get(self.request_id, self.descriptor_id + "_DISABLE_ERRORS", "")
|
|
961
|
+
== "true"
|
|
962
|
+
):
|
|
963
|
+
self.disable_errors = True
|
|
964
|
+
# Exclude regex: descriptor level
|
|
965
|
+
if config.exists(self.request_id, self.descriptor_id + "_FILTER_REGEX_EXCLUDE"):
|
|
966
|
+
self.filter_regex_exclude_descriptor = config.get(
|
|
967
|
+
self.request_id, self.descriptor_id + "_FILTER_REGEX_EXCLUDE"
|
|
968
|
+
)
|
|
969
|
+
# Exclude regex: linter level
|
|
970
|
+
if config.exists(self.request_id, self.name + "_FILTER_REGEX_EXCLUDE"):
|
|
971
|
+
self.filter_regex_exclude_linter = config.get(
|
|
972
|
+
self.request_id, self.name + "_FILTER_REGEX_EXCLUDE"
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
# Resolve the maximum lint command duration from <LINTER_KEY>_TIMEOUT_SECONDS,
|
|
976
|
+
# falling back to the global LINTER_TIMEOUT_SECONDS, then to
|
|
977
|
+
# DEFAULT_LINTER_TIMEOUT_SECONDS (300). An explicit 0 disables the timeout
|
|
978
|
+
# entirely (for repositories that legitimately need very long linter runs).
|
|
979
|
+
# An invalid value (not a positive integer or 0) is ignored with a warning,
|
|
980
|
+
# and the next variable in the list is tried
|
|
981
|
+
def load_timeout_config(self):
|
|
982
|
+
for timeout_var in [self.name + "_TIMEOUT_SECONDS", "LINTER_TIMEOUT_SECONDS"]:
|
|
983
|
+
if not config.exists(self.request_id, timeout_var):
|
|
984
|
+
continue
|
|
985
|
+
timeout_raw = config.get(self.request_id, timeout_var)
|
|
986
|
+
try:
|
|
987
|
+
timeout_value = int(str(timeout_raw).strip())
|
|
988
|
+
except (TypeError, ValueError):
|
|
989
|
+
timeout_value = None
|
|
990
|
+
if timeout_value is None or timeout_value < 0:
|
|
991
|
+
logging.warning(
|
|
992
|
+
f"[Config] Invalid {timeout_var} value [{timeout_raw}]:"
|
|
993
|
+
" it must be a positive integer number of seconds,"
|
|
994
|
+
" or 0 to disable the timeout. Value ignored."
|
|
995
|
+
)
|
|
996
|
+
continue
|
|
997
|
+
if timeout_value == 0:
|
|
998
|
+
# Explicit opt-out: no timeout for this linter
|
|
999
|
+
self.timeout_seconds = None
|
|
1000
|
+
else:
|
|
1001
|
+
self.timeout_seconds = timeout_value
|
|
1002
|
+
self.timeout_config_var = timeout_var
|
|
1003
|
+
return
|
|
1004
|
+
self.timeout_seconds = DEFAULT_LINTER_TIMEOUT_SECONDS
|
|
1005
|
+
self.timeout_config_var = "LINTER_TIMEOUT_SECONDS (default)"
|
|
1006
|
+
|
|
1007
|
+
# If linter is activated only if some file is found, and config file has been overridden
|
|
1008
|
+
# -> add it to the files to check
|
|
1009
|
+
def update_active_if_file_found(self):
|
|
1010
|
+
if (
|
|
1011
|
+
len(self.active_only_if_file_found) > 0
|
|
1012
|
+
and self.config_file_name not in self.active_only_if_file_found
|
|
1013
|
+
):
|
|
1014
|
+
self.active_only_if_file_found.append(self.config_file_name)
|
|
1015
|
+
|
|
1016
|
+
# Processes the linter
|
|
1017
|
+
def run(
|
|
1018
|
+
self,
|
|
1019
|
+
run_commands_before_linters=None,
|
|
1020
|
+
run_commands_after_linters=None,
|
|
1021
|
+
skip_console_reporter=False,
|
|
1022
|
+
):
|
|
1023
|
+
self.start_perf = perf_counter()
|
|
1024
|
+
|
|
1025
|
+
# Pre-compute subprocess environment once per linter run to avoid
|
|
1026
|
+
# rebuilding secured-variable regexes on every file
|
|
1027
|
+
self._cached_subprocess_env = self.native_environment(
|
|
1028
|
+
{
|
|
1029
|
+
**config.build_env(self.request_id, True, self.unsecured_env_variables),
|
|
1030
|
+
"FORCE_COLOR": "0",
|
|
1031
|
+
}
|
|
1032
|
+
)
|
|
1033
|
+
|
|
1034
|
+
# Initialize linter reports
|
|
1035
|
+
for reporter in self.reporters:
|
|
1036
|
+
reporter.initialize()
|
|
1037
|
+
|
|
1038
|
+
# Apply actions defined on Linter class if defined
|
|
1039
|
+
self.before_lint_files()
|
|
1040
|
+
|
|
1041
|
+
# Run commands defined in descriptor, or overridden by user in configuration
|
|
1042
|
+
pre_post_factory.run_linter_pre_commands(
|
|
1043
|
+
self.master, self, run_commands_before_linters
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
# Lint each file one by one
|
|
1047
|
+
if self.cli_lint_mode == "file":
|
|
1048
|
+
index = 0
|
|
1049
|
+
for file in self.files:
|
|
1050
|
+
file_status = "success"
|
|
1051
|
+
index = index + 1
|
|
1052
|
+
return_code, stdout = self.process_linter(file)
|
|
1053
|
+
file_errors_number = 0
|
|
1054
|
+
file_warnings_number = 0
|
|
1055
|
+
file_warnings_number = self.get_total_number_warnings(stdout)
|
|
1056
|
+
self.total_number_warnings += file_warnings_number
|
|
1057
|
+
if return_code != 0:
|
|
1058
|
+
file_status = "error"
|
|
1059
|
+
self.status = "warning" if self.disable_errors is True else "error"
|
|
1060
|
+
self.return_code = (
|
|
1061
|
+
self.return_code if self.disable_errors is True else 1
|
|
1062
|
+
)
|
|
1063
|
+
self.number_errors += 1
|
|
1064
|
+
# Calls external functions to count the number of warnings and errors
|
|
1065
|
+
file_errors_number = self.get_total_number_errors(stdout)
|
|
1066
|
+
|
|
1067
|
+
self.total_number_errors += file_errors_number
|
|
1068
|
+
self.update_files_lint_results(
|
|
1069
|
+
[file],
|
|
1070
|
+
return_code,
|
|
1071
|
+
file_status,
|
|
1072
|
+
stdout,
|
|
1073
|
+
file_errors_number,
|
|
1074
|
+
file_warnings_number,
|
|
1075
|
+
)
|
|
1076
|
+
elif self.cli_lint_mode == "list_of_files" and len(self.files) == 0:
|
|
1077
|
+
# Nothing to lint: appending an empty list of files to the command
|
|
1078
|
+
# line builds an invalid command (ex: "checkov --file" with no value
|
|
1079
|
+
# after it), and the linter would fail on its own arguments instead
|
|
1080
|
+
# of reporting a lint result
|
|
1081
|
+
logging.info(
|
|
1082
|
+
f"{self.name} has no file to lint, so its command has not been run"
|
|
1083
|
+
)
|
|
1084
|
+
else:
|
|
1085
|
+
# Lint all workspace in one command
|
|
1086
|
+
return_code, stdout = self.process_linter()
|
|
1087
|
+
self.stdout = stdout
|
|
1088
|
+
# Count warnings regardless of return code
|
|
1089
|
+
self.total_number_warnings += self.get_total_number_warnings(stdout)
|
|
1090
|
+
if return_code != 0:
|
|
1091
|
+
self.status = "warning" if self.disable_errors is True else "error"
|
|
1092
|
+
self.return_code = 0 if self.disable_errors is True else 1
|
|
1093
|
+
self.number_errors += 1
|
|
1094
|
+
self.total_number_errors += self.get_total_number_errors(stdout)
|
|
1095
|
+
elif self.total_number_warnings > 0:
|
|
1096
|
+
self.status = "warning"
|
|
1097
|
+
# Build result for list of files
|
|
1098
|
+
if self.cli_lint_mode == "list_of_files":
|
|
1099
|
+
self.update_files_lint_results(self.files, None, None, None, None, None)
|
|
1100
|
+
|
|
1101
|
+
# Set return code to 0 if failures in this linter must not make the MegaLinter run fail
|
|
1102
|
+
if self.return_code != 0:
|
|
1103
|
+
# Disable errors: no failure, just warning
|
|
1104
|
+
if self.disable_errors is True:
|
|
1105
|
+
self.return_code = 0
|
|
1106
|
+
elif (
|
|
1107
|
+
self.disable_errors_if_less_than is not None
|
|
1108
|
+
and self.total_number_errors < self.disable_errors_if_less_than
|
|
1109
|
+
):
|
|
1110
|
+
self.return_code = 0
|
|
1111
|
+
|
|
1112
|
+
# Set LLM Advisor suggestions if available
|
|
1113
|
+
try:
|
|
1114
|
+
llm_advisor = LLMAdvisor(self.request_id)
|
|
1115
|
+
if llm_advisor.is_available() and llm_advisor.should_analyze_linter(self):
|
|
1116
|
+
# Get raw linter output for AI analysis
|
|
1117
|
+
raw_linter_output = ""
|
|
1118
|
+
if self.stdout is not None:
|
|
1119
|
+
raw_linter_output = self.stdout
|
|
1120
|
+
else:
|
|
1121
|
+
# Try to read from the linter output file
|
|
1122
|
+
text_report_sub_folder = config.get(
|
|
1123
|
+
self.request_id, "TEXT_REPORTER_SUB_FOLDER", "linters_logs"
|
|
1124
|
+
)
|
|
1125
|
+
text_file_name = (
|
|
1126
|
+
f"{self.report_folder}{os.path.sep}"
|
|
1127
|
+
f"{text_report_sub_folder}{os.path.sep}"
|
|
1128
|
+
f"{self.name}-{self.status.upper()}.log"
|
|
1129
|
+
)
|
|
1130
|
+
if os.path.isfile(text_file_name):
|
|
1131
|
+
try:
|
|
1132
|
+
with open(
|
|
1133
|
+
text_file_name, "r", encoding="utf-8"
|
|
1134
|
+
) as text_file:
|
|
1135
|
+
full_output = text_file.read()
|
|
1136
|
+
# Extract raw linter output (after "Linter raw log:" separator if present)
|
|
1137
|
+
separator_pos = full_output.find("Linter raw log:")
|
|
1138
|
+
if separator_pos != -1:
|
|
1139
|
+
next_newline = full_output.find("\n", separator_pos)
|
|
1140
|
+
if next_newline != -1:
|
|
1141
|
+
raw_linter_output = full_output[
|
|
1142
|
+
next_newline + 1 :
|
|
1143
|
+
].strip()
|
|
1144
|
+
else:
|
|
1145
|
+
raw_linter_output = full_output.strip()
|
|
1146
|
+
except Exception as e:
|
|
1147
|
+
logging.warning(
|
|
1148
|
+
f"Error reading linter output for LLM analysis: {str(e)}"
|
|
1149
|
+
)
|
|
1150
|
+
|
|
1151
|
+
# Get AI suggestions if we have output to analyze
|
|
1152
|
+
if raw_linter_output.strip():
|
|
1153
|
+
self.llm_suggestion = llm_advisor.get_fix_suggestions(
|
|
1154
|
+
self, raw_linter_output
|
|
1155
|
+
)
|
|
1156
|
+
else:
|
|
1157
|
+
logging.debug(
|
|
1158
|
+
f"[LLM Advisor] No linter output available for LLM analysis for {self.name}"
|
|
1159
|
+
)
|
|
1160
|
+
else:
|
|
1161
|
+
logging.debug(
|
|
1162
|
+
f"[LLM Advisor] LLM advisor not available or not analyzing {self.name}"
|
|
1163
|
+
)
|
|
1164
|
+
except Exception as e:
|
|
1165
|
+
logging.warning(
|
|
1166
|
+
f"[LLM Advisor] Error initializing LLM advisor for {self.name}: {str(e)}"
|
|
1167
|
+
)
|
|
1168
|
+
self.llm_suggestion = None
|
|
1169
|
+
|
|
1170
|
+
# Delete locally copied remote config file if necessary
|
|
1171
|
+
if self.remote_config_file_to_delete is not None:
|
|
1172
|
+
os.remove(self.remote_config_file_to_delete)
|
|
1173
|
+
|
|
1174
|
+
# Delete locally copied remote ignore file if necessary
|
|
1175
|
+
if self.remote_ignore_file_to_delete is not None:
|
|
1176
|
+
os.remove(self.remote_ignore_file_to_delete)
|
|
1177
|
+
|
|
1178
|
+
# Run commands defined in descriptor, or overridden by user in configuration
|
|
1179
|
+
pre_post_factory.run_linter_post_commands(
|
|
1180
|
+
self.master, self, run_commands_after_linters
|
|
1181
|
+
)
|
|
1182
|
+
|
|
1183
|
+
# Generate linter reports
|
|
1184
|
+
self.elapsed_time_s = perf_counter() - self.start_perf
|
|
1185
|
+
for reporter in self.reporters:
|
|
1186
|
+
try:
|
|
1187
|
+
# Skip console reporter when running in parallel workers;
|
|
1188
|
+
# it will be produced in the main process to avoid interleaved
|
|
1189
|
+
# CI log section markers (::group::/::endgroup::)
|
|
1190
|
+
if skip_console_reporter and reporter.name == "CONSOLE":
|
|
1191
|
+
continue
|
|
1192
|
+
reporter.produce_report()
|
|
1193
|
+
except Exception as e:
|
|
1194
|
+
logging.error("Unable to process reporter " + reporter.name + str(e))
|
|
1195
|
+
|
|
1196
|
+
return self
|
|
1197
|
+
|
|
1198
|
+
def replace_vars(self, args, additional_variables=None):
|
|
1199
|
+
default_variables = {
|
|
1200
|
+
"{{SARIF_OUTPUT_FILE}}": self.sarif_output_file,
|
|
1201
|
+
"{{REPORT_FOLDER}}": self.report_folder,
|
|
1202
|
+
"{{WORKSPACE}}": self.workspace,
|
|
1203
|
+
}
|
|
1204
|
+
merged_map = default_variables
|
|
1205
|
+
if additional_variables is not None:
|
|
1206
|
+
merged_map.update(additional_variables)
|
|
1207
|
+
args_with_replacements = []
|
|
1208
|
+
for txt in args:
|
|
1209
|
+
for key in merged_map:
|
|
1210
|
+
if key in txt:
|
|
1211
|
+
txt = txt.replace(key, merged_map[key])
|
|
1212
|
+
args_with_replacements += [txt]
|
|
1213
|
+
|
|
1214
|
+
return args_with_replacements
|
|
1215
|
+
|
|
1216
|
+
def update_files_lint_results(
|
|
1217
|
+
self,
|
|
1218
|
+
linted_files,
|
|
1219
|
+
return_code,
|
|
1220
|
+
file_status,
|
|
1221
|
+
stdout,
|
|
1222
|
+
file_errors_number,
|
|
1223
|
+
file_warnings_number,
|
|
1224
|
+
):
|
|
1225
|
+
if self.try_fix is True:
|
|
1226
|
+
updated_files = utils.list_updated_files(self.github_workspace)
|
|
1227
|
+
else:
|
|
1228
|
+
updated_files = []
|
|
1229
|
+
for file in linted_files:
|
|
1230
|
+
if self.try_fix is True:
|
|
1231
|
+
fixed = utils.check_updated_file(
|
|
1232
|
+
file, self.github_workspace, updated_files
|
|
1233
|
+
)
|
|
1234
|
+
else:
|
|
1235
|
+
fixed = False
|
|
1236
|
+
if fixed is True:
|
|
1237
|
+
self.number_fixed = self.number_fixed + 1
|
|
1238
|
+
# store result
|
|
1239
|
+
self.files_lint_results += [
|
|
1240
|
+
{
|
|
1241
|
+
"file": file,
|
|
1242
|
+
"status_code": return_code,
|
|
1243
|
+
"status": file_status,
|
|
1244
|
+
"stdout": stdout,
|
|
1245
|
+
"fixed": fixed,
|
|
1246
|
+
"errors_number": file_errors_number,
|
|
1247
|
+
"warnings_number": file_warnings_number,
|
|
1248
|
+
}
|
|
1249
|
+
]
|
|
1250
|
+
|
|
1251
|
+
# List all reporters, then instantiate each of them
|
|
1252
|
+
def load_reporters(self):
|
|
1253
|
+
reporter_init_params = {"master": self, "report_folder": self.report_folder}
|
|
1254
|
+
self.reporters = utils.list_active_reporters_for_scope(
|
|
1255
|
+
"linter", reporter_init_params
|
|
1256
|
+
)
|
|
1257
|
+
|
|
1258
|
+
def log_file_filters(self):
|
|
1259
|
+
log_object = {
|
|
1260
|
+
"name": self.name,
|
|
1261
|
+
"filter_regex_include": self.filter_regex_include,
|
|
1262
|
+
"filter_regex_exclude_descriptor": self.filter_regex_exclude_descriptor,
|
|
1263
|
+
"filter_regex_exclude_linter": self.filter_regex_exclude_linter,
|
|
1264
|
+
"files_sub_directory": self.files_sub_directory,
|
|
1265
|
+
"lint_all_files": self.lint_all_files,
|
|
1266
|
+
"lint_all_other_linters_files": self.lint_all_other_linters_files,
|
|
1267
|
+
"file_extensions": self.file_extensions,
|
|
1268
|
+
"file_names_regex": self.file_names_regex,
|
|
1269
|
+
"file_names_not_ends_with": self.file_names_not_ends_with,
|
|
1270
|
+
"file_contains_regex": self.file_contains_regex,
|
|
1271
|
+
"file_contains_regex_extensions": self.file_contains_regex_extensions,
|
|
1272
|
+
}
|
|
1273
|
+
logging.debug("[Filters] " + str(log_object))
|
|
1274
|
+
|
|
1275
|
+
# Collect all files that will be analyzed by the current linter
|
|
1276
|
+
def collect_files(self, all_files):
|
|
1277
|
+
self.log_file_filters()
|
|
1278
|
+
# Filter all files to keep only the ones matching with the current linter
|
|
1279
|
+
self.files = utils.filter_files(
|
|
1280
|
+
all_files=all_files,
|
|
1281
|
+
filter_regex_include=self.filter_regex_include,
|
|
1282
|
+
filter_regex_exclude=[
|
|
1283
|
+
self.filter_regex_exclude_descriptor,
|
|
1284
|
+
self.filter_regex_exclude_linter,
|
|
1285
|
+
],
|
|
1286
|
+
file_names_regex=self.file_names_regex,
|
|
1287
|
+
file_extensions=self.file_extensions,
|
|
1288
|
+
ignored_files=[],
|
|
1289
|
+
ignore_generated_files=False, # This filter is applied at MegaLinter level
|
|
1290
|
+
file_names_not_ends_with=self.file_names_not_ends_with,
|
|
1291
|
+
file_contains_regex=self.file_contains_regex,
|
|
1292
|
+
file_contains_regex_extensions=self.file_contains_regex_extensions,
|
|
1293
|
+
files_sub_directory=self.files_sub_directory,
|
|
1294
|
+
lint_all_other_linters_files=self.lint_all_other_linters_files,
|
|
1295
|
+
workspace=self.workspace,
|
|
1296
|
+
)
|
|
1297
|
+
self.files_number = len(self.files)
|
|
1298
|
+
logging.debug(
|
|
1299
|
+
f"{self.name} linter kept {self.files_number} files after applying linter filters:"
|
|
1300
|
+
+ utils.format_bullet_list(self.files)
|
|
1301
|
+
)
|
|
1302
|
+
|
|
1303
|
+
# Drop files whose name can not be carried safely to an interpreter
|
|
1304
|
+
# through an environment variable (line breaks, NUL...)
|
|
1305
|
+
def drop_files_unsafe_for_interpreter(self):
|
|
1306
|
+
safe_files = []
|
|
1307
|
+
for file in self.files:
|
|
1308
|
+
if re.search(r"[\x00-\x1f\x7f]", file):
|
|
1309
|
+
logging.warning(
|
|
1310
|
+
f"[{self.linter_name}] Skipped {file!r}: its name contains "
|
|
1311
|
+
"control characters that can not be passed safely to the linter"
|
|
1312
|
+
)
|
|
1313
|
+
else:
|
|
1314
|
+
safe_files.append(file)
|
|
1315
|
+
self.files = safe_files
|
|
1316
|
+
self.files_number = len(self.files)
|
|
1317
|
+
|
|
1318
|
+
# lint a single file or whole project
|
|
1319
|
+
def process_linter(self, file=None):
|
|
1320
|
+
# Remove previous run SARIF file if necessary
|
|
1321
|
+
if self.sarif_output_file is not None and os.path.isfile(
|
|
1322
|
+
self.sarif_output_file
|
|
1323
|
+
):
|
|
1324
|
+
os.remove(self.sarif_output_file)
|
|
1325
|
+
# Build command using method locally defined on Linter class
|
|
1326
|
+
command = self.build_lint_command(file)
|
|
1327
|
+
# Output command if debug mode
|
|
1328
|
+
logging.debug(f"[{self.linter_name}] command: {str(command)}")
|
|
1329
|
+
# Run command via CLI
|
|
1330
|
+
return_code, return_output = self.execute_lint_command(command)
|
|
1331
|
+
logging.debug(
|
|
1332
|
+
f"[{self.linter_name}] result: {str(return_code)} {return_output}"
|
|
1333
|
+
)
|
|
1334
|
+
return return_code, return_output
|
|
1335
|
+
|
|
1336
|
+
def native_environment(self, env):
|
|
1337
|
+
if self.native_toolchain is None:
|
|
1338
|
+
return env
|
|
1339
|
+
return native_toolchain.build_subprocess_env(
|
|
1340
|
+
env, self.native_toolchain, self.request_id, self.unsecured_env_variables
|
|
1341
|
+
)
|
|
1342
|
+
|
|
1343
|
+
def native_search_path(self, env=None):
|
|
1344
|
+
if self.native_toolchain is None:
|
|
1345
|
+
return None
|
|
1346
|
+
if env is not None:
|
|
1347
|
+
return env.get("PATH")
|
|
1348
|
+
return os.pathsep.join(
|
|
1349
|
+
[*self.native_toolchain.path_dirs, os.environ.get("PATH", "")]
|
|
1350
|
+
)
|
|
1351
|
+
|
|
1352
|
+
# Execute a linting command . Can be overridden for special cases, like use of PowerShell script
|
|
1353
|
+
# noinspection PyMethodMayBeStatic
|
|
1354
|
+
def execute_lint_command(self, command):
|
|
1355
|
+
cwd = os.path.abspath(self.workspace)
|
|
1356
|
+
self.lint_cwd_log = cwd
|
|
1357
|
+
logging.debug(f"[{self.linter_name}] CWD: {cwd}")
|
|
1358
|
+
# Use pre-computed env from run(), fall back to building it if called standalone
|
|
1359
|
+
subprocess_env = getattr(self, "_cached_subprocess_env", None)
|
|
1360
|
+
if subprocess_env is None:
|
|
1361
|
+
subprocess_env = self.native_environment(
|
|
1362
|
+
{
|
|
1363
|
+
**config.build_env(
|
|
1364
|
+
self.request_id, True, self.unsecured_env_variables
|
|
1365
|
+
),
|
|
1366
|
+
"FORCE_COLOR": "0",
|
|
1367
|
+
}
|
|
1368
|
+
)
|
|
1369
|
+
if self.lint_command_env:
|
|
1370
|
+
subprocess_env = {**subprocess_env, **self.lint_command_env}
|
|
1371
|
+
logging.debug(f"[{self.linter_name}] Command env: {self.lint_command_env}")
|
|
1372
|
+
if isinstance(command, str) and sys.platform == "win32":
|
|
1373
|
+
self.lint_command_log.append(command)
|
|
1374
|
+
# Windows: shell=True would wrap the command for cmd.exe, so call Git Bash
|
|
1375
|
+
# directly with the command as a single argument
|
|
1376
|
+
return_code, return_stdout = self._run_lint_subprocess(
|
|
1377
|
+
[find_git_bash() or shutil.which("bash"), "-c", command],
|
|
1378
|
+
{"cwd": cwd, "env": subprocess_env},
|
|
1379
|
+
)
|
|
1380
|
+
elif isinstance(command, str):
|
|
1381
|
+
self.lint_command_log.append(command)
|
|
1382
|
+
# Call linter with a sub-process
|
|
1383
|
+
return_code, return_stdout = self._run_lint_subprocess(
|
|
1384
|
+
command,
|
|
1385
|
+
{
|
|
1386
|
+
"shell": True,
|
|
1387
|
+
"cwd": cwd,
|
|
1388
|
+
"env": subprocess_env,
|
|
1389
|
+
"executable": "/bin/bash",
|
|
1390
|
+
},
|
|
1391
|
+
)
|
|
1392
|
+
else:
|
|
1393
|
+
# Use full executable path if we are on Windows
|
|
1394
|
+
if sys.platform == "win32":
|
|
1395
|
+
cli_absolute = shutil.which(
|
|
1396
|
+
command[0], path=self.native_search_path(subprocess_env)
|
|
1397
|
+
)
|
|
1398
|
+
if cli_absolute is not None:
|
|
1399
|
+
command[0] = cli_absolute
|
|
1400
|
+
else:
|
|
1401
|
+
msg = "Unable to find command: " + command[0]
|
|
1402
|
+
logging.error(msg)
|
|
1403
|
+
return errno.ESRCH, msg
|
|
1404
|
+
self.lint_command_log.append(" ".join(command))
|
|
1405
|
+
# Call linter with a sub-process (RECOMMENDED: with a list of strings corresponding to the command)
|
|
1406
|
+
try:
|
|
1407
|
+
return_code, return_stdout = self._run_lint_subprocess(
|
|
1408
|
+
command,
|
|
1409
|
+
{
|
|
1410
|
+
"env": subprocess_env,
|
|
1411
|
+
"cwd": cwd,
|
|
1412
|
+
},
|
|
1413
|
+
)
|
|
1414
|
+
except FileNotFoundError as err:
|
|
1415
|
+
return_code = 999
|
|
1416
|
+
return_stdout = (
|
|
1417
|
+
f"Fatal error while calling {self.linter_name}: {str(err)}"
|
|
1418
|
+
)
|
|
1419
|
+
except Exception as err:
|
|
1420
|
+
return_code = 99
|
|
1421
|
+
return_stdout = (
|
|
1422
|
+
f"Fatal error while calling {self.linter_name}: {str(err)}"
|
|
1423
|
+
)
|
|
1424
|
+
return_code, return_stdout = self.apply_common_linter_errors(
|
|
1425
|
+
return_code, return_stdout
|
|
1426
|
+
)
|
|
1427
|
+
self.manage_sarif_output(return_stdout)
|
|
1428
|
+
# Return linter result
|
|
1429
|
+
return return_code, return_stdout
|
|
1430
|
+
|
|
1431
|
+
# Run the linter sub-process, enforcing timeout_seconds when set
|
|
1432
|
+
# (from <LINTER_KEY>_TIMEOUT_SECONDS or LINTER_TIMEOUT_SECONDS).
|
|
1433
|
+
# A plain subprocess timeout only kills the direct child: linters like
|
|
1434
|
+
# checkov spawn worker sub-processes (and git cat-file children) that
|
|
1435
|
+
# would survive the kill and keep the stall alive inside the container.
|
|
1436
|
+
# On POSIX (MegaLinter always runs inside Linux containers), the child is
|
|
1437
|
+
# started in its own process group (start_new_session=True) and the whole
|
|
1438
|
+
# group is killed with SIGKILL when the timeout fires. On other platforms
|
|
1439
|
+
# (local dev/test runs), only the direct child is killed.
|
|
1440
|
+
# Returns (return_code, cleaned stdout); return code 124 (GNU timeout
|
|
1441
|
+
# convention) on timeout, with whatever partial output is available
|
|
1442
|
+
def _run_lint_subprocess(self, command, subprocess_kwargs):
|
|
1443
|
+
timeout_seconds = self.timeout_seconds
|
|
1444
|
+
if timeout_seconds is None:
|
|
1445
|
+
# No timeout configured (default): keep the historical
|
|
1446
|
+
# subprocess.run call, with no behavior change
|
|
1447
|
+
process = subprocess.run(
|
|
1448
|
+
command,
|
|
1449
|
+
stdout=subprocess.PIPE,
|
|
1450
|
+
stderr=subprocess.STDOUT,
|
|
1451
|
+
**subprocess_kwargs,
|
|
1452
|
+
)
|
|
1453
|
+
return process.returncode, utils.clean_string(
|
|
1454
|
+
process.stdout, not self.is_formatter
|
|
1455
|
+
)
|
|
1456
|
+
use_process_group = os.name == "posix"
|
|
1457
|
+
if use_process_group:
|
|
1458
|
+
subprocess_kwargs = {**subprocess_kwargs, "start_new_session": True}
|
|
1459
|
+
process = subprocess.Popen(
|
|
1460
|
+
command,
|
|
1461
|
+
stdout=subprocess.PIPE,
|
|
1462
|
+
stderr=subprocess.STDOUT,
|
|
1463
|
+
**subprocess_kwargs,
|
|
1464
|
+
)
|
|
1465
|
+
try:
|
|
1466
|
+
stdout, _ = process.communicate(timeout=timeout_seconds)
|
|
1467
|
+
return process.returncode, utils.clean_string(stdout, not self.is_formatter)
|
|
1468
|
+
except subprocess.TimeoutExpired:
|
|
1469
|
+
if use_process_group:
|
|
1470
|
+
try:
|
|
1471
|
+
# The process group id is the pid of the group leader:
|
|
1472
|
+
# kill the leader and all its descendants at once.
|
|
1473
|
+
# getattr: signal.SIGKILL is missing on Windows, and this
|
|
1474
|
+
# module must stay importable and testable everywhere
|
|
1475
|
+
os.killpg(process.pid, getattr(signal, "SIGKILL", 9))
|
|
1476
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
1477
|
+
process.kill()
|
|
1478
|
+
else:
|
|
1479
|
+
process.kill()
|
|
1480
|
+
# Collect whatever the linter wrote before being killed. All
|
|
1481
|
+
# writers are dead so the pipe closes quickly; keep a bound anyway
|
|
1482
|
+
try:
|
|
1483
|
+
partial_stdout, _ = process.communicate(timeout=30)
|
|
1484
|
+
except Exception:
|
|
1485
|
+
partial_stdout = None
|
|
1486
|
+
partial_output = (
|
|
1487
|
+
utils.clean_string(partial_stdout, not self.is_formatter)
|
|
1488
|
+
if partial_stdout
|
|
1489
|
+
else ""
|
|
1490
|
+
)
|
|
1491
|
+
timeout_config_var = self.timeout_config_var or "LINTER_TIMEOUT_SECONDS"
|
|
1492
|
+
timeout_message = (
|
|
1493
|
+
f"[{self.linter_name}] Timed out after {timeout_seconds} seconds "
|
|
1494
|
+
f"and was killed ({timeout_config_var}={timeout_seconds}). "
|
|
1495
|
+
"Partial output above if any. "
|
|
1496
|
+
"If this repository legitimately needs more time, increase the "
|
|
1497
|
+
f"value with {self.name}_TIMEOUT_SECONDS or LINTER_TIMEOUT_SECONDS "
|
|
1498
|
+
"(0 disables the timeout); a timeout with very low CPU usage "
|
|
1499
|
+
"usually means an I/O stall (e.g. workspace bind-mounted from "
|
|
1500
|
+
"Windows into a WSL2-backed container engine)."
|
|
1501
|
+
)
|
|
1502
|
+
logging.error(timeout_message)
|
|
1503
|
+
if partial_output.strip() != "":
|
|
1504
|
+
return 124, partial_output + "\n\n" + timeout_message
|
|
1505
|
+
return 124, timeout_message
|
|
1506
|
+
|
|
1507
|
+
def apply_common_linter_errors(self, return_code, return_stdout):
|
|
1508
|
+
if return_code == 0 or not self.common_linter_errors:
|
|
1509
|
+
return return_code, return_stdout
|
|
1510
|
+
for entry in self.common_linter_errors:
|
|
1511
|
+
identifier = entry.get("identifier", "")
|
|
1512
|
+
pattern = entry.get("regex")
|
|
1513
|
+
message = entry.get("message", "")
|
|
1514
|
+
if not pattern or not message:
|
|
1515
|
+
continue
|
|
1516
|
+
if re.search(pattern, return_stdout or ""):
|
|
1517
|
+
block = f"[{identifier}] {message}" if identifier else message
|
|
1518
|
+
return_stdout = (return_stdout or "") + "\n\n" + block
|
|
1519
|
+
logging.warning(f"[{self.linter_name}] {block}")
|
|
1520
|
+
return return_code, return_stdout
|
|
1521
|
+
|
|
1522
|
+
def manage_sarif_output(self, return_stdout):
|
|
1523
|
+
sarif_confirmed = False
|
|
1524
|
+
# Move SARIF file if necessary if generated in a fixed place by the linter
|
|
1525
|
+
if (
|
|
1526
|
+
self.can_output_sarif is True
|
|
1527
|
+
and self.output_sarif is True
|
|
1528
|
+
and self.sarif_output_file is not None
|
|
1529
|
+
and self.sarif_default_output_file is not None
|
|
1530
|
+
and not os.path.isfile(self.sarif_output_file)
|
|
1531
|
+
):
|
|
1532
|
+
linter_sarif_report = (
|
|
1533
|
+
self.sarif_default_output_file
|
|
1534
|
+
if os.path.isfile(self.sarif_default_output_file)
|
|
1535
|
+
else os.path.join(self.workspace, self.sarif_default_output_file)
|
|
1536
|
+
)
|
|
1537
|
+
if not os.path.isfile(linter_sarif_report):
|
|
1538
|
+
linter_sarif_report = os.path.join(
|
|
1539
|
+
self.report_folder, self.sarif_default_output_file
|
|
1540
|
+
)
|
|
1541
|
+
|
|
1542
|
+
# Check that a sarif report really exists before moving it etc)
|
|
1543
|
+
if os.path.isfile(linter_sarif_report):
|
|
1544
|
+
shutil.move(linter_sarif_report, self.sarif_output_file)
|
|
1545
|
+
sarif_confirmed = True
|
|
1546
|
+
logging.debug(
|
|
1547
|
+
f"Moved {linter_sarif_report} to {self.sarif_output_file}"
|
|
1548
|
+
)
|
|
1549
|
+
else:
|
|
1550
|
+
logging.debug(
|
|
1551
|
+
f"Could not find {linter_sarif_report} (linter sarif output error?)"
|
|
1552
|
+
)
|
|
1553
|
+
sarif_confirmed = False
|
|
1554
|
+
# Manage case when SARIF output is in stdout (and not generated by the linter)
|
|
1555
|
+
elif (
|
|
1556
|
+
self.can_output_sarif is True
|
|
1557
|
+
and self.output_sarif is True
|
|
1558
|
+
and not os.path.isfile(self.sarif_output_file)
|
|
1559
|
+
):
|
|
1560
|
+
sarif_stdout = utils.find_json_in_stdout(return_stdout)
|
|
1561
|
+
if sarif_stdout != "":
|
|
1562
|
+
with open(self.sarif_output_file, "w", encoding="utf-8") as file:
|
|
1563
|
+
file.write(sarif_stdout)
|
|
1564
|
+
sarif_confirmed = True
|
|
1565
|
+
else:
|
|
1566
|
+
logging.error(
|
|
1567
|
+
"[Sarif] ERROR: there is no SARIF output file found, and stdout doesn't contain SARIF"
|
|
1568
|
+
)
|
|
1569
|
+
logging.error("[Sarif] stdout: " + return_stdout)
|
|
1570
|
+
elif (
|
|
1571
|
+
self.can_output_sarif is True
|
|
1572
|
+
and self.output_sarif is True
|
|
1573
|
+
and os.path.isfile(self.sarif_output_file)
|
|
1574
|
+
):
|
|
1575
|
+
sarif_confirmed = True
|
|
1576
|
+
|
|
1577
|
+
if sarif_confirmed is True:
|
|
1578
|
+
utils_sarif.normalize_sarif_files(self)
|
|
1579
|
+
|
|
1580
|
+
# Convert SARIF into human readable text for Console & Text reporters
|
|
1581
|
+
if sarif_confirmed is True and self.master.sarif_to_human is True:
|
|
1582
|
+
with open(self.sarif_output_file, "r", encoding="utf-8") as file:
|
|
1583
|
+
sarif_text = utils_sarif.remove_suppressed_results_from_text(
|
|
1584
|
+
file.read()
|
|
1585
|
+
)
|
|
1586
|
+
self.stdout_human = utils_reporter.convert_sarif_to_human(
|
|
1587
|
+
sarif_text, self.request_id
|
|
1588
|
+
)
|
|
1589
|
+
|
|
1590
|
+
# Returns linter version (can be overridden in special cases, like version has special format)
|
|
1591
|
+
def get_linter_version(self):
|
|
1592
|
+
if self.linter_version_cache is not None:
|
|
1593
|
+
return self.linter_version_cache
|
|
1594
|
+
# Use the version collected at Docker build time when available, to
|
|
1595
|
+
# avoid spawning one "--version" process per linter at runtime.
|
|
1596
|
+
# VERSION_GET_AT_RUNTIME=true forces calling the linter executable
|
|
1597
|
+
# (used by test cases and the linters auto-update job)
|
|
1598
|
+
if config.get(self.request_id, "VERSION_GET_AT_RUNTIME", "false") != "true":
|
|
1599
|
+
prebuilt_version = utils.get_prebuilt_linter_version(self.linter_name)
|
|
1600
|
+
if prebuilt_version is not None:
|
|
1601
|
+
self.linter_version_cache = prebuilt_version
|
|
1602
|
+
return self.linter_version_cache
|
|
1603
|
+
version_output = self.get_linter_version_output()
|
|
1604
|
+
reg = self.version_extract_regex
|
|
1605
|
+
if isinstance(reg, str):
|
|
1606
|
+
reg = re.compile(reg)
|
|
1607
|
+
m = reg.search(version_output)
|
|
1608
|
+
if m:
|
|
1609
|
+
self.linter_version_cache = ".".join(m.group().split())
|
|
1610
|
+
elif self.native_toolchain is not None and self.native_toolchain.version:
|
|
1611
|
+
# Native mode: some release builds report no version (phplint: dev-main)
|
|
1612
|
+
self.linter_version_cache = self.native_toolchain.version
|
|
1613
|
+
else:
|
|
1614
|
+
logging.error(
|
|
1615
|
+
f"Unable to extract version with regex {str(reg)} from {version_output}"
|
|
1616
|
+
)
|
|
1617
|
+
self.linter_version_cache = "ERROR"
|
|
1618
|
+
return self.linter_version_cache
|
|
1619
|
+
|
|
1620
|
+
# Returns the version of the associated linter (can be overridden in special cases, like version has special format)
|
|
1621
|
+
def get_linter_version_output(self):
|
|
1622
|
+
command = self.build_version_command()
|
|
1623
|
+
logging.debug("Linter version command: " + str(command))
|
|
1624
|
+
cwd = os.getcwd() if command[0] != "npm" else "~/"
|
|
1625
|
+
subprocess_env = self.native_environment(
|
|
1626
|
+
{
|
|
1627
|
+
**config.build_env(self.request_id, True, self.unsecured_env_variables),
|
|
1628
|
+
"FORCE_COLOR": "0",
|
|
1629
|
+
"NO_COLOR": "true",
|
|
1630
|
+
}
|
|
1631
|
+
)
|
|
1632
|
+
try:
|
|
1633
|
+
process = subprocess.run(
|
|
1634
|
+
command,
|
|
1635
|
+
stdout=subprocess.PIPE,
|
|
1636
|
+
stderr=subprocess.STDOUT,
|
|
1637
|
+
cwd=cwd,
|
|
1638
|
+
env=subprocess_env,
|
|
1639
|
+
)
|
|
1640
|
+
return_code = process.returncode
|
|
1641
|
+
output = utils.clean_string(process.stdout)
|
|
1642
|
+
logging.debug("Linter version result: " + str(return_code) + " " + output)
|
|
1643
|
+
except FileNotFoundError:
|
|
1644
|
+
logging.warning("Unable to call command [" + " ".join(command) + "]")
|
|
1645
|
+
return_code = 666
|
|
1646
|
+
output = "ERROR"
|
|
1647
|
+
|
|
1648
|
+
if return_code != self.version_command_return_code:
|
|
1649
|
+
logging.warning(
|
|
1650
|
+
"Unable to get version for linter [" + self.linter_name + "]"
|
|
1651
|
+
)
|
|
1652
|
+
logging.warning(
|
|
1653
|
+
" ".join(command) + f" returned output: ({str(return_code)}) " + output
|
|
1654
|
+
)
|
|
1655
|
+
return "ERROR"
|
|
1656
|
+
else:
|
|
1657
|
+
return output
|
|
1658
|
+
|
|
1659
|
+
# Returns linter help (can be overridden in special cases, like version has special format)
|
|
1660
|
+
def get_linter_help(self):
|
|
1661
|
+
if self.linter_help_cache is not None:
|
|
1662
|
+
return self.linter_help_cache
|
|
1663
|
+
help_command = self.build_help_command()
|
|
1664
|
+
return_code = 666
|
|
1665
|
+
output = ""
|
|
1666
|
+
command = ""
|
|
1667
|
+
for command in [help_command] + self.cli_help_extra_commands:
|
|
1668
|
+
try:
|
|
1669
|
+
if isinstance(command, str):
|
|
1670
|
+
command = command.split(" ")
|
|
1671
|
+
subprocess_env = self.native_environment(
|
|
1672
|
+
{
|
|
1673
|
+
**config.build_env(
|
|
1674
|
+
self.request_id, True, self.unsecured_env_variables
|
|
1675
|
+
),
|
|
1676
|
+
"FORCE_COLOR": "0",
|
|
1677
|
+
}
|
|
1678
|
+
)
|
|
1679
|
+
if sys.platform == "win32":
|
|
1680
|
+
cli_absolute = shutil.which(
|
|
1681
|
+
command[0], path=self.native_search_path(subprocess_env)
|
|
1682
|
+
)
|
|
1683
|
+
if cli_absolute is not None:
|
|
1684
|
+
command[0] = cli_absolute
|
|
1685
|
+
logging.debug("Linter help command: " + str(command))
|
|
1686
|
+
process = subprocess.run(
|
|
1687
|
+
command,
|
|
1688
|
+
stdout=subprocess.PIPE,
|
|
1689
|
+
stderr=subprocess.STDOUT,
|
|
1690
|
+
env=subprocess_env,
|
|
1691
|
+
)
|
|
1692
|
+
return_code = process.returncode
|
|
1693
|
+
output += utils.clean_string(process.stdout)
|
|
1694
|
+
logging.debug("Linter help result: " + str(return_code) + " " + output)
|
|
1695
|
+
except FileNotFoundError:
|
|
1696
|
+
logging.warning("Unable to call command [" + " ".join(command) + "]")
|
|
1697
|
+
return_code = 666
|
|
1698
|
+
output += "ERROR"
|
|
1699
|
+
break
|
|
1700
|
+
|
|
1701
|
+
if return_code != self.help_command_return_code or output.strip() == "":
|
|
1702
|
+
logging.warning("Unable to get help for linter [" + self.linter_name + "]")
|
|
1703
|
+
logging.warning(f"{str(command)} returned output: ({return_code}) {output}")
|
|
1704
|
+
return "ERROR"
|
|
1705
|
+
else:
|
|
1706
|
+
return output
|
|
1707
|
+
|
|
1708
|
+
# noinspection PyMethodMayBeStatic
|
|
1709
|
+
def get_regex(self, reg):
|
|
1710
|
+
if reg is None:
|
|
1711
|
+
raise Exception("You must define a regex !")
|
|
1712
|
+
if isinstance(reg, str):
|
|
1713
|
+
reg = re.compile(reg)
|
|
1714
|
+
return reg
|
|
1715
|
+
|
|
1716
|
+
########################################
|
|
1717
|
+
# Methods that can be overridden below #
|
|
1718
|
+
########################################
|
|
1719
|
+
|
|
1720
|
+
def before_lint_files(self):
|
|
1721
|
+
pass
|
|
1722
|
+
|
|
1723
|
+
# Build the CLI command to call to lint a file (can be overridden)
|
|
1724
|
+
def build_lint_command(self, file=None) -> list:
|
|
1725
|
+
cmd = [*self.cli_executable]
|
|
1726
|
+
|
|
1727
|
+
additional_replace_variables = {
|
|
1728
|
+
"{{FILE}}": file,
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
# Add other lint cli arguments if defined
|
|
1732
|
+
self.cli_lint_extra_args = self.replace_vars(
|
|
1733
|
+
self.cli_lint_extra_args, additional_replace_variables
|
|
1734
|
+
)
|
|
1735
|
+
cmd += self.cli_lint_extra_args
|
|
1736
|
+
|
|
1737
|
+
# Add fix argument if defined
|
|
1738
|
+
if self.apply_fixes is True and (
|
|
1739
|
+
self.cli_lint_fix_arg_name is not None
|
|
1740
|
+
or len(self.cli_lint_fix_remove_args) > 0
|
|
1741
|
+
or str(self.cli_executable_fix) != str(self.cli_executable)
|
|
1742
|
+
):
|
|
1743
|
+
args_pos = len(self.cli_executable)
|
|
1744
|
+
cmd = cmd[args_pos:] # Remove executable elements
|
|
1745
|
+
cmd = self.cli_executable_fix + cmd
|
|
1746
|
+
if self.cli_lint_fix_arg_name is not None:
|
|
1747
|
+
cmd += [self.cli_lint_fix_arg_name]
|
|
1748
|
+
self.try_fix = True
|
|
1749
|
+
|
|
1750
|
+
# Add user-defined extra arguments if defined
|
|
1751
|
+
self.cli_lint_user_args = self.replace_vars(
|
|
1752
|
+
self.cli_lint_user_args, additional_replace_variables
|
|
1753
|
+
)
|
|
1754
|
+
cmd += self.cli_lint_user_args
|
|
1755
|
+
|
|
1756
|
+
# Add config arguments if defined (except for case when no_config_if_fix is True)
|
|
1757
|
+
if (
|
|
1758
|
+
self.cli_config_arg_name in cmd
|
|
1759
|
+
or self.cli_config_arg_name in self.cli_config_extra_args
|
|
1760
|
+
):
|
|
1761
|
+
# User overridden config within LINTER_NAME_ARGUMENTS
|
|
1762
|
+
cmd += self.cli_config_extra_args
|
|
1763
|
+
elif self.config_file is not None:
|
|
1764
|
+
# Config file
|
|
1765
|
+
self.final_config_file = self.config_file
|
|
1766
|
+
if self.cli_config_arg_name.endswith(
|
|
1767
|
+
"="
|
|
1768
|
+
) or self.cli_config_arg_name.endswith(":"):
|
|
1769
|
+
cmd += [self.cli_config_arg_name + self.final_config_file]
|
|
1770
|
+
elif self.cli_config_arg_name != "":
|
|
1771
|
+
cmd += [self.cli_config_arg_name, self.final_config_file]
|
|
1772
|
+
cmd += self.cli_config_extra_args
|
|
1773
|
+
elif self.cli_config_default_value is not None:
|
|
1774
|
+
# Default config value
|
|
1775
|
+
cmd += [self.cli_config_arg_name, self.cli_config_default_value]
|
|
1776
|
+
cmd += self.cli_config_extra_args
|
|
1777
|
+
|
|
1778
|
+
# Manage ignore arguments if necessary
|
|
1779
|
+
cmd += self.get_ignore_arguments(cmd)
|
|
1780
|
+
|
|
1781
|
+
# Manage SARIF arguments if necessary
|
|
1782
|
+
cmd += self.get_sarif_arguments()
|
|
1783
|
+
|
|
1784
|
+
# Add other lint cli arguments after other arguments if defined
|
|
1785
|
+
self.cli_lint_extra_args_after = self.replace_vars(
|
|
1786
|
+
self.cli_lint_extra_args_after, additional_replace_variables
|
|
1787
|
+
)
|
|
1788
|
+
cmd += self.cli_lint_extra_args_after
|
|
1789
|
+
|
|
1790
|
+
if self.cli_lint_mode == "file":
|
|
1791
|
+
self.cli_lint_mode_file_extra_args_after = self.replace_vars(
|
|
1792
|
+
self.cli_lint_mode_file_extra_args_after, additional_replace_variables
|
|
1793
|
+
)
|
|
1794
|
+
|
|
1795
|
+
cmd += self.cli_lint_mode_file_extra_args_after
|
|
1796
|
+
elif self.cli_lint_mode == "list_of_files":
|
|
1797
|
+
# Arguments introducing the list of files (ex: --file) are added
|
|
1798
|
+
# only when there is at least one file to lint, otherwise they would
|
|
1799
|
+
# be left without value and make the linter command invalid
|
|
1800
|
+
if len(self.files) > 0:
|
|
1801
|
+
self.cli_lint_mode_list_of_files_extra_args_after = self.replace_vars(
|
|
1802
|
+
self.cli_lint_mode_list_of_files_extra_args_after,
|
|
1803
|
+
additional_replace_variables,
|
|
1804
|
+
)
|
|
1805
|
+
|
|
1806
|
+
cmd += self.cli_lint_mode_list_of_files_extra_args_after
|
|
1807
|
+
elif self.cli_lint_mode == "project":
|
|
1808
|
+
# Single gate for every excluded-directories forwarding mechanism:
|
|
1809
|
+
# native CLI arguments, generated ignore files (report folder or
|
|
1810
|
+
# workspace) and generated configurations. Nothing is forwarded
|
|
1811
|
+
# when no excluded directory exists in the workspace
|
|
1812
|
+
forward_exclusions = (
|
|
1813
|
+
self.has_project_exclude_forwarding()
|
|
1814
|
+
and self.is_project_exclude_forwarding_active()
|
|
1815
|
+
and len(self.get_project_exclude_directories()) > 0
|
|
1816
|
+
and not self.has_project_exclude_conflicting_args(cmd)
|
|
1817
|
+
)
|
|
1818
|
+
if forward_exclusions:
|
|
1819
|
+
cmd += self.build_project_exclude_arguments(cmd)
|
|
1820
|
+
cmd = self.manage_excluded_directories_config(cmd)
|
|
1821
|
+
self.log_project_gitignored_directories_forwarding()
|
|
1822
|
+
# The paths to analyze (ex: ".") would override the ones defined in
|
|
1823
|
+
# the linter's own configuration (phpcs <file>, phpstan paths...),
|
|
1824
|
+
# making it analyze folders the user deliberately left out
|
|
1825
|
+
if self.is_project_paths_defined_in_config():
|
|
1826
|
+
self.log_lines_pre += [
|
|
1827
|
+
f"[Project paths] {self.config_file_label} defines the paths to "
|
|
1828
|
+
f"analyze, so {' '.join(self.cli_lint_mode_project_extra_args_after)} "
|
|
1829
|
+
"is not added to the command line"
|
|
1830
|
+
]
|
|
1831
|
+
else:
|
|
1832
|
+
self.cli_lint_mode_project_extra_args_after = self.replace_vars(
|
|
1833
|
+
self.cli_lint_mode_project_extra_args_after,
|
|
1834
|
+
additional_replace_variables,
|
|
1835
|
+
)
|
|
1836
|
+
|
|
1837
|
+
cmd += self.cli_lint_mode_project_extra_args_after
|
|
1838
|
+
# Ignore file arguments come after the positional arguments: array
|
|
1839
|
+
# options (ex: v8r --ignore-pattern-files) greedily consume every
|
|
1840
|
+
# following non-option argument
|
|
1841
|
+
if forward_exclusions:
|
|
1842
|
+
cmd += self.build_project_exclude_ignore_file_arguments(cmd)
|
|
1843
|
+
|
|
1844
|
+
# Some linters/formatters update files by default.
|
|
1845
|
+
# To avoid that, declare -megalinter-fix-flag as cli_lint_fix_arg_name
|
|
1846
|
+
if self.try_fix is True:
|
|
1847
|
+
for arg in self.cli_lint_fix_remove_args:
|
|
1848
|
+
cmd.remove(arg)
|
|
1849
|
+
if "--megalinter-fix-flag" in cmd:
|
|
1850
|
+
cmd.remove("--megalinter-fix-flag")
|
|
1851
|
+
|
|
1852
|
+
# Remove arguments at user request
|
|
1853
|
+
cmd = self.remove_command_args(cmd)
|
|
1854
|
+
|
|
1855
|
+
# Append file in command arguments
|
|
1856
|
+
if file is not None:
|
|
1857
|
+
cmd += [file]
|
|
1858
|
+
|
|
1859
|
+
# If mode is "list of files", append all files as cli arguments
|
|
1860
|
+
elif self.cli_lint_mode == "list_of_files":
|
|
1861
|
+
if self.files_separator is not None:
|
|
1862
|
+
cmd += [self.files_separator.join(self.files)]
|
|
1863
|
+
else:
|
|
1864
|
+
cmd += self.files
|
|
1865
|
+
return cmd
|
|
1866
|
+
|
|
1867
|
+
# Remove arguments listed in <LINTER_NAME>_COMMAND_REMOVE_ARGUMENTS.
|
|
1868
|
+
# Arguments that are not in the command line are just ignored, as they can be
|
|
1869
|
+
# conditionally added (or not) by linter subclasses
|
|
1870
|
+
def remove_command_args(self, cmd: list) -> list:
|
|
1871
|
+
for arg in self.cli_command_remove_args:
|
|
1872
|
+
if arg in cmd:
|
|
1873
|
+
cmd.remove(arg)
|
|
1874
|
+
else:
|
|
1875
|
+
logging.debug(
|
|
1876
|
+
f"[{self.name}] Argument {arg} listed in "
|
|
1877
|
+
f"{self.name}_COMMAND_REMOVE_ARGUMENTS is not in the command line, "
|
|
1878
|
+
"so it has not been removed"
|
|
1879
|
+
)
|
|
1880
|
+
return cmd
|
|
1881
|
+
|
|
1882
|
+
# Manage ignore arguments
|
|
1883
|
+
def get_ignore_arguments(self, cmd):
|
|
1884
|
+
ignore_args = []
|
|
1885
|
+
if (
|
|
1886
|
+
self.ignore_file is not None
|
|
1887
|
+
and self.cli_lint_ignore_arg_name is not None
|
|
1888
|
+
and self.cli_lint_ignore_arg_name not in cmd
|
|
1889
|
+
and self.cli_lint_ignore_arg_name not in self.cli_lint_extra_args_after
|
|
1890
|
+
):
|
|
1891
|
+
self.final_ignore_file = self.ignore_file
|
|
1892
|
+
if self.cli_lint_ignore_arg_name.endswith("="):
|
|
1893
|
+
ignore_args += [self.cli_lint_ignore_arg_name + self.final_ignore_file]
|
|
1894
|
+
elif self.cli_lint_ignore_arg_name != "":
|
|
1895
|
+
ignore_args += [self.cli_lint_ignore_arg_name, self.final_ignore_file]
|
|
1896
|
+
return ignore_args
|
|
1897
|
+
|
|
1898
|
+
# Excluded directories forwarding can be disabled globally with
|
|
1899
|
+
# FORWARD_EXCLUDED_DIRECTORIES or per-linter with
|
|
1900
|
+
# <LINTER_KEY>_FORWARD_EXCLUDED_DIRECTORIES
|
|
1901
|
+
def is_project_exclude_forwarding_active(self):
|
|
1902
|
+
return (
|
|
1903
|
+
config.get(
|
|
1904
|
+
self.request_id,
|
|
1905
|
+
self.name + "_FORWARD_EXCLUDED_DIRECTORIES",
|
|
1906
|
+
config.get(self.request_id, "FORWARD_EXCLUDED_DIRECTORIES", "true"),
|
|
1907
|
+
)
|
|
1908
|
+
== "true"
|
|
1909
|
+
)
|
|
1910
|
+
|
|
1911
|
+
def is_project_paths_defined_in_config(self):
|
|
1912
|
+
if (
|
|
1913
|
+
self.cli_lint_mode_project_config_paths_regex is None
|
|
1914
|
+
or len(self.cli_lint_mode_project_extra_args_after) == 0
|
|
1915
|
+
or self.config_file is None
|
|
1916
|
+
or not os.path.isfile(self.config_file)
|
|
1917
|
+
):
|
|
1918
|
+
return False
|
|
1919
|
+
with open(self.config_file, encoding="utf-8", errors="ignore") as file:
|
|
1920
|
+
config_content = file.read()
|
|
1921
|
+
# A commented-out XML entry (ex: <!-- <file>src</file> -->) defines nothing
|
|
1922
|
+
config_content = re.sub(r"<!--.*?-->", "", config_content, flags=re.DOTALL)
|
|
1923
|
+
return (
|
|
1924
|
+
re.search(self.cli_lint_mode_project_config_paths_regex, config_content)
|
|
1925
|
+
is not None
|
|
1926
|
+
)
|
|
1927
|
+
|
|
1928
|
+
def log_project_gitignored_directories_forwarding(self):
|
|
1929
|
+
gitignored_dirs = self.project_gitignored_directories
|
|
1930
|
+
if len(gitignored_dirs) == 0:
|
|
1931
|
+
return
|
|
1932
|
+
path_only_dirs = sorted(
|
|
1933
|
+
set(gitignored_dirs) - set(self.project_gitignored_name_directories)
|
|
1934
|
+
)
|
|
1935
|
+
path_only_info = (
|
|
1936
|
+
f", {len(path_only_dirs)} of them ({', '.join(path_only_dirs[0:5])}) "
|
|
1937
|
+
"only as workspace-relative paths, as tracked directories share their "
|
|
1938
|
+
"name"
|
|
1939
|
+
if len(path_only_dirs) > 0
|
|
1940
|
+
else ""
|
|
1941
|
+
)
|
|
1942
|
+
self.log_project_exclude_forwarding(
|
|
1943
|
+
f"Forwarded {len(gitignored_dirs)} .gitignored directories "
|
|
1944
|
+
f"({', '.join(gitignored_dirs[0:10])}"
|
|
1945
|
+
f"{',…' if len(gitignored_dirs) > 10 else ''}) with the excluded "
|
|
1946
|
+
f"directories{path_only_info} "
|
|
1947
|
+
"(disable with FORWARD_GITIGNORED_DIRECTORIES: false)"
|
|
1948
|
+
)
|
|
1949
|
+
|
|
1950
|
+
def log_project_exclude_forwarding(self, message):
|
|
1951
|
+
log_line = f"[Excluded directories] {message}"
|
|
1952
|
+
if log_line not in self.log_lines_pre:
|
|
1953
|
+
self.log_lines_pre += [log_line]
|
|
1954
|
+
|
|
1955
|
+
# Directories forwarded to project-mode linters: EXCLUDED_DIRECTORIES +
|
|
1956
|
+
# ADDITIONAL_EXCLUDED_DIRECTORIES + directories identified from
|
|
1957
|
+
# FILTER_REGEX_EXCLUDE (global, descriptor or linter scoped) literal
|
|
1958
|
+
# prefixes. Only entries actually found in the workspace are kept, so
|
|
1959
|
+
# linter arguments and generated ignore/config files stay minimal, and
|
|
1960
|
+
# nothing is forwarded when none exists. The lookup follows MegaLinter's
|
|
1961
|
+
# own file filtering: excluded entries are directory basenames matched at
|
|
1962
|
+
# any nesting level, so infrastructure/cdk.out is found for "cdk.out".
|
|
1963
|
+
# REPORT_OUTPUT_FOLDER is the exception: MegaLinter writes into it while
|
|
1964
|
+
# linters run, so it is always forwarded, even when it does not exist yet
|
|
1965
|
+
# when the command line is built
|
|
1966
|
+
def get_project_exclude_directories(self):
|
|
1967
|
+
if getattr(self, "project_exclude_directories", None) is None:
|
|
1968
|
+
self.compute_project_exclude_directories()
|
|
1969
|
+
return self.project_exclude_directories
|
|
1970
|
+
|
|
1971
|
+
# Workspace-relative paths of every directory matching an excluded entry,
|
|
1972
|
+
# for exclusion arguments requiring a real path instead of a directory
|
|
1973
|
+
# name (ex: pmd --exclude, whose value template holds {{WORKSPACE}})
|
|
1974
|
+
def get_project_exclude_directory_paths(self):
|
|
1975
|
+
if getattr(self, "project_exclude_directory_paths", None) is None:
|
|
1976
|
+
self.compute_project_exclude_directories()
|
|
1977
|
+
return self.project_exclude_directory_paths
|
|
1978
|
+
|
|
1979
|
+
# Entries looked up in the workspace, as a couple: the ones matched at any
|
|
1980
|
+
# nesting level (EXCLUDED_DIRECTORIES semantics) and the ones matched at
|
|
1981
|
+
# the workspace root only. Exposed apart from the lookup itself so the main
|
|
1982
|
+
# process can walk the workspace once for the union of every linter
|
|
1983
|
+
def get_project_exclude_search_entries(self):
|
|
1984
|
+
excluded = set(utils.get_excluded_directories(self.request_id))
|
|
1985
|
+
exclude_regexes = (
|
|
1986
|
+
utils.normalize_regex_filter(
|
|
1987
|
+
config.get(self.request_id, "FILTER_REGEX_EXCLUDE", None)
|
|
1988
|
+
)
|
|
1989
|
+
+ utils.normalize_regex_filter(self.filter_regex_exclude_descriptor)
|
|
1990
|
+
+ utils.normalize_regex_filter(self.filter_regex_exclude_linter)
|
|
1991
|
+
)
|
|
1992
|
+
# A ^-anchored regex excludes root-level directories only: matching its
|
|
1993
|
+
# candidates at any nesting level would exclude files the user's own
|
|
1994
|
+
# regex keeps (ex: "^docs/" must not exclude packages/a/docs)
|
|
1995
|
+
root_only = set()
|
|
1996
|
+
for exclude_regex in exclude_regexes:
|
|
1997
|
+
candidates = utils.extract_dir_candidates_from_regex(exclude_regex)
|
|
1998
|
+
if utils.is_root_anchored_regex(exclude_regex):
|
|
1999
|
+
root_only.update(candidates)
|
|
2000
|
+
else:
|
|
2001
|
+
excluded.update(candidates)
|
|
2002
|
+
return excluded, root_only
|
|
2003
|
+
|
|
2004
|
+
def compute_project_exclude_directories(self):
|
|
2005
|
+
excluded, root_only = self.get_project_exclude_search_entries()
|
|
2006
|
+
found = utils.find_workspace_excluded_directories(
|
|
2007
|
+
self.request_id, self.workspace, excluded
|
|
2008
|
+
)
|
|
2009
|
+
names = set(found.keys())
|
|
2010
|
+
paths = {path for dir_paths in found.values() for path in dir_paths}
|
|
2011
|
+
for candidate in utils.normalize_excluded_directories(
|
|
2012
|
+
self.workspace, root_only
|
|
2013
|
+
):
|
|
2014
|
+
if candidate not in names and os.path.isdir(
|
|
2015
|
+
os.path.join(self.workspace, candidate)
|
|
2016
|
+
):
|
|
2017
|
+
names.add(candidate)
|
|
2018
|
+
paths.add(candidate)
|
|
2019
|
+
# .gitignored directories are located by git itself: they exist, and
|
|
2020
|
+
# their workspace-relative path designates exactly the ignored location
|
|
2021
|
+
names |= set(self.project_gitignored_name_directories)
|
|
2022
|
+
paths |= set(self.project_gitignored_directories)
|
|
2023
|
+
always_forwarded = utils.normalize_excluded_directories(
|
|
2024
|
+
self.workspace, [utils.get_report_output_folder_name(self.request_id)]
|
|
2025
|
+
)
|
|
2026
|
+
names |= always_forwarded
|
|
2027
|
+
paths |= always_forwarded
|
|
2028
|
+
self.project_exclude_directories = sorted(names)
|
|
2029
|
+
self.project_exclude_directory_paths = sorted(paths)
|
|
2030
|
+
|
|
2031
|
+
# True when the linter has a way to receive excluded directories: a
|
|
2032
|
+
# descriptor-declared argument or ignore file, or a custom class hooking
|
|
2033
|
+
# into the forwarding. A plain Linter with neither can not forward
|
|
2034
|
+
# anything, so the workspace lookup is not even computed for it
|
|
2035
|
+
def has_project_exclude_forwarding(self):
|
|
2036
|
+
return (
|
|
2037
|
+
self.cli_lint_mode_project_exclude_arg_name is not None
|
|
2038
|
+
or self.cli_lint_mode_project_exclude_ignore_file_arg_name is not None
|
|
2039
|
+
or type(self) is not Linter
|
|
2040
|
+
)
|
|
2041
|
+
|
|
2042
|
+
# {{DIR}} is a bare directory name, matched by the tool at any nesting
|
|
2043
|
+
# level. A template that can not do that receives workspace-relative paths
|
|
2044
|
+
# instead: {{WORKSPACE}} because it builds an absolute path, {{DIR_PATH}}
|
|
2045
|
+
# because the tool anchors its patterns on the workspace root
|
|
2046
|
+
def project_exclude_template_uses_paths(self, template):
|
|
2047
|
+
return "{{WORKSPACE}}" in template or "{{DIR_PATH}}" in template
|
|
2048
|
+
|
|
2049
|
+
def build_project_exclude_values(self, template):
|
|
2050
|
+
excluded_dirs = (
|
|
2051
|
+
self.get_project_exclude_directory_paths()
|
|
2052
|
+
if self.project_exclude_template_uses_paths(template)
|
|
2053
|
+
else self.get_project_exclude_directories()
|
|
2054
|
+
)
|
|
2055
|
+
return [
|
|
2056
|
+
template.replace("{{DIR_PATH}}", excluded_dir).replace(
|
|
2057
|
+
"{{DIR}}", excluded_dir
|
|
2058
|
+
)
|
|
2059
|
+
for excluded_dir in (
|
|
2060
|
+
excluded_dir.replace("\\", "/") for excluded_dir in excluded_dirs
|
|
2061
|
+
)
|
|
2062
|
+
]
|
|
2063
|
+
|
|
2064
|
+
# A path template emits one value per located directory, so a repository
|
|
2065
|
+
# with thousands of nested node_modules or __pycache__ can build a command
|
|
2066
|
+
# line argument above the Linux MAX_ARG_STRLEN limit (128 KiB for a single
|
|
2067
|
+
# argv entry), which makes execve fail with E2BIG and kills the linter.
|
|
2068
|
+
# Keep the shallowest paths, which cover the most files, and say how many
|
|
2069
|
+
# were dropped: the excess directories are simply analyzed, as before
|
|
2070
|
+
def limit_project_exclude_values(self, values):
|
|
2071
|
+
total = 0
|
|
2072
|
+
kept = []
|
|
2073
|
+
for value in sorted(values, key=lambda value: (value.count("/"), value)):
|
|
2074
|
+
total += len(value.encode("utf-8")) + 1
|
|
2075
|
+
if total > MAX_PROJECT_EXCLUDE_ARG_BYTES:
|
|
2076
|
+
break
|
|
2077
|
+
kept += [value]
|
|
2078
|
+
if len(kept) == len(values):
|
|
2079
|
+
return values
|
|
2080
|
+
self.log_project_exclude_forwarding(
|
|
2081
|
+
f"Only {len(kept)} of the {len(values)} excluded directory paths were "
|
|
2082
|
+
f"sent to {self.linter_name}: the command line argument would have "
|
|
2083
|
+
f"exceeded the {MAX_PROJECT_EXCLUDE_ARG_BYTES} bytes system limit. "
|
|
2084
|
+
f"The deepest ones are analyzed: exclude their parent directory to "
|
|
2085
|
+
f"skip them"
|
|
2086
|
+
)
|
|
2087
|
+
return sorted(kept)
|
|
2088
|
+
|
|
2089
|
+
def read_workspace_file_lines(self, file_name):
|
|
2090
|
+
file_path = os.path.join(self.workspace, file_name)
|
|
2091
|
+
if not os.path.isfile(file_path):
|
|
2092
|
+
return []
|
|
2093
|
+
with open(file_path, encoding="utf-8") as file_handler:
|
|
2094
|
+
return [line.strip() for line in file_handler if line.strip() != ""]
|
|
2095
|
+
|
|
2096
|
+
# Forward excluded directories to project-mode linters through their
|
|
2097
|
+
# native exclusion argument, when the descriptor declares one
|
|
2098
|
+
def build_project_exclude_arguments(self, cmd=None):
|
|
2099
|
+
if self.cli_lint_mode_project_exclude_arg_name is None:
|
|
2100
|
+
return []
|
|
2101
|
+
if len(self.get_project_exclude_directories()) == 0:
|
|
2102
|
+
return []
|
|
2103
|
+
arg_name = self.cli_lint_mode_project_exclude_arg_name
|
|
2104
|
+
# The config values come first: a regex starting with global inline
|
|
2105
|
+
# flags (ex: "(?x)") must stay at the start of the joined value
|
|
2106
|
+
seed_values = self.get_config_list_seed_values(cmd)
|
|
2107
|
+
seed_values += list(self.cli_lint_mode_project_exclude_seed_values)
|
|
2108
|
+
seed_values += self.pop_user_project_exclude_values(cmd)
|
|
2109
|
+
arg_value = self.cli_lint_mode_project_exclude_arg_value
|
|
2110
|
+
generated_values = self.build_project_exclude_values(arg_value)
|
|
2111
|
+
# .gitignored directories can be numerous (ex: one build folder per
|
|
2112
|
+
# package of a monorepo), so they are capped like path templates
|
|
2113
|
+
if (
|
|
2114
|
+
self.project_exclude_template_uses_paths(arg_value)
|
|
2115
|
+
or len(self.project_gitignored_name_directories) > 0
|
|
2116
|
+
):
|
|
2117
|
+
generated_values = self.limit_project_exclude_values(generated_values)
|
|
2118
|
+
values = list(dict.fromkeys(seed_values + generated_values))
|
|
2119
|
+
values = self.replace_vars(values)
|
|
2120
|
+
if len(values) == 0:
|
|
2121
|
+
return []
|
|
2122
|
+
if self.cli_lint_mode_project_exclude_separator is not None:
|
|
2123
|
+
# Single occurrence with joined values (ex: --skip dir1,dir2)
|
|
2124
|
+
values = [self.cli_lint_mode_project_exclude_separator.join(values)]
|
|
2125
|
+
exclude_args = []
|
|
2126
|
+
for value in values:
|
|
2127
|
+
if arg_name == "":
|
|
2128
|
+
# Empty arg name: positional value (ex: ktlint '!**/node_modules/**')
|
|
2129
|
+
exclude_args += [value]
|
|
2130
|
+
elif arg_name.endswith("=") or arg_name.endswith(":"):
|
|
2131
|
+
exclude_args += [arg_name + value]
|
|
2132
|
+
else:
|
|
2133
|
+
exclude_args += [arg_name, value]
|
|
2134
|
+
arg_label = arg_name if arg_name != "" else "negated patterns"
|
|
2135
|
+
self.log_project_exclude_forwarding(
|
|
2136
|
+
f"Forwarded EXCLUDED_DIRECTORIES to {self.linter_name} through {arg_label} "
|
|
2137
|
+
f"(disable with {self.name}_FORWARD_EXCLUDED_DIRECTORIES: false)"
|
|
2138
|
+
)
|
|
2139
|
+
return exclude_args
|
|
2140
|
+
|
|
2141
|
+
# Generic forwarding of excluded directories through an ignore file, driven
|
|
2142
|
+
# by the cli_lint_mode_project_exclude_ignore_file_* descriptor properties.
|
|
2143
|
+
# The generated file always lands in the report folder: MegaLinter never
|
|
2144
|
+
# writes inside the analyzed sources, as a file appearing then disappearing
|
|
2145
|
+
# there crashes the project-mode linters walking the tree at the same time
|
|
2146
|
+
def build_project_exclude_ignore_file_arguments(self, cmd):
|
|
2147
|
+
arg_name = self.cli_lint_mode_project_exclude_ignore_file_arg_name
|
|
2148
|
+
if arg_name is None:
|
|
2149
|
+
return []
|
|
2150
|
+
if len(self.get_project_exclude_directories()) == 0:
|
|
2151
|
+
return []
|
|
2152
|
+
if arg_name in cmd:
|
|
2153
|
+
return []
|
|
2154
|
+
seed_lines = []
|
|
2155
|
+
for seed_file in self.cli_lint_mode_project_exclude_ignore_file_seed_files:
|
|
2156
|
+
# Not deduplicated: in ignore files, the order of a repeated pattern
|
|
2157
|
+
# and a negation decides what is ignored
|
|
2158
|
+
seed_lines += self.read_workspace_file_lines(seed_file)
|
|
2159
|
+
if (
|
|
2160
|
+
len(seed_lines) > 0
|
|
2161
|
+
and not self.cli_lint_mode_project_exclude_ignore_file_merge_all_seeds
|
|
2162
|
+
):
|
|
2163
|
+
break
|
|
2164
|
+
# Ignore files listed in the user config replace the default ones that
|
|
2165
|
+
# the generated file argument would make the tool forget (ex: v8r
|
|
2166
|
+
# ignorePatternFiles)
|
|
2167
|
+
pass_existing = (
|
|
2168
|
+
self.get_config_list_seed_values(cmd)
|
|
2169
|
+
or self.cli_lint_mode_project_exclude_ignore_file_pass_existing
|
|
2170
|
+
)
|
|
2171
|
+
ignore_args = []
|
|
2172
|
+
for existing_file in pass_existing:
|
|
2173
|
+
existing_path = os.path.join(self.workspace, existing_file)
|
|
2174
|
+
if os.path.isfile(existing_path):
|
|
2175
|
+
ignore_args += self.build_ignore_file_argument(arg_name, existing_path)
|
|
2176
|
+
ignore_args += self.build_ignore_file_argument(
|
|
2177
|
+
arg_name,
|
|
2178
|
+
self.build_project_exclude_ignore_file(
|
|
2179
|
+
f"{self.linter_name}-ignore-paths.txt",
|
|
2180
|
+
line_template=self.cli_lint_mode_project_exclude_arg_value,
|
|
2181
|
+
seed_lines=seed_lines,
|
|
2182
|
+
),
|
|
2183
|
+
)
|
|
2184
|
+
return ignore_args
|
|
2185
|
+
|
|
2186
|
+
# Argument names ending with = are concatenated with their value: required
|
|
2187
|
+
# for array options that would greedily consume following positionals
|
|
2188
|
+
# (ex: v8r --ignore-pattern-files=). Paths are passed relative to the
|
|
2189
|
+
# workspace (the linter CWD): some tools silently drop the patterns of an
|
|
2190
|
+
# ignore file referenced by absolute path (ex: v8r)
|
|
2191
|
+
def build_ignore_file_argument(self, arg_name, value):
|
|
2192
|
+
if os.path.isabs(value):
|
|
2193
|
+
try:
|
|
2194
|
+
value = os.path.relpath(value, self.workspace).replace("\\", "/")
|
|
2195
|
+
except ValueError:
|
|
2196
|
+
# Windows: the report folder can be on another drive than the workspace
|
|
2197
|
+
value = value.replace("\\", "/")
|
|
2198
|
+
if arg_name.endswith("=") or arg_name.endswith(":"):
|
|
2199
|
+
return [arg_name + value]
|
|
2200
|
+
return [arg_name, value]
|
|
2201
|
+
|
|
2202
|
+
# Hook for linter classes forwarding excluded directories through a
|
|
2203
|
+
# generated configuration, called in project lint mode when forwarding
|
|
2204
|
+
# is active
|
|
2205
|
+
def manage_excluded_directories_config(self, cmd):
|
|
2206
|
+
return cmd
|
|
2207
|
+
|
|
2208
|
+
def has_project_exclude_conflicting_args(self, cmd):
|
|
2209
|
+
conflicting_args = [
|
|
2210
|
+
arg
|
|
2211
|
+
for arg in self.cli_lint_mode_project_exclude_conflicting_args
|
|
2212
|
+
if arg in cmd
|
|
2213
|
+
]
|
|
2214
|
+
if len(conflicting_args) == 0:
|
|
2215
|
+
return False
|
|
2216
|
+
self.log_project_exclude_forwarding(
|
|
2217
|
+
f"EXCLUDED_DIRECTORIES are not forwarded to {self.linter_name}: "
|
|
2218
|
+
f"{', '.join(conflicting_args)} can not be combined with the forwarding"
|
|
2219
|
+
)
|
|
2220
|
+
return True
|
|
2221
|
+
|
|
2222
|
+
# A single-occurrence exclusion argument (joined values) already set by the
|
|
2223
|
+
# user in ARGUMENTS would be overridden by the generated one, or override
|
|
2224
|
+
# it (phpcs --ignore= keeps the first one): remove it from the command
|
|
2225
|
+
# line and return its value, merged first in the generated occurrence
|
|
2226
|
+
def pop_user_project_exclude_values(self, cmd):
|
|
2227
|
+
arg_name = self.cli_lint_mode_project_exclude_arg_name
|
|
2228
|
+
if cmd is None or self.cli_lint_mode_project_exclude_separator is None:
|
|
2229
|
+
return []
|
|
2230
|
+
if arg_name == "":
|
|
2231
|
+
return []
|
|
2232
|
+
bare_arg_name = arg_name.rstrip("=:")
|
|
2233
|
+
value_prefixes = self.get_cli_argument_value_prefixes(arg_name)
|
|
2234
|
+
values = []
|
|
2235
|
+
index = 0
|
|
2236
|
+
while index < len(cmd):
|
|
2237
|
+
arg = cmd[index]
|
|
2238
|
+
if (
|
|
2239
|
+
arg == bare_arg_name
|
|
2240
|
+
and index + 1 < len(cmd)
|
|
2241
|
+
and not cmd[index + 1].startswith("-")
|
|
2242
|
+
):
|
|
2243
|
+
values += [cmd[index + 1]]
|
|
2244
|
+
del cmd[index : index + 2]
|
|
2245
|
+
elif arg.startswith(value_prefixes):
|
|
2246
|
+
prefix = next(p for p in value_prefixes if arg.startswith(p))
|
|
2247
|
+
values += [arg[len(prefix) :]]
|
|
2248
|
+
del cmd[index]
|
|
2249
|
+
else:
|
|
2250
|
+
index += 1
|
|
2251
|
+
return [value for value in values if value != ""]
|
|
2252
|
+
|
|
2253
|
+
# Config files where the user may define the setting that the exclusion
|
|
2254
|
+
# argument replaces: the one MegaLinter resolved, else the one passed by the
|
|
2255
|
+
# user in ARGUMENTS, then the files the linter discovers itself. Reading a
|
|
2256
|
+
# file the linter does not use can only exclude more, never less
|
|
2257
|
+
def get_project_exclude_config_files(self, cmd=None):
|
|
2258
|
+
config_files = []
|
|
2259
|
+
if self.final_config_file is not None:
|
|
2260
|
+
config_files += [self.final_config_file]
|
|
2261
|
+
elif cmd is not None and self.cli_config_arg_name != "":
|
|
2262
|
+
user_config_file = self.find_cli_argument_value(
|
|
2263
|
+
cmd, self.cli_config_arg_name
|
|
2264
|
+
)
|
|
2265
|
+
if user_config_file is not None:
|
|
2266
|
+
config_files += [user_config_file]
|
|
2267
|
+
config_files += self.cli_lint_mode_project_exclude_config_files
|
|
2268
|
+
config_paths = []
|
|
2269
|
+
for config_file in config_files:
|
|
2270
|
+
config_path = (
|
|
2271
|
+
config_file
|
|
2272
|
+
if os.path.isabs(config_file)
|
|
2273
|
+
else os.path.join(self.workspace, config_file)
|
|
2274
|
+
)
|
|
2275
|
+
if os.path.isfile(config_path) and config_path not in config_paths:
|
|
2276
|
+
config_paths += [config_path]
|
|
2277
|
+
return config_paths
|
|
2278
|
+
|
|
2279
|
+
# Entries of the user config setting that the exclusion argument would
|
|
2280
|
+
# replace, located by the cli_lint_mode_project_exclude_config_key dotted
|
|
2281
|
+
# key path(s): the first config file defining one of them wins
|
|
2282
|
+
def get_config_list_seed_values(self, cmd=None):
|
|
2283
|
+
if self.cli_lint_mode_project_exclude_config_key is None:
|
|
2284
|
+
return []
|
|
2285
|
+
key_paths = self.cli_lint_mode_project_exclude_config_key
|
|
2286
|
+
if isinstance(key_paths, str):
|
|
2287
|
+
key_paths = [key_paths]
|
|
2288
|
+
for config_path in self.get_project_exclude_config_files(cmd):
|
|
2289
|
+
config_content = utils.load_config_file_content(config_path)
|
|
2290
|
+
for key_path in key_paths:
|
|
2291
|
+
values = self.normalize_config_seed_values(
|
|
2292
|
+
utils.find_config_key_value(config_content, key_path.split("."))
|
|
2293
|
+
)
|
|
2294
|
+
if len(values) > 0:
|
|
2295
|
+
return values
|
|
2296
|
+
return []
|
|
2297
|
+
|
|
2298
|
+
def normalize_config_seed_values(self, value):
|
|
2299
|
+
if isinstance(value, list):
|
|
2300
|
+
return [str(item) for item in value if str(item).strip() != ""]
|
|
2301
|
+
if not isinstance(value, str) or value.strip() == "":
|
|
2302
|
+
return []
|
|
2303
|
+
if self.cli_lint_mode_project_exclude_separator == "|":
|
|
2304
|
+
# A regex is kept whole. A multi-line one is a verbose regex (black
|
|
2305
|
+
# compiles it with re.VERBOSE): closed on a new line so that a
|
|
2306
|
+
# trailing comment does not swallow the joined alternatives, and
|
|
2307
|
+
# grouped unless it starts with global inline flags, which must
|
|
2308
|
+
# stay at the start of the joined regex
|
|
2309
|
+
value = value.strip()
|
|
2310
|
+
if "\n" not in value:
|
|
2311
|
+
return [value]
|
|
2312
|
+
if re.match(r"\(\?[aiLmsux]+\)", value):
|
|
2313
|
+
return [value + "\n"]
|
|
2314
|
+
return [f"(?:{value}\n)"]
|
|
2315
|
+
# Comma or newline separated lists (ex: flake8 extend-exclude, pylint ignore)
|
|
2316
|
+
return [item.strip() for item in re.split(r"[,\n]", value) if item.strip()]
|
|
2317
|
+
|
|
2318
|
+
# Value of a CLI argument in a command line, given as "name value",
|
|
2319
|
+
# "name=value", or concatenated when the argument name ends with = or :
|
|
2320
|
+
def find_cli_argument_value(self, cmd, arg_name):
|
|
2321
|
+
bare_arg_name = arg_name.rstrip("=:")
|
|
2322
|
+
value_prefixes = self.get_cli_argument_value_prefixes(arg_name)
|
|
2323
|
+
for index, arg in enumerate(cmd):
|
|
2324
|
+
if arg == bare_arg_name and index + 1 < len(cmd):
|
|
2325
|
+
return cmd[index + 1]
|
|
2326
|
+
if arg.startswith(value_prefixes):
|
|
2327
|
+
prefix = next(p for p in value_prefixes if arg.startswith(p))
|
|
2328
|
+
return arg[len(prefix) :]
|
|
2329
|
+
return None
|
|
2330
|
+
|
|
2331
|
+
# Prefixes of an argument holding its value: "--name=", plus the argument
|
|
2332
|
+
# name itself when it ends with = or : (ex: "--ignore=", "-config:")
|
|
2333
|
+
def get_cli_argument_value_prefixes(self, arg_name):
|
|
2334
|
+
bare_arg_name = arg_name.rstrip("=:")
|
|
2335
|
+
prefixes = [bare_arg_name + "="]
|
|
2336
|
+
if arg_name != bare_arg_name:
|
|
2337
|
+
prefixes += [arg_name]
|
|
2338
|
+
return tuple(dict.fromkeys(prefixes))
|
|
2339
|
+
|
|
2340
|
+
# Locate the value of a CLI argument in a command line
|
|
2341
|
+
def find_cli_argument_value_index(self, cmd, arg_names):
|
|
2342
|
+
for index, arg in enumerate(cmd):
|
|
2343
|
+
if arg in arg_names and index + 1 < len(cmd):
|
|
2344
|
+
return index + 1
|
|
2345
|
+
return None
|
|
2346
|
+
|
|
2347
|
+
def replace_or_append_cli_argument(self, cmd, value_index, arg_name, value):
|
|
2348
|
+
if value_index is not None:
|
|
2349
|
+
cmd[value_index] = value
|
|
2350
|
+
else:
|
|
2351
|
+
cmd += [arg_name, value]
|
|
2352
|
+
return cmd
|
|
2353
|
+
|
|
2354
|
+
def write_report_generated_file(self, file_name, content_lines):
|
|
2355
|
+
file_path = os.path.join(self.report_folder, file_name)
|
|
2356
|
+
os.makedirs(self.report_folder, exist_ok=True)
|
|
2357
|
+
with open(file_path, "w", encoding="utf-8") as file_handler:
|
|
2358
|
+
file_handler.write("\n".join(content_lines) + "\n")
|
|
2359
|
+
return file_path
|
|
2360
|
+
|
|
2361
|
+
# Write a generated ignore file merging seed lines with excluded
|
|
2362
|
+
# directories, for linters whose exclusions can only come from a file
|
|
2363
|
+
def build_project_exclude_ignore_file(
|
|
2364
|
+
self, file_name, line_template="{{DIR}}/", seed_lines=None, seed_file_name=None
|
|
2365
|
+
):
|
|
2366
|
+
lines = list(seed_lines or [])
|
|
2367
|
+
if seed_file_name is not None:
|
|
2368
|
+
lines += self.read_workspace_file_lines(seed_file_name)
|
|
2369
|
+
# Same {{DIR}} / {{DIR_PATH}} contract as build_project_exclude_arguments.
|
|
2370
|
+
# replace_vars is applied to the generated lines only, so that a
|
|
2371
|
+
# {{WORKSPACE}} template resolves without touching the seeded user lines
|
|
2372
|
+
for line in self.replace_vars(self.build_project_exclude_values(line_template)):
|
|
2373
|
+
if line not in lines:
|
|
2374
|
+
lines.append(line)
|
|
2375
|
+
exclude_file = os.path.join(self.report_folder, file_name)
|
|
2376
|
+
os.makedirs(self.report_folder, exist_ok=True)
|
|
2377
|
+
with open(exclude_file, "w", encoding="utf-8") as file_handler:
|
|
2378
|
+
file_handler.write("\n".join(lines) + "\n")
|
|
2379
|
+
self.log_project_exclude_forwarding(
|
|
2380
|
+
f"Generated {exclude_file} to forward EXCLUDED_DIRECTORIES to "
|
|
2381
|
+
f"{self.linter_name} "
|
|
2382
|
+
f"(disable with {self.name}_FORWARD_EXCLUDED_DIRECTORIES: false)"
|
|
2383
|
+
)
|
|
2384
|
+
return exclude_file
|
|
2385
|
+
|
|
2386
|
+
# Manage SARIF arguments
|
|
2387
|
+
def get_sarif_arguments(self):
|
|
2388
|
+
if self.can_output_sarif is True and self.output_sarif is True:
|
|
2389
|
+
self.sarif_output_file = (
|
|
2390
|
+
self.report_folder + os.sep + "sarif" + os.sep + self.name + ".sarif"
|
|
2391
|
+
)
|
|
2392
|
+
os.makedirs(os.path.dirname(self.sarif_output_file), exist_ok=True)
|
|
2393
|
+
self.cli_sarif_args = self.replace_vars(self.cli_sarif_args)
|
|
2394
|
+
return self.cli_sarif_args
|
|
2395
|
+
return []
|
|
2396
|
+
|
|
2397
|
+
# Find number of errors in linter stdout log
|
|
2398
|
+
def get_total_number_errors(self, stdout: str):
|
|
2399
|
+
total_errors = self.get_result_count(
|
|
2400
|
+
stdout, "error", "cli_lint_errors_count", "cli_lint_errors_regex"
|
|
2401
|
+
)
|
|
2402
|
+
|
|
2403
|
+
# Return result if found, else default value according to status
|
|
2404
|
+
if total_errors > 0:
|
|
2405
|
+
return total_errors
|
|
2406
|
+
if self.cli_lint_errors_count is not None and self.output_sarif is False:
|
|
2407
|
+
logging.warning(
|
|
2408
|
+
f"Unable to get number of errors with {self.cli_lint_errors_count} "
|
|
2409
|
+
f"and {str(self.cli_lint_errors_regex)}"
|
|
2410
|
+
)
|
|
2411
|
+
|
|
2412
|
+
# If no regex is defined, return 0 errors if there is a success or 1 error if there are any
|
|
2413
|
+
if self.status == "success":
|
|
2414
|
+
return 0
|
|
2415
|
+
else:
|
|
2416
|
+
return 1
|
|
2417
|
+
|
|
2418
|
+
# Find number of warnings in linter stdout log
|
|
2419
|
+
def get_total_number_warnings(self, stdout: str):
|
|
2420
|
+
total_warnings = self.get_result_count(
|
|
2421
|
+
stdout, "warning", "cli_lint_warnings_count", "cli_lint_warnings_regex"
|
|
2422
|
+
)
|
|
2423
|
+
|
|
2424
|
+
if self.cli_lint_warnings_count is not None and total_warnings is None:
|
|
2425
|
+
logging.warning(
|
|
2426
|
+
f"Unable to get number of warnings with {self.cli_lint_warnings_count} "
|
|
2427
|
+
f"and {str(self.cli_lint_warnings_regex)}"
|
|
2428
|
+
)
|
|
2429
|
+
|
|
2430
|
+
if total_warnings is None:
|
|
2431
|
+
total_warnings = 0
|
|
2432
|
+
|
|
2433
|
+
return total_warnings
|
|
2434
|
+
|
|
2435
|
+
# Find number of results by level in linter stdout log
|
|
2436
|
+
def get_result_count(
|
|
2437
|
+
self, stdout: str, level: str, count_property: str, regex_property: str
|
|
2438
|
+
):
|
|
2439
|
+
total_result = 0
|
|
2440
|
+
|
|
2441
|
+
# Count using SARIF output file
|
|
2442
|
+
if self.output_sarif is True:
|
|
2443
|
+
return self.get_sarif_result_count(stdout, level)
|
|
2444
|
+
# Get number with a single regex. Used when linter prints out Found _ errors/warnings
|
|
2445
|
+
elif getattr(self, count_property) == "regex_number":
|
|
2446
|
+
reg = self.get_regex(getattr(self, regex_property))
|
|
2447
|
+
m = re.search(reg, utils.normalize_log_string(stdout))
|
|
2448
|
+
if m:
|
|
2449
|
+
total_result = int(m.group(1))
|
|
2450
|
+
# Count the number of occurrences of a regex corresponding to
|
|
2451
|
+
# an error or warning in linter log (parses linter log)
|
|
2452
|
+
elif getattr(self, count_property) == "regex_count":
|
|
2453
|
+
reg = self.get_regex(getattr(self, regex_property))
|
|
2454
|
+
total_result = len(re.findall(reg, utils.normalize_log_string(stdout)))
|
|
2455
|
+
# Sum of all numbers found in linter logs with a regex.
|
|
2456
|
+
# Found when each file prints out total number of errors or warnings
|
|
2457
|
+
elif getattr(self, count_property) == "regex_sum":
|
|
2458
|
+
reg = self.get_regex(self.cli_lint_errors_regex)
|
|
2459
|
+
matches = re.findall(reg, utils.normalize_log_string(stdout))
|
|
2460
|
+
total_result = sum(int(m) for m in matches)
|
|
2461
|
+
# Count all lines of the linter log
|
|
2462
|
+
elif getattr(self, count_property) == "total_lines":
|
|
2463
|
+
total_result = sum(
|
|
2464
|
+
not line.isspace() and line != "" for line in stdout.splitlines()
|
|
2465
|
+
)
|
|
2466
|
+
# Count number of results in sarif format
|
|
2467
|
+
elif getattr(self, count_property) == "sarif":
|
|
2468
|
+
sarif = None
|
|
2469
|
+
sarif_stdout = utils.find_json_in_stdout(stdout)
|
|
2470
|
+
try:
|
|
2471
|
+
sarif = json.loads(sarif_stdout)
|
|
2472
|
+
except ValueError as e:
|
|
2473
|
+
logging.warning(f"Unable to parse sarif ({str(e)}):" + stdout)
|
|
2474
|
+
if sarif and sarif["runs"]:
|
|
2475
|
+
for run in sarif["runs"]:
|
|
2476
|
+
for result in run["results"]:
|
|
2477
|
+
if result["level"] == level and not utils_sarif.is_suppressed(
|
|
2478
|
+
result
|
|
2479
|
+
):
|
|
2480
|
+
total_result += 1
|
|
2481
|
+
else:
|
|
2482
|
+
logging.warning("Unable to find results in:" + stdout)
|
|
2483
|
+
return total_result
|
|
2484
|
+
|
|
2485
|
+
def get_sarif_result_count(self, stdout: str, level: str):
|
|
2486
|
+
total_result = 0
|
|
2487
|
+
|
|
2488
|
+
try:
|
|
2489
|
+
# SARIF is in MegaLinter named file
|
|
2490
|
+
if self.sarif_output_file is not None and os.path.isfile(
|
|
2491
|
+
self.sarif_output_file
|
|
2492
|
+
):
|
|
2493
|
+
with open(self.sarif_output_file, "r", encoding="utf-8") as sarif_file:
|
|
2494
|
+
sarif_output = yaml.safe_load(sarif_file)
|
|
2495
|
+
# SARIF is in default output file
|
|
2496
|
+
elif self.sarif_default_output_file is not None and os.path.isfile(
|
|
2497
|
+
self.sarif_default_output_file
|
|
2498
|
+
):
|
|
2499
|
+
with open(
|
|
2500
|
+
self.sarif_default_output_file, "r", encoding="utf-8"
|
|
2501
|
+
) as sarif_file:
|
|
2502
|
+
sarif_output = yaml.safe_load(sarif_file)
|
|
2503
|
+
# SARIF is in stdout
|
|
2504
|
+
else:
|
|
2505
|
+
# SARIF is in stdout
|
|
2506
|
+
sarif_output = yaml.safe_load(stdout)
|
|
2507
|
+
|
|
2508
|
+
for run in sarif_output["runs"]:
|
|
2509
|
+
rule_default_level_map = {}
|
|
2510
|
+
|
|
2511
|
+
if "rules" in run["tool"]["driver"]:
|
|
2512
|
+
for rule in run["tool"]["driver"]["rules"]:
|
|
2513
|
+
if (
|
|
2514
|
+
"defaultConfiguration" in rule
|
|
2515
|
+
and "level" in rule["defaultConfiguration"]
|
|
2516
|
+
):
|
|
2517
|
+
rule_default_level_map[rule["id"]] = rule[
|
|
2518
|
+
"defaultConfiguration"
|
|
2519
|
+
]["level"]
|
|
2520
|
+
|
|
2521
|
+
for result in run["results"]:
|
|
2522
|
+
if utils_sarif.is_suppressed(result):
|
|
2523
|
+
continue
|
|
2524
|
+
count_results = False
|
|
2525
|
+
|
|
2526
|
+
if "level" in result:
|
|
2527
|
+
# Check if the level matches. Otherwise, if the level to be checked is "warning",
|
|
2528
|
+
# counts them as if they were to cover "note" and "none" levels
|
|
2529
|
+
# https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790898
|
|
2530
|
+
if result["level"] == level or (
|
|
2531
|
+
result["level"] not in ["error", "warning"]
|
|
2532
|
+
and level == "warning"
|
|
2533
|
+
):
|
|
2534
|
+
count_results = True
|
|
2535
|
+
elif (
|
|
2536
|
+
"ruleId" in result
|
|
2537
|
+
and result["ruleId"] in rule_default_level_map
|
|
2538
|
+
and rule_default_level_map[result["ruleId"]] == level
|
|
2539
|
+
):
|
|
2540
|
+
count_results = True
|
|
2541
|
+
# If the level property does not exist, the default value is warning
|
|
2542
|
+
# https://json.schemastore.org/sarif-2.1.0.json
|
|
2543
|
+
elif level == "warning":
|
|
2544
|
+
count_results = True
|
|
2545
|
+
|
|
2546
|
+
if count_results is True:
|
|
2547
|
+
total_result += len(result["locations"])
|
|
2548
|
+
|
|
2549
|
+
return total_result
|
|
2550
|
+
except Exception as e:
|
|
2551
|
+
total_result = 1
|
|
2552
|
+
logging.error(
|
|
2553
|
+
f"Error while getting total {level}s from SARIF output.\nError:"
|
|
2554
|
+
+ str(e)
|
|
2555
|
+
+ "\nstdout: "
|
|
2556
|
+
+ stdout
|
|
2557
|
+
)
|
|
2558
|
+
return total_result
|
|
2559
|
+
|
|
2560
|
+
# Build the CLI command to get linter version (can be overridden if --version is not the way to get the version)
|
|
2561
|
+
def build_version_command(self):
|
|
2562
|
+
cmd = [*self.cli_executable_version]
|
|
2563
|
+
cli_absolute = shutil.which(cmd[0], path=self.native_search_path())
|
|
2564
|
+
if cli_absolute is not None:
|
|
2565
|
+
cmd[0] = cli_absolute
|
|
2566
|
+
cmd += self.cli_version_extra_args
|
|
2567
|
+
if self.cli_version_arg_name != "":
|
|
2568
|
+
cmd += [self.cli_version_arg_name]
|
|
2569
|
+
return cmd
|
|
2570
|
+
|
|
2571
|
+
# Build the CLI command to get linter version (can be overridden if --version is not the way to get the version)
|
|
2572
|
+
def build_help_command(self):
|
|
2573
|
+
cmd = [*self.cli_executable_help]
|
|
2574
|
+
cmd += self.cli_help_extra_args
|
|
2575
|
+
cmd += [self.cli_help_arg_name]
|
|
2576
|
+
return cmd
|
|
2577
|
+
|
|
2578
|
+
# Provide additional details in text reporter logs
|
|
2579
|
+
# noinspection PyMethodMayBeStatic
|
|
2580
|
+
def complete_text_reporter_report(self, _reporter_self):
|
|
2581
|
+
return []
|
|
2582
|
+
|
|
2583
|
+
def pre_test(self, test_name):
|
|
2584
|
+
pass
|
|
2585
|
+
|
|
2586
|
+
def post_test(self, test_name):
|
|
2587
|
+
pass
|