continuous-delivery-scripts 3.3.0__tar.gz → 3.4.1__tar.gz

{continuous_delivery_scripts-3.3.0/continuous_delivery_scripts.egg-info → continuous_delivery_scripts-3.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: continuous-delivery-scripts
-Version: 3.3.0
+Version: 3.4.1
 Summary: Continuous Delivery scripts to increase automation
 Home-page: https://github.com/ARMmbed/continuous-delivery-scripts
 Author: CMSIS team
@@ -29,15 +29,15 @@ Requires-Dist: semver~=2.13.0
 Requires-Dist: python-dotenv
 Requires-Dist: twine
 Requires-Dist: boto3
+Requires-Dist: detect-secrets[gibberish]==1.5.0
 Requires-Dist: packaging
 Requires-Dist: licenseheaders<0.8.9
 Requires-Dist: spdx-tools==0.6.1
 Requires-Dist: license-expression
 Requires-Dist: wcmatch
 Requires-Dist: jellyfish
-Requires-Dist: jinja2==2.11.3
+Requires-Dist: jinja2<4,>=3
 Requires-Dist: dataclasses; python_version < "3.7"
-Requires-Dist: markupsafe==2.0.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/_version.py

@@ -11,8 +11,8 @@ References
 This file is autogenerated, do not modify by hand.
 """
 
-__version__ = "3.3.0"
-COMMIT = "0bb13bc2abd9af0d72e709afd285d977cc06d8ba"
+__version__ = "3.4.1"
+COMMIT = "3cb7eefdafb8306ca4750f371325574e1a2ec9c7"
 MAJOR = 3
-MINOR = 3
-PATCH = 0
+MINOR = 4
+PATCH = 1

continuous_delivery_scripts-3.4.1/continuous_delivery_scripts/detect_secrets.py

@@ -0,0 +1,93 @@
+#
+# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Check tracked files against the project's recorded secret registry."""
+
+import argparse
+import logging
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from continuous_delivery_scripts.update_secrets_registry import (
+    _determine_exclude_files,
+    _get_secrets_baseline_file,
+    _get_secrets_baseline_filename,
+)
+from continuous_delivery_scripts.utils.git_helpers import ProjectGitWrapper
+from continuous_delivery_scripts.utils.logging import log_exception, set_log_level
+
+logger = logging.getLogger(__name__)
+
+
+def _generate_detect_secrets_hook_command_list(
+    baseline_file: Path, exclude_files: List[str], tracked_files: List[str]
+) -> List[str]:
+    command = ["detect-secrets-hook", "--baseline", str(baseline_file)]
+    for exclude_file in exclude_files:
+        command.extend(["--exclude-files", exclude_file])
+    command.extend(tracked_files)
+    return command
+
+
+def _filter_tracked_files(tracked_files: List[str], project_root: Path, registry_file: Path) -> List[str]:
+    """Remove the registry file from tracked files to avoid scanning recorded accepted findings."""
+    try:
+        registry_relative_path = registry_file.relative_to(project_root).as_posix()
+    except ValueError:
+        return tracked_files
+    return [path for path in tracked_files if path != registry_relative_path]
+
+
+def detect_secrets(baseline_file: Optional[Path] = None) -> None:
+    """Check tracked files so new secrets are not introduced."""
+    git = ProjectGitWrapper()
+    project_root = Path(str(git.root))
+    resolved_baseline = _get_secrets_baseline_file(baseline_file)
+    tracked_files = _filter_tracked_files(git.list_tracked_files(), project_root, resolved_baseline)
+    if not tracked_files:
+        logger.info("No tracked files found for detect-secrets.")
+        return
+    subprocess.check_call(
+        _generate_detect_secrets_hook_command_list(resolved_baseline, _determine_exclude_files(), tracked_files),
+        cwd=str(project_root),
+    )
+
+
+def main() -> int:
+    """Script CLI."""
+    parser = argparse.ArgumentParser(
+        description=(
+            "Check tracked files against the recorded secret registry so new secrets are not committed. "
+            "This uses Yelp detect-secrets (https://github.com/Yelp/detect-secrets)."
+        )
+    )
+    parser.add_argument(
+        "-r",
+        "--registry-file",
+        default=Path(_get_secrets_baseline_filename()),
+        help="Secret registry file to use.",
+        type=Path,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Verbosity, by default errors are reported.",
+    )
+    args = parser.parse_args()
+    set_log_level(args.verbose)
+
+    try:
+        detect_secrets(args.registry_file)
+        return 0
+    except Exception as e:
+        log_exception(logger, e)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/plugins/golang.py

@@ -203,6 +203,15 @@ class Go(BaseLanguage):
         """States whether project metadata can be retrieved."""
         return False
 
+    def get_secret_registry_exclude_files(self) -> List[str]:
+        """Gets additional detect-secrets exclude patterns for Go projects."""
+        return [
+            r".*go\.sum$",
+            r"^\.circleci[\\/].*",
+            r"^workflows/.*",
+            r"^\.github[\\/]workflows[\\/].*",
+        ]
+
     def get_current_spdx_project(self) -> Optional["SpdxProject"]:
         """Gets current SPDX description."""
         # TODO

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/plugins/python.py

@@ -154,7 +154,7 @@ class Python(BaseLanguage):
 
     def get_related_language(self) -> str:
         """Gets related language."""
-        return get_language_from_file_name(__file__)
+        return str(get_language_from_file_name(__file__))
 
     def package_software(self, mode: CommitType, version: str) -> None:
         """Packages the software into a wheel."""
@@ -189,6 +189,19 @@ class Python(BaseLanguage):
         # (SetuptoolsDeprecationWarning: License classifiers are deprecated)
         return False
 
+    def get_secret_registry_exclude_files(self) -> List[str]:
+        """Gets additional detect-secrets exclude patterns for Python projects."""
+        return [
+            r".*Pipfile\.lock$",
+            r".*poetry\.lock$",
+            r".*uv\.lock$",
+            r".*pdm\.lock$",
+            r".*version\.py$",
+            r"^\.circleci[\\/].*",
+            r"^workflows/.*",
+            r"^\.github[\\/]workflows[\\/].*",
+        ]
+
     def should_include_spdx_in_package(self) -> bool:
         """States whether the SPDX documents should be included in the package."""
         # FIXME Comment out SPDX package as no longer working

continuous_delivery_scripts-3.4.1/continuous_delivery_scripts/update_secrets_registry.py

@@ -0,0 +1,105 @@
+#
+# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Record the project's accepted secret registry using detect-secrets."""
+
+import argparse
+import logging
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from continuous_delivery_scripts.language_specifics import get_language_specifics
+from continuous_delivery_scripts.utils.configuration import (
+    configuration,
+    ConfigurationVariable,
+)
+from continuous_delivery_scripts.utils.logging import log_exception, set_log_level
+
+logger = logging.getLogger(__name__)
+
+GENERIC_SECRET_REGISTRY_EXCLUDE_FILES = [
+    r".*\.html$",
+    r".*\.properties$",
+    r".*ci\.yml$",
+    r"^\.git[\\/]",
+]
+
+
+def _get_secrets_baseline_filename() -> str:
+    return str(configuration.get_value(ConfigurationVariable.SECRETS_BASELINE_FILENAME))
+
+
+def _get_secrets_baseline_file(output_file: Optional[Path] = None) -> Path:
+    project_root = Path(str(configuration.get_value(ConfigurationVariable.PROJECT_ROOT)))
+    baseline_file = output_file or Path(_get_secrets_baseline_filename())
+    return baseline_file if baseline_file.is_absolute() else project_root.joinpath(baseline_file)
+
+
+def _generate_detect_secrets_command_list(exclude_files: List[str]) -> List[str]:
+    command = ["detect-secrets", "scan", "--all-files"]
+    for exclude_file in exclude_files:
+        command.extend(["--exclude-files", exclude_file])
+    return command
+
+
+def _determine_exclude_files() -> List[str]:
+    exclude_files = list(GENERIC_SECRET_REGISTRY_EXCLUDE_FILES)
+    baseline_file_pattern = re.escape(_get_secrets_baseline_filename()).replace(r"\\", r"[\\/]")
+    exclude_files.append(rf"^{baseline_file_pattern}$")
+    exclude_files.extend(get_language_specifics().get_secret_registry_exclude_files())
+    return list(dict.fromkeys(exclude_files))
+
+
+def update_secrets_registry(output_file: Optional[Path] = None) -> Path:
+    """Record the project's accepted secret registry for detect-secrets."""
+    project_root = Path(str(configuration.get_value(ConfigurationVariable.PROJECT_ROOT)))
+    baseline_file = _get_secrets_baseline_file(output_file)
+    logger.info("Updating secrets baseline at [%s].", baseline_file)
+    baseline_contents = subprocess.check_output(
+        _generate_detect_secrets_command_list(_determine_exclude_files()),
+        cwd=str(project_root),
+        encoding="utf8",
+    )
+    baseline_file.write_text(baseline_contents, encoding="utf8")
+    return baseline_file
+
+
+def main() -> int:
+    """Script CLI."""
+    parser = argparse.ArgumentParser(
+        description=(
+            "Record the project's accepted secret registry so allowed findings are not flagged again. "
+            "This uses Yelp detect-secrets (https://github.com/Yelp/detect-secrets)."
+        )
+    )
+    parser.add_argument(
+        "-r",
+        "--registry-file",
+        default=_get_secrets_baseline_filename(),
+        help="Secret registry file to generate.",
+        type=Path,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Verbosity, by default errors are reported.",
+    )
+    args = parser.parse_args()
+    set_log_level(args.verbose)
+
+    try:
+        update_secrets_registry(args.registry_file)
+        return 0
+    except Exception as e:
+        log_exception(logger, e)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/utils/configuration.py

@@ -91,6 +91,8 @@ class ConfigurationVariable(enum.Enum):
     """States whether the release should be tagged with `latest`"""
     TAG_VERSION_SHORTCUTS = 36
     """States whether the release should be tagged with shortcuts i.e. major, major+minor"""
+    SECRETS_BASELINE_FILENAME = 37
+    """Filename for the detect-secrets baseline."""
 
     @staticmethod
     def choices() -> List[str]:
@@ -196,10 +198,20 @@ class StaticConfig(GenericConfig):
     AUTOGENERATE_NEWS_FILE_ON_DEPENDENCY_UPDATE = True
     TAG_LATEST = False
     TAG_VERSION_SHORTCUTS = False
+    SECRETS_BASELINE_FILENAME = ".secrets.baseline"
     DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}"
     DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix
     DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P<DEPENDENCY>.+)"
-    ACCEPTED_THIRD_PARTY_LICENCES = ["Apache-2.0", "BSD*", "JSON", "MIT", "Python-2.0", "PSF-2.0", "MPL-2.0"]
+    ACCEPTED_THIRD_PARTY_LICENCES = [
+        "Apache-2.0",
+        "BSD*",
+        "CC-BY-*",
+        "JSON",
+        "MIT",
+        "Python-2.0",
+        "PSF-2.0",
+        "MPL-2.0",
+    ]
     PACKAGES_WITH_CHECKED_LICENCE: List[str] = []
 
     def _fetch_value(self, key: str) -> Any:
@@ -277,7 +289,7 @@ class FileConfig(GenericConfig):
     @staticmethod
     def _look_for_config_file_walking_up_tree() -> Optional[str]:
         try:
-            return find_file_in_tree(FileConfig.CONFIG_FILE_NAME, top=True)
+            return str(find_file_in_tree(FileConfig.CONFIG_FILE_NAME, top=True))
         except FileNotFoundError as e:
             logger.warning(e)
         return None
@@ -287,7 +299,7 @@ class FileConfig(GenericConfig):
         if file_path and os.path.exists(file_path):
             return file_path
         try:
-            return find_file_in_tree(FileConfig.CONFIG_FILE_NAME)
+            return str(find_file_in_tree(FileConfig.CONFIG_FILE_NAME))
         except FileNotFoundError:
             return FileConfig._look_for_config_file_walking_up_tree()
 

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/utils/git_helpers.py

@@ -61,7 +61,9 @@ class GitWrapper:
         """
         try:
             git_clone = self.repo.clone_from(
-                url=self.get_remote_url(), to_path=str(path), multi_options=["--recurse-submodules"]
+                url=self.get_remote_url(),
+                to_path=str(path),
+                multi_options=["--recurse-submodules"],
             )
         except GitCommandError as e:
             logger.info("failed cloning repository: %s" % e)
@@ -194,7 +196,13 @@ class GitWrapper:
 
     def fetch(self) -> None:
         """Fetches latest changes."""
-        self.repo.git.fetch(all=True, tags=True, force=True)
+        try:
+            self.repo.git.fetch(all=True, tags=True, force=True)
+        except GitCommandError as e:
+            logger.info("failed fetching repository: %s" % e)
+            logger.info("Retrying a different way")
+            self.set_remote_url(self._git_url_ssh_to_https(self.get_remote_url()))
+            self.repo.git.fetch(all=True, tags=True, force=True)
 
     def get_branch(self, branch_name: str) -> Any:
         """Gets a specific local branch.
@@ -235,6 +243,12 @@ class GitWrapper:
             current_branch = self._get_branch_from_abbreviation("HEAD")
         return current_branch
 
+    def list_tracked_files(self) -> List[str]:
+        """List git-tracked files relative to the repository root."""
+        tracked_files = self.repo.git.ls_files("-z")
+        tracked_files = tracked_files if isinstance(tracked_files, str) else tracked_files.decode("utf-8")
+        return [path for path in tracked_files.split("\0") if path]
+
     def _get_branch_from_advanced_feature(self) -> Any:
         if version.parse(self.git_version()) >= version.parse("2.22"):
             current_branch = self.repo.git.branch(show_current=True)
@@ -298,7 +312,10 @@ class GitWrapper:
         current_branch = self.get_current_branch()
         merge_base = self.repo.merge_base(branch, current_branch)
         self.repo.index.merge_tree(current_branch, base=merge_base)
-        self.commit(f"Merge from {str(branch)}", parent_commits=(branch.commit, current_branch.commit))
+        self.commit(
+            f"Merge from {str(branch)}",
+            parent_commits=(branch.commit, current_branch.commit),
+        )
 
     def get_remote_url(self) -> str:
         """Gets the URL of the remote repository.
@@ -416,7 +433,11 @@ class GitWrapper:
         return changes
 
     def get_changes_list(
-        self, commit1: Any, commit2: Any, change_type: Optional[str] = None, dir: Optional[str] = None
+        self,
+        commit1: Any,
+        commit2: Any,
+        change_type: Optional[str] = None,
+        dir: Optional[str] = None,
     ) -> List[str]:
         """Gets change list.
 
@@ -459,7 +480,12 @@ class GitWrapper:
         Pushes changes to the remote repository.
         Pushes also relevant annotated tags when pushing branches out.
         """
-        self.repo.git.push("--follow-tags", "--set-upstream", self._get_remote(), self.get_current_branch())
+        self.repo.git.push(
+            "--follow-tags",
+            "--set-upstream",
+            self._get_remote(),
+            self.get_current_branch(),
+        )
 
     def push_tag(self) -> None:
         """Pushes commits and tags.
@@ -797,4 +823,7 @@ class ProjectTempClone(GitTempClone):
             system will try to identify the current branch in the repository which
             will work in most cases but probably not on CI.
         """
-        super().__init__(desired_branch_name=desired_branch_name, repository_to_clone=ProjectGitWrapper())
+        super().__init__(
+            desired_branch_name=desired_branch_name,
+            repository_to_clone=ProjectGitWrapper(),
+        )

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts/utils/language_specifics_base.py

@@ -7,7 +7,7 @@
 import logging
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Optional, Dict
+from typing import TYPE_CHECKING, Optional, Dict, List
 
 from continuous_delivery_scripts.utils.configuration import (
     configuration,
@@ -72,6 +72,10 @@ class BaseLanguage(ABC):
         """States whether project metadata can be retrieved."""
         return False
 
+    def get_secret_registry_exclude_files(self) -> List[str]:
+        """Gets additional detect-secrets exclude patterns for this plugin."""
+        return list()
+
     def should_include_spdx_in_package(self) -> bool:
         """States whether the SPDX documents should be included in the package."""
         return False

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1/continuous_delivery_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: continuous-delivery-scripts
-Version: 3.3.0
+Version: 3.4.1
 Summary: Continuous Delivery scripts to increase automation
 Home-page: https://github.com/ARMmbed/continuous-delivery-scripts
 Author: CMSIS team
@@ -29,15 +29,15 @@ Requires-Dist: semver~=2.13.0
 Requires-Dist: python-dotenv
 Requires-Dist: twine
 Requires-Dist: boto3
+Requires-Dist: detect-secrets[gibberish]==1.5.0
 Requires-Dist: packaging
 Requires-Dist: licenseheaders<0.8.9
 Requires-Dist: spdx-tools==0.6.1
 Requires-Dist: license-expression
 Requires-Dist: wcmatch
 Requires-Dist: jellyfish
-Requires-Dist: jinja2==2.11.3
+Requires-Dist: jinja2<4,>=3
 Requires-Dist: dataclasses; python_version < "3.7"
-Requires-Dist: markupsafe==2.0.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ continuous_delivery_scripts/__init__.py
 continuous_delivery_scripts/_version.py
 continuous_delivery_scripts/assert_news.py
 continuous_delivery_scripts/create_news_file.py
+continuous_delivery_scripts/detect_secrets.py
 continuous_delivery_scripts/generate_docs.py
 continuous_delivery_scripts/generate_news.py
 continuous_delivery_scripts/get_config.py
@@ -16,6 +17,7 @@ continuous_delivery_scripts/language_specifics.py
 continuous_delivery_scripts/license_files.py
 continuous_delivery_scripts/report_third_party_ip.py
 continuous_delivery_scripts/tag_and_release.py
+continuous_delivery_scripts/update_secrets_registry.py
 continuous_delivery_scripts.egg-info/PKG-INFO
 continuous_delivery_scripts.egg-info/SOURCES.txt
 continuous_delivery_scripts.egg-info/dependency_links.txt

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts.egg-info/entry_points.txt

@@ -1,10 +1,12 @@
 [console_scripts]
 cd-assert-news = continuous_delivery_scripts.assert_news:main
 cd-create-news-file = continuous_delivery_scripts.create_news_file:main
+cd-detect-secrets = continuous_delivery_scripts.detect_secrets:main
 cd-determine-version = continuous_delivery_scripts.get_version:main
 cd-generate-docs = continuous_delivery_scripts.generate_docs:main
 cd-generate-news = continuous_delivery_scripts.generate_news:main
 cd-generate-spdx = continuous_delivery_scripts.report_third_party_ip:main
 cd-get-config = continuous_delivery_scripts.get_config:main
 cd-license-files = continuous_delivery_scripts.license_files:main
+cd-record-secrets = continuous_delivery_scripts.update_secrets_registry:main
 cd-tag-and-release = continuous_delivery_scripts.tag_and_release:main

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/continuous_delivery_scripts.egg-info/requires.txt

@@ -8,14 +8,14 @@ semver~=2.13.0
 python-dotenv
 twine
 boto3
+detect-secrets[gibberish]==1.5.0
 packaging
 licenseheaders<0.8.9
 spdx-tools==0.6.1
 license-expression
 wcmatch
 jellyfish
-jinja2==2.11.3
-markupsafe==2.0.1
+jinja2<4,>=3
 
 [:python_version < "3.7"]
 dataclasses

{continuous_delivery_scripts-3.3.0 → continuous_delivery_scripts-3.4.1}/setup.py

@@ -48,6 +48,8 @@ setup(
             f"cd-get-config={SOURCE_DIR}.get_config:main",
             f"cd-license-files={SOURCE_DIR}.license_files:main",
             f"cd-generate-spdx={SOURCE_DIR}.report_third_party_ip:main",
+            f"cd-detect-secrets={SOURCE_DIR}.detect_secrets:main",
+            f"cd-record-secrets={SOURCE_DIR}.update_secrets_registry:main",
         ]
     },
     keywords="Arm Tools CI CD Continuous Delivery Scripts Automation",
@@ -65,16 +67,15 @@ setup(
         "python-dotenv",
         "twine",
         "boto3",
+        "detect-secrets[gibberish]==1.5.0",
         "packaging",
         "licenseheaders<0.8.9",
         "spdx-tools==0.6.1",
         "license-expression",
         "wcmatch",
         "jellyfish",
-        "jinja2==2.11.3",
+        "jinja2>=3,<4",
         "dataclasses; python_version<'3.7'",
-        # FIXME fixing markupsafe to solve https://github.com/pallets/markupsafe/issues/284 until jinja is upgraded
-        "markupsafe==2.0.1",
     ],
     license="Apache 2.0",
     long_description_content_type="text/markdown",