guarddog 2.7.1__py3-none-any.whl → 2.8.4__py3-none-any.whl

guarddog/analyzer/metadata/__init__.py

@@ -3,6 +3,7 @@ from guarddog.analyzer.metadata.npm import NPM_METADATA_RULES
 from guarddog.analyzer.metadata.pypi import PYPI_METADATA_RULES
 from guarddog.analyzer.metadata.go import GO_METADATA_RULES
 from guarddog.analyzer.metadata.github_action import GITHUB_ACTION_METADATA_RULES
+from guarddog.analyzer.metadata.rubygems import RUBYGEMS_METADATA_RULES
 from guarddog.ecosystems import ECOSYSTEM
 
 
@@ -18,3 +19,5 @@ def get_metadata_detectors(ecosystem: ECOSYSTEM) -> dict[str, Detector]:
             return GITHUB_ACTION_METADATA_RULES
         case ECOSYSTEM.EXTENSION:
             return {}  # No metadata detectors for extensions currently
+        case ECOSYSTEM.RUBYGEMS:
+            return RUBYGEMS_METADATA_RULES

guarddog/analyzer/metadata/pypi/repository_integrity_mismatch.py

@@ -4,14 +4,12 @@ Detects if a package contains an empty description
 """
 
 import configparser
-import hashlib
 import logging
 import os
 import re
 import requests
 from typing import Optional, Tuple
 
-import pygit2  # type: ignore
 import urllib3.util
 
 from guarddog.analyzer.metadata.repository_integrity_mismatch import IntegrityMismatch
@@ -90,18 +88,6 @@ def dict_generator(indict, pre=None):
         yield pre + [indict]
 
 
-def get_file_hash(path):
-    with open(path, "rb") as f:
-        # Read the contents of the file
-        file_contents = f.read()
-        # Create a hash object
-        hash_object = hashlib.sha256()
-        # Feed the file contents to the hash object
-        hash_object.update(file_contents)
-        # Get the hexadecimal hash value
-        return hash_object.hexdigest(), str(file_contents).strip().splitlines()
-
-
 def _ensure_proper_url(url):
     parsed = urllib3.util.parse_url(url)
     if parsed.scheme is None:
@@ -140,80 +126,6 @@ def find_github_candidates(package_info) -> Tuple[set[str], Optional[str]]:
     return github_urls, best
 
 
-EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
-
-
-def exclude_result(file_name, repo_root, pkg_root):
-    """
-    This method filters out some results that are known false positives:
-    * if the file is a documentation file (based on its extension)
-    * if the file is a setup.cfg file with the egg_info claim present on Pypi and not on GitHub
-    """
-    for extension in EXCLUDED_EXTENSIONS:
-        if file_name.endswith(extension):
-            return True
-    if file_name.endswith("setup.cfg"):
-        repo_cfg = configparser.ConfigParser()
-        repo_cfg.read(os.path.join(repo_root, file_name))
-        pkg_cfg = configparser.ConfigParser()
-        pkg_cfg.read(os.path.join(pkg_root, file_name))
-        repo_sections = list(repo_cfg.keys())
-        pkg_sections = list(pkg_cfg.keys())
-        if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
-            return True
-    return False
-
-
-def find_mismatch_for_tag(repo, tag, base_path, repo_path):
-    repo.checkout(tag)
-    mismatch = []
-    for root, dirs, files in os.walk(base_path):
-        relative_path = os.path.relpath(root, base_path)
-        repo_root = os.path.join(repo_path, relative_path)
-        if not os.path.exists(repo_root):
-            continue
-        repo_files = list(
-            filter(
-                lambda x: os.path.isfile(os.path.join(repo_root, x)),
-                os.listdir(repo_root),
-            )
-        )
-        for file_name in repo_files:
-            if file_name not in files:  # ignore files we don't have in the distribution
-                continue
-            repo_hash, repo_content = get_file_hash(os.path.join(repo_root, file_name))
-            pkg_hash, pkg_content = get_file_hash(os.path.join(root, file_name))
-            if repo_hash != pkg_hash:
-                if exclude_result(file_name, repo_root, root):
-                    continue
-                res = {
-                    "file": os.path.join(relative_path, file_name),
-                    "repo_sha256": repo_hash,
-                    "pkg_sha256": pkg_hash,
-                }
-                mismatch.append(res)
-    return mismatch
-
-
-def find_suitable_tags_in_list(tags, version):
-    tag_candidates = []
-    for tag_name in tags:
-        if tag_name.endswith(version):
-            tag_candidates.append(tag_name)
-    return tag_candidates
-
-
-def find_suitable_tags(repo, version):
-    tags_regex = re.compile("^refs/tags/(.*)")
-    tags = []
-    for ref in repo.references:
-        match = tags_regex.match(ref)
-        if match is not None:
-            tags.append(match.group(0))
-
-    return find_suitable_tags_in_list(tags, version)
-
-
 # Note: we should have the GitHub related logic factored out as we will need it when we check for signed commits
 class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
@@ -228,94 +140,71 @@ class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
 
     RULE_NAME = "repository_integrity_mismatch"
+    EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
 
-    def detect(
-        self,
-        package_info,
-        path: Optional[str] = None,
-        name: Optional[str] = None,
-        version: Optional[str] = None,
-    ) -> tuple[bool, str]:
-        if name is None:
-            raise Exception("Detector needs the name of the package")
-        if path is None:
-            raise Exception("Detector needs the path of the package")
-
-        log.debug(
-            f"Running repository integrity mismatch heuristic on PyPI package {name} version {version}"
-        )
-        # let's extract a source repository (GitHub only for now) if we can
+    def extract_github_url(self, package_info, name: str) -> Optional[str]:
+        """Extract GitHub URL from PyPI metadata."""
         github_urls, best_github_candidate = find_github_candidates(package_info)
         if len(github_urls) == 0:
-            return False, "Could not find any GitHub url in the project's description"
-        # now, let's find the right url
+            return None
 
         github_url = find_best_github_candidate(
             (github_urls, best_github_candidate), name
         )
+        return github_url
 
-        if github_url is None:
-            return (
-                False,
-                "Could not find a good GitHub url in the project's description",
-            )
-
-        log.debug(f"Using GitHub URL {github_url}")
-        # ok, now let's try to find the version! (I need to know which version we are scanning)
-        if version is None:
-            version = package_info["info"]["version"]
-        if version is None:
-            raise Exception("Could not find suitable version to scan")
-        tmp_dir = os.path.dirname(path)
-        if tmp_dir is None:
-            raise Exception("no current scanning directory")
-
-        repo_path = os.path.join(tmp_dir, "sources", name)
-        try:
-            repo = pygit2.clone_repository(url=github_url, path=repo_path)
-        except pygit2.GitError as git_error:
-            # Handle generic Git-related errors
-            raise Exception(
-                f"Error while cloning repository {str(git_error)} with github url {github_url}"
-            )
-        except Exception as e:
-            # Catch any other unexpected exceptions
-            raise Exception(
-                f"An unexpected error occurred: {str(e)}.  github url {github_url}"
-            )
-
-        tag_candidates = find_suitable_tags(repo, version)
-
-        if len(tag_candidates) == 0:
-            return False, "Could not find any suitable tag in repository"
-
-        target_tag = None
-        # TODO: this one is a bit weak. let's find something stronger - maybe use the closest string?
-        for tag in tag_candidates:
-            target_tag = tag
-
-        # Idea: parse the code of the package to find the real version - we can grep the project files for
-        #  the version, git bisect until we have a file with the same version? will not work if main has not
-        #  been bumped yet in version so tags and releases are out only solutions here print(tag_candidates)
-        #  Well, that works if we run integrity check for multiple commits
-
-        #  should be good, let's open the sources
+    def get_base_path(self, path: str, name: str) -> str:
+        """
+        PyPI: find the subdirectory containing the package files.
+        The extracted archive typically has a subdirectory with the package name.
+        """
         base_dir_name = None
         for entry in os.listdir(path):
             if entry.lower().startswith(
                 name.lower().replace("-", "_")
             ) or entry.lower().startswith(name.lower()):
                 base_dir_name = entry
+
+        if base_dir_name is None or base_dir_name == "sources":
+            raise Exception("Could not find package directory in extracted files")
+
+        return os.path.join(path, base_dir_name)
+
+    def get_version(self, package_info, version: Optional[str]) -> Optional[str]:
+        """Get version from PyPI metadata or use provided version."""
+        if version is None:
+            version = package_info["info"]["version"]
+        return version
+
+    def exclude_result(
+        self,
+        file_name: str,
+        repo_root: Optional[str] = None,
+        pkg_root: Optional[str] = None,
+    ) -> bool:
+        """
+        Override base class method to add PyPI-specific exclusion logic.
+
+        This method filters out some results that are known false positives:
+        * if the file is a documentation file (based on its extension)
+        * if the file is a setup.cfg file with the egg_info claim present on PyPI and not on GitHub
+        """
+        # First check standard extensions using base class logic
+        if super().exclude_result(file_name, repo_root, pkg_root):
+            return True
+
+        # PyPI-specific: check for setup.cfg with egg_info differences
         if (
-            base_dir_name is None or base_dir_name == "sources"
-        ):  # I am not sure how we can get there
-            raise Exception("something went wrong when opening the package")
-        base_path = os.path.join(path, base_dir_name)
-
-        mismatch = find_mismatch_for_tag(repo, target_tag, base_path, repo_path)
-        message = "\n".join(map(lambda x: "* " + x["file"], mismatch))
-        return (
-            len(mismatch) > 0,
-            f"Some files present in the package are different from the ones on GitHub for "
-            f"the same version of the package: \n{message}",
-        )
+            file_name.endswith("setup.cfg")
+            and repo_root is not None
+            and pkg_root is not None
+        ):
+            repo_cfg = configparser.ConfigParser()
+            repo_cfg.read(os.path.join(repo_root, file_name))
+            pkg_cfg = configparser.ConfigParser()
+            pkg_cfg.read(os.path.join(pkg_root, file_name))
+            repo_sections = list(repo_cfg.keys())
+            pkg_sections = list(pkg_cfg.keys())
+            if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
+                return True
+        return False

guarddog/analyzer/metadata/repository_integrity_mismatch.py

@@ -1,13 +1,22 @@
+import logging
+import os
+import re
 from abc import abstractmethod
-from typing import Optional
+from typing import List, Optional
+
+import pygit2
 
 from guarddog.analyzer.metadata.detector import Detector
+from guarddog.analyzer.metadata.utils import get_file_hash
+
+log = logging.getLogger("guarddog")
 
 
 class IntegrityMismatch(Detector):
     """This package contains files that have been tampered with between the source repository and the package CDN"""
 
     RULE_NAME = "repository_integrity_mismatch"
+    EXCLUDED_EXTENSIONS: List[str] = []
 
     def __init__(self):
         super().__init__(
@@ -17,6 +26,47 @@ class IntegrityMismatch(Detector):
         )
 
     @abstractmethod
+    def extract_github_url(self, package_info, name: str) -> Optional[str]:
+        """
+        Extract GitHub URL from package metadata.
+
+        Args:
+            package_info: Package metadata dictionary
+            name: Package name
+
+        Returns:
+            GitHub URL if found, None otherwise
+        """
+        pass
+
+    @abstractmethod
+    def get_base_path(self, path: str, name: str) -> str:
+        """
+        Get the base path where package files are located.
+
+        Args:
+            path: Root extraction path
+            name: Package name
+
+        Returns:
+            Path to the package source files
+        """
+        pass
+
+    @abstractmethod
+    def get_version(self, package_info, version: Optional[str]) -> Optional[str]:
+        """
+        Extract version from package info or use provided version.
+
+        Args:
+            package_info: Package metadata dictionary
+            version: Optional version string
+
+        Returns:
+            Version string
+        """
+        pass
+
     def detect(
         self,
         package_info,
@@ -24,4 +74,154 @@ class IntegrityMismatch(Detector):
         name: Optional[str] = None,
         version: Optional[str] = None,
     ) -> tuple[bool, str]:
-        pass
+        """
+        Template method for detecting repository integrity mismatches.
+
+        This method implements the common algorithm for comparing package files
+        with their source repository. Subclasses customize behavior by implementing
+        the abstract methods for URL extraction, path resolution, and version handling.
+        """
+        if name is None:
+            return False, "Detector needs the name of the package"
+        if path is None:
+            return False, "Detector needs the path of the package"
+
+        log.debug(f"Running repository integrity mismatch heuristic on package {name}")
+
+        # Step 1: Extract GitHub URL (ecosystem-specific)
+        github_url = self.extract_github_url(package_info, name)
+        if github_url is None:
+            return False, "Could not find a GitHub URL in the package metadata"
+
+        log.debug(f"Using GitHub URL {github_url}")
+
+        # Step 2: Get version (ecosystem-specific)
+        version = self.get_version(package_info, version)
+        if version is None:
+            return False, "Could not determine version to scan"
+
+        # Step 3: Clone repository
+        tmp_dir = os.path.dirname(path)
+        repo_path = os.path.join(tmp_dir, "sources", name)
+
+        try:
+            repo = pygit2.clone_repository(url=github_url, path=repo_path)
+        except Exception as e:
+            return False, f"Could not clone repository: {str(e)}"
+
+        # Step 4: Find matching git tag
+        tag_candidates = self.find_suitable_tags(repo, version)
+        if len(tag_candidates) == 0:
+            return False, f"Could not find a tag matching version {version}"
+
+        target_tag = tag_candidates[-1]
+
+        # Step 5: Get base path where files are located (ecosystem-specific)
+        try:
+            base_path = self.get_base_path(path, name)
+        except Exception as e:
+            return False, f"Could not locate package files: {str(e)}"
+
+        # Step 6: Compare files
+        mismatch = self.find_mismatch_for_tag(repo, target_tag, base_path, repo_path)
+
+        if len(mismatch) == 0:
+            return False, ""
+
+        # Step 7: Format result message
+        message = "\n".join(map(lambda x: "* " + x["file"], mismatch))
+        return (
+            True,
+            f"Files in package differ from GitHub repository for version {version}:\n{message}",
+        )
+
+    def find_suitable_tags(self, repo, version: str) -> List[str]:
+        """
+        Find git tags that match the given version.
+
+        Args:
+            repo: pygit2.Repository object
+            version: version string to match
+
+        Returns:
+            List of tag references that match the version
+        """
+        tags_regex = re.compile("^refs/tags/(.*)")
+        tags = []
+        for ref in repo.references:
+            match = tags_regex.match(ref)
+            if match is not None:
+                tags.append(match.group(0))
+
+        tag_candidates = []
+        for tag_name in tags:
+            tag_ref = tag_name.rsplit("/", 1)[-1]
+            if tag_ref == version or tag_ref == f"v{version}":
+                tag_candidates.append(tag_name)
+        return tag_candidates
+
+    def exclude_result(
+        self,
+        file_name: str,
+        repo_root: Optional[str] = None,
+        pkg_root: Optional[str] = None,
+    ) -> bool:
+        """
+        Check if a file should be excluded from integrity checking.
+
+        Args:
+            file_name: name of the file to check
+            repo_root: path to the repository directory (optional, for subclass-specific logic)
+            pkg_root: path to the package directory (optional, for subclass-specific logic)
+
+        Returns:
+            True if the file should be excluded, False otherwise
+        """
+        for extension in self.EXCLUDED_EXTENSIONS:
+            if file_name.endswith(extension):
+                return True
+        return False
+
+    def find_mismatch_for_tag(
+        self, repo, tag: str, base_path: str, repo_path: str
+    ) -> list[dict]:
+        """
+        Find files that differ between the repository and the package.
+
+        Args:
+            repo: pygit2.Repository object
+            tag: git tag reference to checkout
+            base_path: path to the extracted package
+            repo_path: path to the cloned repository
+
+        Returns:
+            List of dictionaries describing mismatched files
+        """
+        repo.checkout(tag)
+        mismatch = []
+        for root, dirs, files in os.walk(base_path):
+            relative_path = os.path.relpath(root, base_path)
+            repo_root = os.path.join(repo_path, relative_path)
+            if not os.path.exists(repo_root):
+                continue
+            repo_files = list(
+                filter(
+                    lambda x: os.path.isfile(os.path.join(repo_root, x)),
+                    os.listdir(repo_root),
+                )
+            )
+            for file_name in repo_files:
+                if file_name not in files:
+                    continue
+                if self.exclude_result(file_name, repo_root, root):
+                    continue
+                repo_hash, _ = get_file_hash(os.path.join(repo_root, file_name))
+                pkg_hash, _ = get_file_hash(os.path.join(root, file_name))
+                if repo_hash != pkg_hash:
+                    res = {
+                        "file": os.path.join(relative_path, file_name),
+                        "repo_sha256": repo_hash,
+                        "pkg_sha256": pkg_hash,
+                    }
+                    mismatch.append(res)
+        return mismatch