guarddog 2.7.0__py3-none-any.whl → 2.8.4__py3-none-any.whl

guarddog/analyzer/metadata/__init__.py

@@ -3,6 +3,7 @@ from guarddog.analyzer.metadata.npm import NPM_METADATA_RULES
 from guarddog.analyzer.metadata.pypi import PYPI_METADATA_RULES
 from guarddog.analyzer.metadata.go import GO_METADATA_RULES
 from guarddog.analyzer.metadata.github_action import GITHUB_ACTION_METADATA_RULES
+from guarddog.analyzer.metadata.rubygems import RUBYGEMS_METADATA_RULES
 from guarddog.ecosystems import ECOSYSTEM
 
 
@@ -18,3 +19,5 @@ def get_metadata_detectors(ecosystem: ECOSYSTEM) -> dict[str, Detector]:
             return GITHUB_ACTION_METADATA_RULES
         case ECOSYSTEM.EXTENSION:
             return {}  # No metadata detectors for extensions currently
+        case ECOSYSTEM.RUBYGEMS:
+            return RUBYGEMS_METADATA_RULES

guarddog/analyzer/metadata/go/typosquatting.py

@@ -1,10 +1,13 @@
 import json
+import logging
 import os
 from typing import Optional
 
 from guarddog.analyzer.metadata.typosquatting import TyposquatDetector
 from guarddog.utils.config import TOP_PACKAGES_CACHE_LOCATION
 
+log = logging.getLogger("guarddog")
+
 
 class GoTyposquatDetector(TyposquatDetector):
     """Detector for typosquatting attacks for go modules. Checks for distance one Levenshtein,
@@ -25,12 +28,7 @@ class GoTyposquatDetector(TyposquatDetector):
             )
 
         top_packages_path = os.path.join(resources_dir, top_packages_filename)
-
-        top_packages_information = None
-
-        if top_packages_filename in os.listdir(resources_dir):
-            with open(top_packages_path, "r") as top_packages_file:
-                top_packages_information = json.load(top_packages_file)
+        top_packages_information = self._get_top_packages_local(top_packages_path)
 
         if top_packages_information is None:
             raise Exception(
@@ -39,6 +37,15 @@ class GoTyposquatDetector(TyposquatDetector):
 
         return set(top_packages_information)
 
+    def _get_top_packages_local(self, path: str) -> list[dict] | None:
+        try:
+            with open(path, "r") as f:
+                result = json.load(f)
+                return result
+        except FileNotFoundError:
+            log.error(f"File not found: {path}")
+            return None
+
     def detect(
         self,
         package_info,

guarddog/analyzer/metadata/npm/typosquatting.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 from datetime import datetime, timedelta
 from typing import Optional
@@ -7,6 +8,8 @@ from guarddog.analyzer.metadata.typosquatting import TyposquatDetector
 from guarddog.utils.config import TOP_PACKAGES_CACHE_LOCATION
 import requests
 
+log = logging.getLogger("guarddog")
+
 
 class NPMTyposquatDetector(TyposquatDetector):
     """Detector for typosquatting attacks. Detects if a package name is a typosquat of one of the top 5000 packages.
@@ -32,24 +35,52 @@ class NPMTyposquatDetector(TyposquatDetector):
             )
 
         top_packages_path = os.path.join(resources_dir, top_packages_filename)
+        top_packages_information = self._get_top_packages_local(top_packages_path)
 
-        top_packages_information = None
-
-        if top_packages_filename in os.listdir(resources_dir):
-            update_time = datetime.fromtimestamp(os.path.getmtime(top_packages_path))
+        if self._file_is_expired(top_packages_path, days=30):
+            new_information = self._get_top_packages_network(popular_packages_url)
+            if new_information is not None:
+                top_packages_information = new_information
 
-            if datetime.now() - update_time <= timedelta(days=30):
-                with open(top_packages_path, "r") as top_packages_file:
-                    top_packages_information = json.load(top_packages_file)
+                with open(top_packages_path, "w+") as f:
+                    json.dump(new_information, f, ensure_ascii=False, indent=4)
 
         if top_packages_information is None:
-            response = requests.get(popular_packages_url).json()
-            top_packages_information = list([i["name"] for i in response[0:8000]])
-            with open(top_packages_path, "w+") as f:
-                json.dump(top_packages_information, f, ensure_ascii=False, indent=4)
-
+            return set()
         return set(top_packages_information)
 
+    def _file_is_expired(self, path: str, days: int) -> bool:
+        try:
+            update_time = datetime.fromtimestamp(os.path.getmtime(path))
+            return datetime.now() - update_time > timedelta(days=days)
+        except FileNotFoundError:
+            return True
+
+    def _get_top_packages_local(self, path: str) -> list[dict] | None:
+        try:
+            with open(path, "r") as f:
+                result = json.load(f)
+                return result
+        except FileNotFoundError:
+            log.error(f"File not found: {path}")
+            return None
+
+    def _get_top_packages_network(self, url: str) -> list[dict] | None:
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+
+            response_data = response.json()
+            result = list([i["name"] for i in response_data[0:8000]])
+
+            return result
+        except json.JSONDecodeError:
+            log.error(f'Couldn`t convert to json: "{response.text}"')
+            return None
+        except requests.exceptions.RequestException as e:
+            log.error(f"Network error: {e}")
+            return None
+
     def detect(
         self,
         package_info,

guarddog/analyzer/metadata/pypi/repository_integrity_mismatch.py

@@ -4,14 +4,12 @@ Detects if a package contains an empty description
 """
 
 import configparser
-import hashlib
 import logging
 import os
 import re
 import requests
 from typing import Optional, Tuple
 
-import pygit2  # type: ignore
 import urllib3.util
 
 from guarddog.analyzer.metadata.repository_integrity_mismatch import IntegrityMismatch
@@ -90,18 +88,6 @@ def dict_generator(indict, pre=None):
         yield pre + [indict]
 
 
-def get_file_hash(path):
-    with open(path, "rb") as f:
-        # Read the contents of the file
-        file_contents = f.read()
-        # Create a hash object
-        hash_object = hashlib.sha256()
-        # Feed the file contents to the hash object
-        hash_object.update(file_contents)
-        # Get the hexadecimal hash value
-        return hash_object.hexdigest(), str(file_contents).strip().splitlines()
-
-
 def _ensure_proper_url(url):
     parsed = urllib3.util.parse_url(url)
     if parsed.scheme is None:
@@ -140,80 +126,6 @@ def find_github_candidates(package_info) -> Tuple[set[str], Optional[str]]:
     return github_urls, best
 
 
-EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
-
-
-def exclude_result(file_name, repo_root, pkg_root):
-    """
-    This method filters out some results that are known false positives:
-    * if the file is a documentation file (based on its extension)
-    * if the file is a setup.cfg file with the egg_info claim present on Pypi and not on GitHub
-    """
-    for extension in EXCLUDED_EXTENSIONS:
-        if file_name.endswith(extension):
-            return True
-    if file_name.endswith("setup.cfg"):
-        repo_cfg = configparser.ConfigParser()
-        repo_cfg.read(os.path.join(repo_root, file_name))
-        pkg_cfg = configparser.ConfigParser()
-        pkg_cfg.read(os.path.join(pkg_root, file_name))
-        repo_sections = list(repo_cfg.keys())
-        pkg_sections = list(pkg_cfg.keys())
-        if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
-            return True
-    return False
-
-
-def find_mismatch_for_tag(repo, tag, base_path, repo_path):
-    repo.checkout(tag)
-    mismatch = []
-    for root, dirs, files in os.walk(base_path):
-        relative_path = os.path.relpath(root, base_path)
-        repo_root = os.path.join(repo_path, relative_path)
-        if not os.path.exists(repo_root):
-            continue
-        repo_files = list(
-            filter(
-                lambda x: os.path.isfile(os.path.join(repo_root, x)),
-                os.listdir(repo_root),
-            )
-        )
-        for file_name in repo_files:
-            if file_name not in files:  # ignore files we don't have in the distribution
-                continue
-            repo_hash, repo_content = get_file_hash(os.path.join(repo_root, file_name))
-            pkg_hash, pkg_content = get_file_hash(os.path.join(root, file_name))
-            if repo_hash != pkg_hash:
-                if exclude_result(file_name, repo_root, root):
-                    continue
-                res = {
-                    "file": os.path.join(relative_path, file_name),
-                    "repo_sha256": repo_hash,
-                    "pkg_sha256": pkg_hash,
-                }
-                mismatch.append(res)
-    return mismatch
-
-
-def find_suitable_tags_in_list(tags, version):
-    tag_candidates = []
-    for tag_name in tags:
-        if tag_name.endswith(version):
-            tag_candidates.append(tag_name)
-    return tag_candidates
-
-
-def find_suitable_tags(repo, version):
-    tags_regex = re.compile("^refs/tags/(.*)")
-    tags = []
-    for ref in repo.references:
-        match = tags_regex.match(ref)
-        if match is not None:
-            tags.append(match.group(0))
-
-    return find_suitable_tags_in_list(tags, version)
-
-
 # Note: we should have the GitHub related logic factored out as we will need it when we check for signed commits
 class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
@@ -228,94 +140,71 @@ class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
 
     RULE_NAME = "repository_integrity_mismatch"
+    EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
 
-    def detect(
-        self,
-        package_info,
-        path: Optional[str] = None,
-        name: Optional[str] = None,
-        version: Optional[str] = None,
-    ) -> tuple[bool, str]:
-        if name is None:
-            raise Exception("Detector needs the name of the package")
-        if path is None:
-            raise Exception("Detector needs the path of the package")
-
-        log.debug(
-            f"Running repository integrity mismatch heuristic on PyPI package {name} version {version}"
-        )
-        # let's extract a source repository (GitHub only for now) if we can
+    def extract_github_url(self, package_info, name: str) -> Optional[str]:
+        """Extract GitHub URL from PyPI metadata."""
         github_urls, best_github_candidate = find_github_candidates(package_info)
         if len(github_urls) == 0:
-            return False, "Could not find any GitHub url in the project's description"
-        # now, let's find the right url
+            return None
 
         github_url = find_best_github_candidate(
             (github_urls, best_github_candidate), name
         )
+        return github_url
 
-        if github_url is None:
-            return (
-                False,
-                "Could not find a good GitHub url in the project's description",
-            )
-
-        log.debug(f"Using GitHub URL {github_url}")
-        # ok, now let's try to find the version! (I need to know which version we are scanning)
-        if version is None:
-            version = package_info["info"]["version"]
-        if version is None:
-            raise Exception("Could not find suitable version to scan")
-        tmp_dir = os.path.dirname(path)
-        if tmp_dir is None:
-            raise Exception("no current scanning directory")
-
-        repo_path = os.path.join(tmp_dir, "sources", name)
-        try:
-            repo = pygit2.clone_repository(url=github_url, path=repo_path)
-        except pygit2.GitError as git_error:
-            # Handle generic Git-related errors
-            raise Exception(
-                f"Error while cloning repository {str(git_error)} with github url {github_url}"
-            )
-        except Exception as e:
-            # Catch any other unexpected exceptions
-            raise Exception(
-                f"An unexpected error occurred: {str(e)}.  github url {github_url}"
-            )
-
-        tag_candidates = find_suitable_tags(repo, version)
-
-        if len(tag_candidates) == 0:
-            return False, "Could not find any suitable tag in repository"
-
-        target_tag = None
-        # TODO: this one is a bit weak. let's find something stronger - maybe use the closest string?
-        for tag in tag_candidates:
-            target_tag = tag
-
-        # Idea: parse the code of the package to find the real version - we can grep the project files for
-        #  the version, git bisect until we have a file with the same version? will not work if main has not
-        #  been bumped yet in version so tags and releases are out only solutions here print(tag_candidates)
-        #  Well, that works if we run integrity check for multiple commits
-
-        #  should be good, let's open the sources
+    def get_base_path(self, path: str, name: str) -> str:
+        """
+        PyPI: find the subdirectory containing the package files.
+        The extracted archive typically has a subdirectory with the package name.
+        """
         base_dir_name = None
         for entry in os.listdir(path):
             if entry.lower().startswith(
                 name.lower().replace("-", "_")
             ) or entry.lower().startswith(name.lower()):
                 base_dir_name = entry
+
+        if base_dir_name is None or base_dir_name == "sources":
+            raise Exception("Could not find package directory in extracted files")
+
+        return os.path.join(path, base_dir_name)
+
+    def get_version(self, package_info, version: Optional[str]) -> Optional[str]:
+        """Get version from PyPI metadata or use provided version."""
+        if version is None:
+            version = package_info["info"]["version"]
+        return version
+
+    def exclude_result(
+        self,
+        file_name: str,
+        repo_root: Optional[str] = None,
+        pkg_root: Optional[str] = None,
+    ) -> bool:
+        """
+        Override base class method to add PyPI-specific exclusion logic.
+
+        This method filters out some results that are known false positives:
+        * if the file is a documentation file (based on its extension)
+        * if the file is a setup.cfg file with the egg_info claim present on PyPI and not on GitHub
+        """
+        # First check standard extensions using base class logic
+        if super().exclude_result(file_name, repo_root, pkg_root):
+            return True
+
+        # PyPI-specific: check for setup.cfg with egg_info differences
         if (
-            base_dir_name is None or base_dir_name == "sources"
-        ):  # I am not sure how we can get there
-            raise Exception("something went wrong when opening the package")
-        base_path = os.path.join(path, base_dir_name)
-
-        mismatch = find_mismatch_for_tag(repo, target_tag, base_path, repo_path)
-        message = "\n".join(map(lambda x: "* " + x["file"], mismatch))
-        return (
-            len(mismatch) > 0,
-            f"Some files present in the package are different from the ones on GitHub for "
-            f"the same version of the package: \n{message}",
-        )
+            file_name.endswith("setup.cfg")
+            and repo_root is not None
+            and pkg_root is not None
+        ):
+            repo_cfg = configparser.ConfigParser()
+            repo_cfg.read(os.path.join(repo_root, file_name))
+            pkg_cfg = configparser.ConfigParser()
+            pkg_cfg.read(os.path.join(pkg_root, file_name))
+            repo_sections = list(repo_cfg.keys())
+            pkg_sections = list(pkg_cfg.keys())
+            if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
+                return True
+        return False

guarddog/analyzer/metadata/pypi/typosquatting.py

@@ -51,27 +51,61 @@ class PypiTyposquatDetector(TyposquatDetector):
             )
 
         top_packages_path = os.path.join(resources_dir, top_packages_filename)
+        top_packages_information = self._get_top_packages_local(top_packages_path)
 
-        top_packages_information = None
+        if self._file_is_expired(top_packages_path, days=30):
+            new_information = self._get_top_packages_network(popular_packages_url)
+            if new_information is not None:
+                top_packages_information = new_information
 
-        if top_packages_filename in os.listdir(resources_dir):
-            update_time = datetime.fromtimestamp(os.path.getmtime(top_packages_path))
-
-            if datetime.now() - update_time <= timedelta(days=30):
-                with open(top_packages_path, "r") as top_packages_file:
-                    top_packages_information = json.load(top_packages_file)["rows"]
+                with open(top_packages_path, "w+") as f:
+                    json.dump(new_information, f, ensure_ascii=False, indent=4)
 
         if top_packages_information is None:
-            response = requests.get(popular_packages_url).json()
-            with open(top_packages_path, "w+") as f:
-                json.dump(response, f, ensure_ascii=False, indent=4)
-
-            top_packages_information = response["rows"]
-
-        def get_safe_name(package):
-            return packaging.utils.canonicalize_name(package["project"])
-
-        return set(map(get_safe_name, top_packages_information))
+            return set()
+        return set(map(self.get_safe_name, top_packages_information))
+
+    @staticmethod
+    def get_safe_name(package):
+        return packaging.utils.canonicalize_name(package["project"])
+
+    def _file_is_expired(self, path: str, days: int) -> bool:
+        try:
+            update_time = datetime.fromtimestamp(os.path.getmtime(path))
+            return datetime.now() - update_time > timedelta(days=days)
+        except FileNotFoundError:
+            return True
+
+    def _get_top_packages_local(self, path: str) -> list[dict] | None:
+        try:
+            with open(path, "r") as f:
+                result = json.load(f)
+                return self.extract_information(result)
+        except FileNotFoundError:
+            log.error(f"File not found: {path}")
+            return None
+
+    def _get_top_packages_network(self, url: str) -> list[dict] | None:
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+
+            response_data = response.json()
+            result = response_data
+
+            return self.extract_information(result)
+        except json.JSONDecodeError:
+            log.error(f'Couldn`t convert to json: "{response.text}"')
+            return None
+        except requests.exceptions.RequestException as e:
+            log.error(f"Network error: {e}")
+            return None
+
+    @staticmethod
+    def extract_information(data: dict | None) -> list[dict] | None:
+        if data is not None:
+            return data.get("rows")
+        return None
 
     def detect(
         self,