guarddog 2.7.1__py3-none-any.whl → 2.9.0__py3-none-any.whl

guarddog/analyzer/metadata/__init__.py

@@ -3,6 +3,7 @@ from guarddog.analyzer.metadata.npm import NPM_METADATA_RULES
 from guarddog.analyzer.metadata.pypi import PYPI_METADATA_RULES
 from guarddog.analyzer.metadata.go import GO_METADATA_RULES
 from guarddog.analyzer.metadata.github_action import GITHUB_ACTION_METADATA_RULES
+from guarddog.analyzer.metadata.rubygems import RUBYGEMS_METADATA_RULES
 from guarddog.ecosystems import ECOSYSTEM
 
 
@@ -18,3 +19,5 @@ def get_metadata_detectors(ecosystem: ECOSYSTEM) -> dict[str, Detector]:
             return GITHUB_ACTION_METADATA_RULES
         case ECOSYSTEM.EXTENSION:
             return {}  # No metadata detectors for extensions currently
+        case ECOSYSTEM.RUBYGEMS:
+            return RUBYGEMS_METADATA_RULES

guarddog/analyzer/metadata/go/typosquatting.py

@@ -1,12 +1,6 @@
-import json
-import logging
-import os
 from typing import Optional
 
 from guarddog.analyzer.metadata.typosquatting import TyposquatDetector
-from guarddog.utils.config import TOP_PACKAGES_CACHE_LOCATION
-
-log = logging.getLogger("guarddog")
 
 
 class GoTyposquatDetector(TyposquatDetector):
@@ -19,32 +13,21 @@ class GoTyposquatDetector(TyposquatDetector):
     """
 
     def _get_top_packages(self) -> set:
-        top_packages_filename = "top_go_packages.json"
-
-        resources_dir = TOP_PACKAGES_CACHE_LOCATION
-        if resources_dir is None:
-            resources_dir = os.path.abspath(
-                os.path.join(os.path.dirname(__file__), "..", "resources")
-            )
-
-        top_packages_path = os.path.join(resources_dir, top_packages_filename)
-        top_packages_information = self._get_top_packages_local(top_packages_path)
+        """
+        Gets the top Go packages from local cache.
+        Uses the base class implementation without network refresh.
+        """
+        packages = self._get_top_packages_with_refresh(
+            packages_filename="top_go_packages.json",
+            popular_packages_url=None,  # No URL = no auto-refresh
+        )
 
-        if top_packages_information is None:
+        if not packages:
             raise Exception(
-                f"Could not retrieve top Go packages from {top_packages_path}"
+                "Could not retrieve top Go packages from top_go_packages.json"
             )
 
-        return set(top_packages_information)
-
-    def _get_top_packages_local(self, path: str) -> list[dict] | None:
-        try:
-            with open(path, "r") as f:
-                result = json.load(f)
-                return result
-        except FileNotFoundError:
-            log.error(f"File not found: {path}")
-            return None
+        return packages
 
     def detect(
         self,

guarddog/analyzer/metadata/npm/direct_url_dependency.py

@@ -10,7 +10,6 @@ from guarddog.analyzer.metadata.detector import Detector
 
 from urllib.parse import urlparse
 
-
 github_project_pattern = re.compile(r"^([\w\-\.]+)/([\w\-\.]+)")
 
 

guarddog/analyzer/metadata/npm/typosquatting.py

@@ -1,14 +1,6 @@
-import json
-import logging
-import os
-from datetime import datetime, timedelta
 from typing import Optional
 
 from guarddog.analyzer.metadata.typosquatting import TyposquatDetector
-from guarddog.utils.config import TOP_PACKAGES_CACHE_LOCATION
-import requests
-
-log = logging.getLogger("guarddog")
 
 
 class NPMTyposquatDetector(TyposquatDetector):
@@ -21,65 +13,38 @@ class NPMTyposquatDetector(TyposquatDetector):
     """
 
     def _get_top_packages(self) -> set:
-
-        popular_packages_url = (
-            "https://github.com/LeoDog896/npm-rank/releases/download/latest/raw.json"
+        """
+        Gets the top 8000 most popular NPM packages.
+        Uses the base class implementation with NPM-specific parameters.
+        """
+        return self._get_top_packages_with_refresh(
+            packages_filename="top_npm_packages.json",
+            popular_packages_url="https://github.com/LeoDog896/npm-rank/releases/download/latest/raw.json",
+            refresh_days=30,
         )
 
-        top_packages_filename = "top_npm_packages.json"
+    def _extract_package_names(self, data: dict | list | None) -> list | None:
+        """
+        Extract package names from NPM data structure.
 
-        resources_dir = TOP_PACKAGES_CACHE_LOCATION
-        if resources_dir is None:
-            resources_dir = os.path.abspath(
-                os.path.join(os.path.dirname(__file__), "..", "resources")
-            )
+        Network response format: [{"name": "package-name", ...}, ...]
+        Local file format: ["package-name", "package-name", ...]
 
-        top_packages_path = os.path.join(resources_dir, top_packages_filename)
-        top_packages_information = self._get_top_packages_local(top_packages_path)
-
-        if self._file_is_expired(top_packages_path, days=30):
-            new_information = self._get_top_packages_network(popular_packages_url)
-            if new_information is not None:
-                top_packages_information = new_information
-
-                with open(top_packages_path, "w+") as f:
-                    json.dump(new_information, f, ensure_ascii=False, indent=4)
-
-        if top_packages_information is None:
-            return set()
-        return set(top_packages_information)
-
-    def _file_is_expired(self, path: str, days: int) -> bool:
-        try:
-            update_time = datetime.fromtimestamp(os.path.getmtime(path))
-            return datetime.now() - update_time > timedelta(days=days)
-        except FileNotFoundError:
-            return True
-
-    def _get_top_packages_local(self, path: str) -> list[dict] | None:
-        try:
-            with open(path, "r") as f:
-                result = json.load(f)
-                return result
-        except FileNotFoundError:
-            log.error(f"File not found: {path}")
+        This method handles both formats and limits to top 8000 packages.
+        """
+        if data is None:
             return None
 
-    def _get_top_packages_network(self, url: str) -> list[dict] | None:
-        try:
-            response = requests.get(url)
-            response.raise_for_status()
+        # If data is already a list of strings (local file format)
+        if isinstance(data, list) and len(data) > 0:
+            if isinstance(data[0], str):
+                return data
 
-            response_data = response.json()
-            result = list([i["name"] for i in response_data[0:8000]])
+            # If data is list of dicts (network response format)
+            if isinstance(data[0], dict) and "name" in data[0]:
+                return [item["name"] for item in data[0:8000]]
 
-            return result
-        except json.JSONDecodeError:
-            log.error(f'Couldn`t convert to json: "{response.text}"')
-            return None
-        except requests.exceptions.RequestException as e:
-            log.error(f"Network error: {e}")
-            return None
+        return None
 
     def detect(
         self,

guarddog/analyzer/metadata/pypi/repository_integrity_mismatch.py

@@ -4,14 +4,12 @@ Detects if a package contains an empty description
 """
 
 import configparser
-import hashlib
 import logging
 import os
 import re
 import requests
 from typing import Optional, Tuple
 
-import pygit2  # type: ignore
 import urllib3.util
 
 from guarddog.analyzer.metadata.repository_integrity_mismatch import IntegrityMismatch
@@ -90,18 +88,6 @@ def dict_generator(indict, pre=None):
         yield pre + [indict]
 
 
-def get_file_hash(path):
-    with open(path, "rb") as f:
-        # Read the contents of the file
-        file_contents = f.read()
-        # Create a hash object
-        hash_object = hashlib.sha256()
-        # Feed the file contents to the hash object
-        hash_object.update(file_contents)
-        # Get the hexadecimal hash value
-        return hash_object.hexdigest(), str(file_contents).strip().splitlines()
-
-
 def _ensure_proper_url(url):
     parsed = urllib3.util.parse_url(url)
     if parsed.scheme is None:
@@ -140,80 +126,6 @@ def find_github_candidates(package_info) -> Tuple[set[str], Optional[str]]:
     return github_urls, best
 
 
-EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
-
-
-def exclude_result(file_name, repo_root, pkg_root):
-    """
-    This method filters out some results that are known false positives:
-    * if the file is a documentation file (based on its extension)
-    * if the file is a setup.cfg file with the egg_info claim present on Pypi and not on GitHub
-    """
-    for extension in EXCLUDED_EXTENSIONS:
-        if file_name.endswith(extension):
-            return True
-    if file_name.endswith("setup.cfg"):
-        repo_cfg = configparser.ConfigParser()
-        repo_cfg.read(os.path.join(repo_root, file_name))
-        pkg_cfg = configparser.ConfigParser()
-        pkg_cfg.read(os.path.join(pkg_root, file_name))
-        repo_sections = list(repo_cfg.keys())
-        pkg_sections = list(pkg_cfg.keys())
-        if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
-            return True
-    return False
-
-
-def find_mismatch_for_tag(repo, tag, base_path, repo_path):
-    repo.checkout(tag)
-    mismatch = []
-    for root, dirs, files in os.walk(base_path):
-        relative_path = os.path.relpath(root, base_path)
-        repo_root = os.path.join(repo_path, relative_path)
-        if not os.path.exists(repo_root):
-            continue
-        repo_files = list(
-            filter(
-                lambda x: os.path.isfile(os.path.join(repo_root, x)),
-                os.listdir(repo_root),
-            )
-        )
-        for file_name in repo_files:
-            if file_name not in files:  # ignore files we don't have in the distribution
-                continue
-            repo_hash, repo_content = get_file_hash(os.path.join(repo_root, file_name))
-            pkg_hash, pkg_content = get_file_hash(os.path.join(root, file_name))
-            if repo_hash != pkg_hash:
-                if exclude_result(file_name, repo_root, root):
-                    continue
-                res = {
-                    "file": os.path.join(relative_path, file_name),
-                    "repo_sha256": repo_hash,
-                    "pkg_sha256": pkg_hash,
-                }
-                mismatch.append(res)
-    return mismatch
-
-
-def find_suitable_tags_in_list(tags, version):
-    tag_candidates = []
-    for tag_name in tags:
-        if tag_name.endswith(version):
-            tag_candidates.append(tag_name)
-    return tag_candidates
-
-
-def find_suitable_tags(repo, version):
-    tags_regex = re.compile("^refs/tags/(.*)")
-    tags = []
-    for ref in repo.references:
-        match = tags_regex.match(ref)
-        if match is not None:
-            tags.append(match.group(0))
-
-    return find_suitable_tags_in_list(tags, version)
-
-
 # Note: we should have the GitHub related logic factored out as we will need it when we check for signed commits
 class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
@@ -228,94 +140,71 @@ class PypiIntegrityMismatchDetector(IntegrityMismatch):
     """
 
     RULE_NAME = "repository_integrity_mismatch"
+    EXCLUDED_EXTENSIONS = [".rst", ".md", ".txt"]
 
-    def detect(
-        self,
-        package_info,
-        path: Optional[str] = None,
-        name: Optional[str] = None,
-        version: Optional[str] = None,
-    ) -> tuple[bool, str]:
-        if name is None:
-            raise Exception("Detector needs the name of the package")
-        if path is None:
-            raise Exception("Detector needs the path of the package")
-
-        log.debug(
-            f"Running repository integrity mismatch heuristic on PyPI package {name} version {version}"
-        )
-        # let's extract a source repository (GitHub only for now) if we can
+    def extract_github_url(self, package_info, name: str) -> Optional[str]:
+        """Extract GitHub URL from PyPI metadata."""
         github_urls, best_github_candidate = find_github_candidates(package_info)
         if len(github_urls) == 0:
-            return False, "Could not find any GitHub url in the project's description"
-        # now, let's find the right url
+            return None
 
         github_url = find_best_github_candidate(
             (github_urls, best_github_candidate), name
         )
+        return github_url
 
-        if github_url is None:
-            return (
-                False,
-                "Could not find a good GitHub url in the project's description",
-            )
-
-        log.debug(f"Using GitHub URL {github_url}")
-        # ok, now let's try to find the version! (I need to know which version we are scanning)
-        if version is None:
-            version = package_info["info"]["version"]
-        if version is None:
-            raise Exception("Could not find suitable version to scan")
-        tmp_dir = os.path.dirname(path)
-        if tmp_dir is None:
-            raise Exception("no current scanning directory")
-
-        repo_path = os.path.join(tmp_dir, "sources", name)
-        try:
-            repo = pygit2.clone_repository(url=github_url, path=repo_path)
-        except pygit2.GitError as git_error:
-            # Handle generic Git-related errors
-            raise Exception(
-                f"Error while cloning repository {str(git_error)} with github url {github_url}"
-            )
-        except Exception as e:
-            # Catch any other unexpected exceptions
-            raise Exception(
-                f"An unexpected error occurred: {str(e)}.  github url {github_url}"
-            )
-
-        tag_candidates = find_suitable_tags(repo, version)
-
-        if len(tag_candidates) == 0:
-            return False, "Could not find any suitable tag in repository"
-
-        target_tag = None
-        # TODO: this one is a bit weak. let's find something stronger - maybe use the closest string?
-        for tag in tag_candidates:
-            target_tag = tag
-
-        # Idea: parse the code of the package to find the real version - we can grep the project files for
-        #  the version, git bisect until we have a file with the same version? will not work if main has not
-        #  been bumped yet in version so tags and releases are out only solutions here print(tag_candidates)
-        #  Well, that works if we run integrity check for multiple commits
-
-        #  should be good, let's open the sources
+    def get_base_path(self, path: str, name: str) -> str:
+        """
+        PyPI: find the subdirectory containing the package files.
+        The extracted archive typically has a subdirectory with the package name.
+        """
         base_dir_name = None
         for entry in os.listdir(path):
             if entry.lower().startswith(
                 name.lower().replace("-", "_")
             ) or entry.lower().startswith(name.lower()):
                 base_dir_name = entry
+
+        if base_dir_name is None or base_dir_name == "sources":
+            raise Exception("Could not find package directory in extracted files")
+
+        return os.path.join(path, base_dir_name)
+
+    def get_version(self, package_info, version: Optional[str]) -> Optional[str]:
+        """Get version from PyPI metadata or use provided version."""
+        if version is None:
+            version = package_info["info"]["version"]
+        return version
+
+    def exclude_result(
+        self,
+        file_name: str,
+        repo_root: Optional[str] = None,
+        pkg_root: Optional[str] = None,
+    ) -> bool:
+        """
+        Override base class method to add PyPI-specific exclusion logic.
+
+        This method filters out some results that are known false positives:
+        * if the file is a documentation file (based on its extension)
+        * if the file is a setup.cfg file with the egg_info claim present on PyPI and not on GitHub
+        """
+        # First check standard extensions using base class logic
+        if super().exclude_result(file_name, repo_root, pkg_root):
+            return True
+
+        # PyPI-specific: check for setup.cfg with egg_info differences
         if (
-            base_dir_name is None or base_dir_name == "sources"
-        ):  # I am not sure how we can get there
-            raise Exception("something went wrong when opening the package")
-        base_path = os.path.join(path, base_dir_name)
-
-        mismatch = find_mismatch_for_tag(repo, target_tag, base_path, repo_path)
-        message = "\n".join(map(lambda x: "* " + x["file"], mismatch))
-        return (
-            len(mismatch) > 0,
-            f"Some files present in the package are different from the ones on GitHub for "
-            f"the same version of the package: \n{message}",
-        )
+            file_name.endswith("setup.cfg")
+            and repo_root is not None
+            and pkg_root is not None
+        ):
+            repo_cfg = configparser.ConfigParser()
+            repo_cfg.read(os.path.join(repo_root, file_name))
+            pkg_cfg = configparser.ConfigParser()
+            pkg_cfg.read(os.path.join(pkg_root, file_name))
+            repo_sections = list(repo_cfg.keys())
+            pkg_sections = list(pkg_cfg.keys())
+            if "egg_info" in pkg_sections and "egg_info" not in repo_sections:
+                return True
+        return False

guarddog/analyzer/metadata/pypi/typosquatting.py

@@ -1,14 +1,9 @@
-import json
 import logging
-import os
-from datetime import datetime, timedelta
 from typing import Optional
 
-import requests
 import packaging.utils
 
 from guarddog.analyzer.metadata.typosquatting import TyposquatDetector
-from guarddog.utils.config import TOP_PACKAGES_CACHE_LOCATION
 
 log = logging.getLogger("guarddog")
 
@@ -25,87 +20,35 @@ class PypiTyposquatDetector(TyposquatDetector):
 
     def _get_top_packages(self) -> set:
         """
-        Gets the package information of the top 5000 most downloaded PyPI packages
-
-        Returns:
-            set: set of package data in the format:
-                {
-                    ...
-                    {
-                        download_count: ...
-                        project: <package-name>
-                    }
-                    ...
-                }
+        Gets the package information of the top 5000 most downloaded PyPI packages.
+        Uses the base class implementation with PyPI-specific parameters.
         """
-
-        popular_packages_url = (
-            "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
+        packages = self._get_top_packages_with_refresh(
+            packages_filename="top_pypi_packages.json",
+            popular_packages_url="https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json",
+            refresh_days=30,
         )
 
-        top_packages_filename = "top_pypi_packages.json"
-        resources_dir = TOP_PACKAGES_CACHE_LOCATION
-        if resources_dir is None:
-            resources_dir = os.path.abspath(
-                os.path.join(os.path.dirname(__file__), "..", "resources")
-            )
-
-        top_packages_path = os.path.join(resources_dir, top_packages_filename)
-        top_packages_information = self._get_top_packages_local(top_packages_path)
-
-        if self._file_is_expired(top_packages_path, days=30):
-            new_information = self._get_top_packages_network(popular_packages_url)
-            if new_information is not None:
-                top_packages_information = new_information
-
-                with open(top_packages_path, "w+") as f:
-                    json.dump(new_information, f, ensure_ascii=False, indent=4)
-
-        if top_packages_information is None:
-            return set()
-        return set(map(self.get_safe_name, top_packages_information))
+        # Apply canonicalization to PyPI package names
+        return set(map(self._canonicalize_name, packages))
 
-    @staticmethod
-    def get_safe_name(package):
-        return packaging.utils.canonicalize_name(package["project"])
-
-    def _file_is_expired(self, path: str, days: int) -> bool:
-        try:
-            update_time = datetime.fromtimestamp(os.path.getmtime(path))
-            return datetime.now() - update_time > timedelta(days=days)
-        except FileNotFoundError:
-            return True
-
-    def _get_top_packages_local(self, path: str) -> list[dict] | None:
-        try:
-            with open(path, "r") as f:
-                result = json.load(f)
-                return self.extract_information(result)
-        except FileNotFoundError:
-            log.error(f"File not found: {path}")
+    def _extract_package_names(self, data: dict | list | None) -> list | None:
+        """
+        Extract package names from PyPI data structure.
+        PyPI data has format: {"rows": [{"project": "name", "download_count": ...}, ...]}
+        """
+        if data is None:
             return None
 
-    def _get_top_packages_network(self, url: str) -> list[dict] | None:
-        try:
-            response = requests.get(url)
-            response.raise_for_status()
-
-            response_data = response.json()
-            result = response_data
+        if isinstance(data, dict) and "rows" in data:
+            return [row["project"] for row in data["rows"]]
 
-            return self.extract_information(result)
-        except json.JSONDecodeError:
-            log.error(f'Couldn`t convert to json: "{response.text}"')
-            return None
-        except requests.exceptions.RequestException as e:
-            log.error(f"Network error: {e}")
-            return None
+        return None
 
     @staticmethod
-    def extract_information(data: dict | None) -> list[dict] | None:
-        if data is not None:
-            return data.get("rows")
-        return None
+    def _canonicalize_name(package_name: str) -> str:
+        """Canonicalize PyPI package names according to PEP 503."""
+        return packaging.utils.canonicalize_name(package_name)
 
     def detect(
         self,