guarddog 2.4.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

guarddog/scanners/npm_project_scanner.py

@@ -1,11 +1,15 @@
 import json
 import logging
+import os
+import re
+from typing import List
+
 import requests
 from semantic_version import NpmSpec, Version  # type:ignore
 
-from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 from guarddog.scanners.npm_package_scanner import NPMPackageScanner
-from guarddog.scanners.scanner import ProjectScanner
+from guarddog.scanners.scanner import Dependency, DependencyVersion, ProjectScanner
+from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 
 log = logging.getLogger("guarddog")
 
@@ -21,7 +25,7 @@ class NPMRequirementsScanner(ProjectScanner):
     def __init__(self) -> None:
         super().__init__(NPMPackageScanner())
 
-    def parse_requirements(self, raw_requirements: str) -> dict:
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
         """
         Parses requirements.txt specification and finds all valid
         versions of each dependency
@@ -40,8 +44,8 @@ class NPMRequirementsScanner(ProjectScanner):
             }
         """
         package = json.loads(raw_requirements)
-        dependencies = package["dependencies"] if "dependencies" in package else {}
-        dev_dependencies = (
+        dependencies_attr = package["dependencies"] if "dependencies" in package else {}
+        dev_dependencies_attr = (
             package["devDependencies"] if "devDependencies" in package else {}
         )
 
@@ -82,23 +86,63 @@ class NPMRequirementsScanner(ProjectScanner):
             return versions
 
         merged = {}  # type: dict[str, set[str]]
-        for package, selector in list(dependencies.items()) + list(
-            dev_dependencies.items()
+        for package, selector in list(dependencies_attr.items()) + list(
+            dev_dependencies_attr.items()
         ):
             if package not in merged:
                 merged[package] = set()
             merged[package].add(selector)
 
-        results = {}
+        dependencies: List[Dependency] = []
         for package, all_selectors in merged.items():
             versions = set()  # type: set[str]
             for selector in all_selectors:
                 versions = versions.union(
                     get_matched_versions(find_all_versions(package), selector)
                 )
+
             if len(versions) == 0:
                 log.error(f"Package/Version {package} not on NPM\n")
                 continue
 
-            results[package] = versions
-        return results
+            idx = next(
+                iter(
+                    [
+                        ix
+                        for ix, line in enumerate(raw_requirements.splitlines())
+                        if package in line
+                    ]
+                ),
+                0,
+            )
+
+            dep_versions = list(
+                map(
+                    lambda d: DependencyVersion(version=d, location=idx + 1),
+                    versions,
+                )
+            )
+
+            # find the dep with the same name or create a new one
+            dep = next(
+                filter(
+                    lambda d: d.name == package,
+                    dependencies,
+                ),
+                None,
+            )
+            if not dep:
+                dep = Dependency(name=package, versions=set())
+                dependencies.append(dep)
+
+            dep.versions.update(dep_versions)
+
+        return dependencies
+
+    def find_requirements(self, directory: str) -> list[str]:
+        requirement_files = []
+        for root, dirs, files in os.walk(directory):
+            for name in files:
+                if re.match(r"^package\.json$", name, flags=re.IGNORECASE):
+                    requirement_files.append(os.path.join(root, name))
+        return requirement_files

guarddog/scanners/pypi_project_scanner.py

@@ -1,11 +1,14 @@
 import logging
+import os
 import re
+from typing import List
+
 import pkg_resources
 import requests
 from packaging.specifiers import Specifier, Version
 
 from guarddog.scanners.pypi_package_scanner import PypiPackageScanner
-from guarddog.scanners.scanner import ProjectScanner
+from guarddog.scanners.scanner import Dependency, DependencyVersion, ProjectScanner
 from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 
 log = logging.getLogger("guarddog")
@@ -37,17 +40,20 @@ class PypiRequirementsScanner(ProjectScanner):
 
         for line in requirements:
             is_requirement = re.match(r"\w", line)
-            if is_requirement:
-                if "\\" in line:
-                    line = line.replace("\\", "")
 
-                stripped_line = line.strip()
-                if len(stripped_line) > 0:
-                    sanitized_lines.append(stripped_line)
+            if not is_requirement:
+                sanitized_lines.append("")  # empty line to keep the line number
+                continue
+
+            if "\\" in line:
+                line = line.replace("\\", "")
+
+            stripped_line = line.strip()
+            sanitized_lines.append(stripped_line)
 
         return sanitized_lines
 
-    def parse_requirements(self, raw_requirements: str) -> dict[str, set[str]]:
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
         """
         Parses requirements.txt specification and finds all valid
         versions of each dependency
@@ -57,17 +63,10 @@ class PypiRequirementsScanner(ProjectScanner):
 
         Returns:
             dict: mapping of dependencies to valid versions
-
-            ex.
-            {
-                ....
-                <dependency-name>: [0.0.1, 0.0.2, ...],
-                ...
-            }
         """
         requirements = raw_requirements.splitlines()
         sanitized_requirements = self._sanitize_requirements(requirements)
-        dependencies = {}
+        dependencies: List[Dependency] = []
 
         def get_matched_versions(versions: set[str], semver_range: str) -> set[str]:
             """
@@ -77,8 +76,11 @@ class PypiRequirementsScanner(ProjectScanner):
 
             # Filters to specified versions
             try:
-                spec = Specifier(semver_range)
-                result = [Version(m) for m in spec.filter(versions)]
+                matching_versions = versions
+                if semver_range:
+                    spec = Specifier(semver_range)
+                    matching_versions = set(spec.filter(versions))
+                result = [Version(m) for m in matching_versions]
             except ValueError:
                 # use it raw
                 return set([semver_range])
@@ -142,8 +144,47 @@ class PypiRequirementsScanner(ProjectScanner):
                     )
                     continue
 
-                dependencies[requirement.project_name] = versions
+                idx = next(
+                    iter(
+                        [
+                            ix
+                            for ix, line in enumerate(requirements)
+                            if str(requirement) in line
+                        ]
+                    ),
+                    0,
+                )
+
+                dep_versions = list(
+                    map(
+                        lambda d: DependencyVersion(version=d, location=idx + 1),
+                        versions,
+                    )
+                )
+
+                # find the dep with the same name or create a new one
+                dep = next(
+                    filter(
+                        lambda d: d.name == requirement.project_name,
+                        dependencies,
+                    ),
+                    None,
+                )
+                if not dep:
+                    dep = Dependency(name=requirement.project_name, versions=set())
+                    dependencies.append(dep)
+
+                dep.versions.update(dep_versions)
+
         except Exception as e:
             log.error(f"Received error {str(e)}")
 
         return dependencies
+
+    def find_requirements(self, directory: str) -> list[str]:
+        requirement_files = []
+        for root, dirs, files in os.walk(directory):
+            for name in files:
+                if re.match(r"^requirements(-dev)?\.txt$", name, flags=re.IGNORECASE):
+                    requirement_files.append(os.path.join(root, name))
+        return requirement_files

guarddog/scanners/scanner.py

@@ -2,11 +2,12 @@ import concurrent.futures
 import json
 import logging
 import os
-import sys
 import tempfile
 import typing
 from abc import abstractmethod
 from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import List, Optional, Set, Tuple
 
 import requests
 
@@ -21,183 +22,65 @@ def noop(arg: typing.Any) -> None:
     pass
 
 
-class ProjectScanner:
-    def __init__(self, package_scanner):
-        super().__init__()
-        self.package_scanner = package_scanner
-
-    def _authenticate_by_access_token(self) -> tuple[str, str]:
-        """
-        Gives GitHub authentication through access token
-
-        Returns:
-            tuple[str, str]: username, personal access token
-        """
-
-        user = os.getenv("GIT_USERNAME")
-        personal_access_token = os.getenv("GH_TOKEN")
-        if not user or not personal_access_token:
-            log.error(
-                """WARNING: Please set GIT_USERNAME (Github handle) and GH_TOKEN
-                (generate a personal access token in Github settings > developer)
-                as environment variables before proceeding."""
-            )
-            exit(1)
-        return (user, personal_access_token)
-
-    def scan_requirements(
-        self,
-        requirements: str,
-        rules=None,
-        callback: typing.Callable[[dict], None] = noop,
-    ) -> dict:
-        """
-        Reads the requirements.txt file and scans each possible
-        dependency and version
-
-        Args:
-            requirements (str): contents of requirements.txt file
-            rules: list of rules to apply
-            callback: callback to call for each result
-
-        Returns:
-            dict: mapping of dependencies to scan results
-
-            ex.
-            {
-                ....
-                <dependency-name>: {
-                        issues: ...,
-                        results: {
-                            ...
-                        }
-                    },
-                ...
-            }
-        """
-
-        def scan_single_dependency(dependency, version):
-            log.debug(f"Scanning {dependency} version {version}")
-            result = self.package_scanner.scan_remote(dependency, version, rules)
-            return {"dependency": dependency, "version": version, "result": result}
-
-        dependencies = self.parse_requirements(requirements)
-        num_workers = PARALLELISM
-
-        log.info(
-            f"Scanning using at most {num_workers} parallel worker threads\n"
-        )
-        with ThreadPoolExecutor(max_workers=num_workers) as pool:
-            try:
-                futures: typing.List[concurrent.futures.Future] = []
-                for dependency, versions in dependencies.items():
-                    assert versions is None or len(versions) > 0
-                    if versions is None:
-                        # this will cause scan_remote to use the latest version
-                        futures.append(
-                            pool.submit(scan_single_dependency, dependency, None)
-                        )
-                    else:
-                        futures.extend(
-                            map(
-                                lambda version: pool.submit(
-                                    scan_single_dependency, dependency, version
-                                ),
-                                versions,
-                            )
-                        )
-
-                results = []
-                for future in concurrent.futures.as_completed(futures):
-                    result = future.result()
-                    if callback is not None:
-                        callback(result)
-                    results.append(result)
-            except KeyboardInterrupt:
-                log.warning("Received keyboard interrupt, cancelling scan\n")
-                pool.shutdown(wait=False, cancel_futures=True)
-
-        return results  # type: ignore
+@dataclass
+class DependencyVersion:
+    """
+    This class represents the identified dependency versions in a project,
+    usually defined in a specification file (requirements.txt, package.json, etc.)
 
-    def scan_remote(self, url: str, branch: str, requirements_name: str) -> dict:
-        """
-        Scans remote requirements.txt file
+    Attributes:
+        version (str): The version of the dependency. e.g., "1.0.0"
+        location (int): This indicates the line number in the specification file where the dependency is defined.
+    """
 
-        Args:
-            url (str): url of the GitHub repo
-            branch (str): branch containing requirements.txt
-            requirements_name (str, optional): name of requirements file.
-                Defaults to "requirements.txt".
+    version: str  # the version number of the dependency
+    location: int
 
-        Returns:
-            dict: mapping of dependencies to scan results
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.version == other
+        if isinstance(other, DependencyVersion):
+            return self.version == other.version
+        return NotImplemented
 
-            ex.
-            {
-                ....
-                <dependency-name>: {
-                        issues: ...,
-                        results: {
-                            ...
-                        }
-                    },
-                ...
-            }
-        """
+    def __hash__(self):
+        return hash(self.version)
 
-        token = self._authenticate_by_access_token()
-        githubusercontent_url = url.replace("github", "raw.githubusercontent")
+    def __repr__(self):
+        return f"DependencyVersion({self.version!r})"
 
-        req_url = f"{githubusercontent_url}/{branch}/{requirements_name}"
-        resp = requests.get(url=req_url, auth=token)
 
-        if resp.status_code == 200:
-            return self.scan_requirements(resp.content.decode())
-        else:
-            log.error(
-                f"{req_url} does not exist. Check your link or branch name."
-            )
-            sys.exit(255)
+@dataclass
+class Dependency:
+    """
+    This class represents a dependency in a project, usually defined in a specification file
 
-    def scan_local(
-        self, path, rules=None, callback: typing.Callable[[dict], None] = noop
-    ):
-        """
-        Scans a local requirements.txt file
+    Attributes:
+        name (str): The name of the dependency. e.g., "requests"
+        versions (Set[DependencyVersion]): A set of identified versions of the dependency.
+    """
+    name: str
+    versions: Set[DependencyVersion]
 
-        Args:
-            path (str): path to requirements.txt file
-            rules: list of rules to apply
-            callback: callback to call for each result
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.name == other
+        if isinstance(other, Dependency):
+            return self.name == other.name
+        return NotImplemented
 
-        Returns:
-            dict: mapping of dependencies to scan results
+    def __repr__(self):
+        return f"Dependency({self.name!r})"
 
-            ex.
-            {
-                ....
-                <dependency-name>: {
-                        issues: ...,
-                        results: {
-                            ...
-                        }
-                    },
-                ...
-            }
-        """
 
-        try:
-            with open(path, "r") as f:
-                return self.scan_requirements(f.read(), rules, callback)
-        except Exception as e:
-            log.error(f"Received {e}")
-            sys.exit(255)
+@dataclass
+class DependencyFile:
+    """
+    This class represents a specification file for a project (requirements.txt, package.json, etc.)
+    """
 
-    @abstractmethod
-    def parse_requirements(
-        self, raw_requirements: str
-    ) -> dict[str, set[str]]:  # returns { package: version }
-        pass
+    file_path: str
+    dependencies: List[Dependency]
 
 
 class PackageScanner:
@@ -324,3 +207,202 @@ class PackageScanner:
         finally:
             log.debug(f"Removing temporary archive file {archive_path}")
             os.remove(archive_path)
+
+
+class ProjectScanner:
+    def __init__(self, package_scanner: PackageScanner):
+        super().__init__()
+        self.package_scanner = package_scanner
+
+    def _authenticate_by_access_token(self) -> tuple[str, str]:
+        """
+        Gives GitHub authentication through access token
+
+        Returns:
+            tuple[str, str]: username, personal access token
+        """
+
+        user = os.getenv("GIT_USERNAME")
+        personal_access_token = os.getenv("GH_TOKEN")
+        if not user or not personal_access_token:
+            log.error(
+                """WARNING: Please set GIT_USERNAME (Github handle) and GH_TOKEN
+                (generate a personal access token in Github settings > developer)
+                as environment variables before proceeding."""
+            )
+            exit(1)
+        return (user, personal_access_token)
+
+    def scan_dependencies(
+        self,
+        dependencies: List[Dependency],
+        rules=None,
+        callback: typing.Callable[[dict], None] = noop,
+    ) -> list[dict]:
+        """
+        scans each possible dependency and version supplied
+
+        Args:
+            dependencies a list of dependencies to scan
+            rules: list of rules to apply
+            callback: callback to call for each result
+
+        Returns:
+            dict: mapping of dependencies to scan results
+
+            ex.
+            {
+                ....
+                <dependency-name>: {
+                        issues: ...,
+                        results: {
+                            ...
+                        }
+                    },
+                ...
+            }
+        """
+
+        def scan_single_dependency(dependency: str, version: Optional[str]) -> dict:
+            log.debug(f"Scanning {dependency} version {version}")
+            result = self.package_scanner.scan_remote(dependency, version, rules)
+            return {"dependency": dependency, "version": version, "result": result}
+
+        num_workers = PARALLELISM
+
+        log.info(f"Scanning using at most {num_workers} parallel worker threads\n")
+        with ThreadPoolExecutor(max_workers=num_workers) as pool:
+            try:
+                futures: typing.List[concurrent.futures.Future] = []
+                for dependency in dependencies:
+                    versions = dependency.versions
+                    if not versions:
+                        # this will cause scan_remote to use the latest version
+                        futures.append(
+                            pool.submit(scan_single_dependency, dependency.name, None)
+                        )
+                    else:
+                        futures.extend(
+                            map(
+                                lambda version: pool.submit(
+                                    scan_single_dependency,
+                                    dependency.name,
+                                    version.version,
+                                ),
+                                versions,
+                            )
+                        )
+
+                results = []
+                for future in concurrent.futures.as_completed(futures):
+                    result = future.result()
+                    if callback is not None:
+                        callback(result)
+                    results.append(result)
+            except KeyboardInterrupt:
+                log.warning("Received keyboard interrupt, cancelling scan\n")
+                pool.shutdown(wait=False, cancel_futures=True)
+
+        return results
+
+    def scan_remote(
+        self, url: str, branch: str, requirements_name: str
+    ) -> tuple[List[Dependency], list[dict]]:
+        """
+        Scans remote requirements.txt file
+
+        Args:
+            url (str): url of the GitHub repo
+            branch (str): branch containing requirements.txt
+            requirements_name (str, optional): name of requirements file.
+                Defaults to "requirements.txt".
+
+        Returns:
+            deps: list of dependencies to scan
+            results: mapping of dependencies to scan results
+            ex.
+            {
+                ....
+                <dependency-name>: {
+                        issues: ...,
+                        results: {
+                            ...
+                        }
+                    },
+                ...
+            }
+        """
+
+        token = self._authenticate_by_access_token()
+        githubusercontent_url = url.replace("github", "raw.githubusercontent")
+        req_url = f"{githubusercontent_url}/{branch}/{requirements_name}"
+        resp = requests.get(url=req_url, auth=token)
+        resp.raise_for_status()
+        dependencies = self.parse_requirements(resp.content.decode())
+        return dependencies, self.scan_dependencies(dependencies)
+
+    def scan_local(
+        self, path, rules=None, callback: typing.Callable[[dict], None] = noop
+    ) -> Tuple[List[DependencyFile], list[dict]]:
+        """
+        Scans a local requirements files (requirements.txt, package.json, etc.)
+
+        Args:
+            path (str): path to requirements file or directory to search
+            rules: list of rules to apply
+            callback: callback to call for each result
+
+        Returns:
+            deps: list of dependencies to scan
+            results: mapping of dependencies to scan results
+            ex.
+            {
+                ....
+                <dependency-name>: {
+                        issues: ...,
+                        results: {
+                            ...
+                        }
+                    },
+                ...
+            }
+
+        """
+
+        requirement_paths = []
+
+        try:
+            if os.path.isfile(path):
+                requirement_paths.append(path)
+            elif os.path.isdir(path):
+                requirement_paths.extend(self.find_requirements(path))
+            else:
+                raise ValueError(f"unable to find file or directory {path}")
+
+            dep_files: List[DependencyFile] = []
+
+            for req in requirement_paths:
+                with open(req, "r") as f:
+                    dep_files.append(
+                        DependencyFile(
+                            file_path=req,
+                            dependencies=self.parse_requirements(f.read()),
+                        )
+                    )
+            deps_to_scan = [d for d_file in dep_files for d in d_file.dependencies]
+            results = self.scan_dependencies(deps_to_scan, rules, callback)
+            return dep_files, results
+        except Exception as e:
+            log.error(f"Error while scanning. Received {e}")
+            raise e
+
+    @abstractmethod
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
+        pass
+
+    @abstractmethod
+    def find_requirements(
+        self,
+        directory: str,
+    ) -> list[str]:  # returns paths of files
+        pass