guarddog 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

guarddog/scanners/go_project_scanner.py

@@ -1,9 +1,12 @@
 import logging
+import os
+import re
 from dataclasses import dataclass
 from typing import List
 
 from guarddog.scanners.go_package_scanner import GoModuleScanner
 from guarddog.scanners.scanner import ProjectScanner
+from guarddog.scanners.scanner import Dependency, DependencyVersion
 
 log = logging.getLogger("guarddog")
 
@@ -26,13 +29,39 @@ class GoDependenciesScanner(ProjectScanner):
     def __init__(self) -> None:
         super().__init__(GoModuleScanner())
 
-    def parse_requirements(self, raw_requirements: str) -> dict[str, set[str]]:
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
         main_mod = self.parse_go_mod_file(raw_requirements)
 
-        return {
-            requirement.module: set([requirement.version])
-            for requirement in main_mod.requirements
-        }
+        dependencies: List[Dependency] = []
+        for dependency in main_mod.requirements:
+            version = dependency.version
+            name = dependency.module
+            idx = next(
+                iter(
+                    [
+                        ix
+                        for ix, line in enumerate(raw_requirements.splitlines())
+                        if name in line
+                    ]
+                ),
+                0,
+            )
+
+            dep_versions = [DependencyVersion(version=version, location=idx + 1)]
+
+            dep = next(
+                filter(
+                    lambda d: d.name == name,
+                    dependencies,
+                ),
+                None,
+            )
+            if not dep:
+                dep = Dependency(name=name, versions=set())
+                dependencies.append(dep)
+
+            dep.versions.update(dep_versions)
+        return dependencies
 
     # Read https://go.dev/ref/mod#go-mod-file to learn more about the go.mod syntax
     def parse_go_mod_file(self, go_mod_content: str) -> GoModule:
@@ -66,3 +95,11 @@ class GoDependenciesScanner(ProjectScanner):
             # TODO: support exclude, replace and retract statements
 
         return GoModule(module, go, toolchain, requirements)
+
+    def find_requirements(self, directory: str) -> list[str]:
+        requirement_files = []
+        for root, dirs, files in os.walk(directory):
+            for name in files:
+                if re.match(r"^go\.mod$", name, flags=re.IGNORECASE):
+                    requirement_files.append(os.path.join(root, name))
+        return requirement_files

guarddog/scanners/npm_package_scanner.py

@@ -17,9 +17,13 @@ class NPMPackageScanner(PackageScanner):
     def __init__(self) -> None:
         super().__init__(Analyzer(ECOSYSTEM.NPM))
 
-    def download_and_get_package_info(self, directory: str, package_name: str, version=None) -> typing.Tuple[dict, str]:
+    def download_and_get_package_info(
+        self, directory: str, package_name: str, version=None
+    ) -> typing.Tuple[dict, str]:
         git_target = None
-        if urlparse(package_name).hostname is not None and package_name.endswith('.git'):
+        if urlparse(package_name).hostname is not None and package_name.endswith(
+            ".git"
+        ):
             git_target = package_name
 
         if not package_name.startswith("@") and package_name.count("/") == 1:
@@ -33,7 +37,9 @@ class NPMPackageScanner(PackageScanner):
         response = requests.get(url)
 
         if response.status_code != 200:
-            raise Exception("Received status code: " + str(response.status_code) + " from npm")
+            raise Exception(
+                "Received status code: " + str(response.status_code) + " from npm"
+            )
         data = response.json()
         if "name" not in data:
             raise Exception(f"Error retrieving package: {package_name}")
@@ -45,7 +51,9 @@ class NPMPackageScanner(PackageScanner):
 
         tarball_url = details["dist"]["tarball"]
         file_extension = pathlib.Path(tarball_url).suffix
-        zippath = os.path.join(directory, package_name.replace("/", "-") + file_extension)
+        zippath = os.path.join(
+            directory, package_name.replace("/", "-") + file_extension
+        )
         unzippedpath = zippath.removesuffix(file_extension)
         self.download_compressed(tarball_url, zippath, unzippedpath)
 

guarddog/scanners/npm_project_scanner.py

@@ -1,11 +1,15 @@
 import json
 import logging
+import os
+import re
+from typing import List
+
 import requests
 from semantic_version import NpmSpec, Version  # type:ignore
 
-from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 from guarddog.scanners.npm_package_scanner import NPMPackageScanner
-from guarddog.scanners.scanner import ProjectScanner
+from guarddog.scanners.scanner import Dependency, DependencyVersion, ProjectScanner
+from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 
 log = logging.getLogger("guarddog")
 
@@ -21,7 +25,7 @@ class NPMRequirementsScanner(ProjectScanner):
     def __init__(self) -> None:
         super().__init__(NPMPackageScanner())
 
-    def parse_requirements(self, raw_requirements: str) -> dict:
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
         """
         Parses requirements.txt specification and finds all valid
         versions of each dependency
@@ -40,8 +44,8 @@ class NPMRequirementsScanner(ProjectScanner):
             }
         """
         package = json.loads(raw_requirements)
-        dependencies = package["dependencies"] if "dependencies" in package else {}
-        dev_dependencies = (
+        dependencies_attr = package["dependencies"] if "dependencies" in package else {}
+        dev_dependencies_attr = (
             package["devDependencies"] if "devDependencies" in package else {}
         )
 
@@ -82,23 +86,63 @@ class NPMRequirementsScanner(ProjectScanner):
             return versions
 
         merged = {}  # type: dict[str, set[str]]
-        for package, selector in list(dependencies.items()) + list(
-            dev_dependencies.items()
+        for package, selector in list(dependencies_attr.items()) + list(
+            dev_dependencies_attr.items()
         ):
             if package not in merged:
                 merged[package] = set()
             merged[package].add(selector)
 
-        results = {}
+        dependencies: List[Dependency] = []
         for package, all_selectors in merged.items():
             versions = set()  # type: set[str]
             for selector in all_selectors:
                 versions = versions.union(
                     get_matched_versions(find_all_versions(package), selector)
                 )
+
             if len(versions) == 0:
                 log.error(f"Package/Version {package} not on NPM\n")
                 continue
 
-            results[package] = versions
-        return results
+            idx = next(
+                iter(
+                    [
+                        ix
+                        for ix, line in enumerate(raw_requirements.splitlines())
+                        if package in line
+                    ]
+                ),
+                0,
+            )
+
+            dep_versions = list(
+                map(
+                    lambda d: DependencyVersion(version=d, location=idx + 1),
+                    versions,
+                )
+            )
+
+            # find the dep with the same name or create a new one
+            dep = next(
+                filter(
+                    lambda d: d.name == package,
+                    dependencies,
+                ),
+                None,
+            )
+            if not dep:
+                dep = Dependency(name=package, versions=set())
+                dependencies.append(dep)
+
+            dep.versions.update(dep_versions)
+
+        return dependencies
+
+    def find_requirements(self, directory: str) -> list[str]:
+        requirement_files = []
+        for root, dirs, files in os.walk(directory):
+            for name in files:
+                if re.match(r"^package\.json$", name, flags=re.IGNORECASE):
+                    requirement_files.append(os.path.join(root, name))
+        return requirement_files

guarddog/scanners/pypi_package_scanner.py

@@ -12,7 +12,9 @@ class PypiPackageScanner(PackageScanner):
     def __init__(self) -> None:
         super().__init__(Analyzer(ECOSYSTEM.PYPI))
 
-    def download_and_get_package_info(self, directory: str, package_name: str, version=None) -> typing.Tuple[dict, str]:
+    def download_and_get_package_info(
+        self, directory: str, package_name: str, version=None
+    ) -> typing.Tuple[dict, str]:
         extract_dir = self.download_package(package_name, directory, version)
         return get_package_info(package_name), extract_dir
 
@@ -40,7 +42,9 @@ class PypiPackageScanner(PackageScanner):
             version = data["info"]["version"]
 
         if version not in releases:
-            raise Exception(f"Version {version} for package {package_name} doesn't exist.")
+            raise Exception(
+                f"Version {version} for package {package_name} doesn't exist."
+            )
 
         files = releases[version]
         url, file_extension = None, None
@@ -52,7 +56,9 @@ class PypiPackageScanner(PackageScanner):
                 break
 
         if not (url and file_extension):
-            raise Exception(f"Compressed file for {package_name} does not exist on PyPI.")
+            raise Exception(
+                f"Compressed file for {package_name} does not exist on PyPI."
+            )
 
         # Path to compressed package
         zippath = os.path.join(directory, package_name + file_extension)

guarddog/scanners/pypi_project_scanner.py

@@ -1,11 +1,14 @@
 import logging
+import os
 import re
-import pkg_resources
+from typing import List
+
+from packaging.requirements import Requirement
 import requests
 from packaging.specifiers import Specifier, Version
 
 from guarddog.scanners.pypi_package_scanner import PypiPackageScanner
-from guarddog.scanners.scanner import ProjectScanner
+from guarddog.scanners.scanner import Dependency, DependencyVersion, ProjectScanner
 from guarddog.utils.config import VERIFY_EXHAUSTIVE_DEPENDENCIES
 
 log = logging.getLogger("guarddog")
@@ -37,17 +40,20 @@ class PypiRequirementsScanner(ProjectScanner):
 
         for line in requirements:
             is_requirement = re.match(r"\w", line)
-            if is_requirement:
-                if "\\" in line:
-                    line = line.replace("\\", "")
 
-                stripped_line = line.strip()
-                if len(stripped_line) > 0:
-                    sanitized_lines.append(stripped_line)
+            if not is_requirement:
+                sanitized_lines.append("")  # empty line to keep the line number
+                continue
+
+            if "\\" in line:
+                line = line.replace("\\", "")
+
+            stripped_line = line.strip()
+            sanitized_lines.append(stripped_line)
 
         return sanitized_lines
 
-    def parse_requirements(self, raw_requirements: str) -> dict[str, set[str]]:
+    def parse_requirements(self, raw_requirements: str) -> List[Dependency]:
         """
         Parses requirements.txt specification and finds all valid
         versions of each dependency
@@ -57,17 +63,10 @@ class PypiRequirementsScanner(ProjectScanner):
 
         Returns:
             dict: mapping of dependencies to valid versions
-
-            ex.
-            {
-                ....
-                <dependency-name>: [0.0.1, 0.0.2, ...],
-                ...
-            }
         """
         requirements = raw_requirements.splitlines()
         sanitized_requirements = self._sanitize_requirements(requirements)
-        dependencies = {}
+        dependencies: List[Dependency] = []
 
         def get_matched_versions(versions: set[str], semver_range: str) -> set[str]:
             """
@@ -77,8 +76,11 @@ class PypiRequirementsScanner(ProjectScanner):
 
             # Filters to specified versions
             try:
-                spec = Specifier(semver_range)
-                result = [Version(m) for m in spec.filter(versions)]
+                matching_versions = versions
+                if semver_range:
+                    spec = Specifier(semver_range)
+                    matching_versions = set(spec.filter(versions))
+                result = [Version(m) for m in matching_versions]
             except ValueError:
                 # use it raw
                 return set([semver_range])
@@ -109,12 +111,11 @@ class PypiRequirementsScanner(ProjectScanner):
             """
             This helper function yields one valid requirement line at a time
             """
-            parsed = pkg_resources.parse_requirements(req)
-            while True:
+            for req_line in req:
+                if not req_line.strip():
+                    continue
                 try:
-                    yield next(parsed)
-                except StopIteration:
-                    break
+                    yield Requirement(req_line)
                 except Exception as e:
                     log.error(
                         f"Error when parsing requirements, received error {str(e)}. This entry will be "
@@ -128,7 +129,7 @@ class PypiRequirementsScanner(ProjectScanner):
                     continue
 
                 versions = get_matched_versions(
-                    find_all_versions(requirement.project_name),
+                    find_all_versions(requirement.name),
                     (
                         requirement.url
                         if requirement.url
@@ -137,13 +138,50 @@ class PypiRequirementsScanner(ProjectScanner):
                 )
 
                 if len(versions) == 0:
-                    log.error(
-                        f"Package/Version {requirement.project_name} not on PyPI\n"
-                    )
+                    log.error(f"Package/Version {requirement.name} not on PyPI\n")
                     continue
 
-                dependencies[requirement.project_name] = versions
+                idx = next(
+                    iter(
+                        [
+                            ix
+                            for ix, line in enumerate(requirements)
+                            if str(requirement) in line
+                        ]
+                    ),
+                    0,
+                )
+
+                dep_versions = list(
+                    map(
+                        lambda d: DependencyVersion(version=d, location=idx + 1),
+                        versions,
+                    )
+                )
+
+                # find the dep with the same name or create a new one
+                dep = next(
+                    filter(
+                        lambda d: d.name == requirement.name,
+                        dependencies,
+                    ),
+                    None,
+                )
+                if not dep:
+                    dep = Dependency(name=requirement.name, versions=set())
+                    dependencies.append(dep)
+
+                dep.versions.update(dep_versions)
+
         except Exception as e:
             log.error(f"Received error {str(e)}")
 
         return dependencies
+
+    def find_requirements(self, directory: str) -> list[str]:
+        requirement_files = []
+        for root, dirs, files in os.walk(directory):
+            for name in files:
+                if re.match(r"^requirements(-dev)?\.txt$", name, flags=re.IGNORECASE):
+                    requirement_files.append(os.path.join(root, name))
+        return requirement_files