scanoss 1.27.1__py3-none-any.whl → 1.43.1__py3-none-any.whl

scanoss/scanners/folder_hasher.py

@@ -6,6 +6,7 @@ from typing import Dict, List, Literal, Optional
 
 from progress.bar import Bar
 
+from scanoss.constants import DEFAULT_HFH_DEPTH
 from scanoss.file_filters import FileFilters
 from scanoss.scanoss_settings import ScanossSettings
 from scanoss.scanossbase import ScanossBase
@@ -15,8 +16,6 @@ from scanoss.utils.simhash import WordFeatureSet, fingerprint, simhash, vectoriz
 
 MINIMUM_FILE_COUNT = 8
 MINIMUM_CONCATENATED_NAME_LENGTH = 32
-MAXIMUM_FILE_NAME_LENGTH = 32
-
 
 class DirectoryNode:
     """
@@ -72,6 +71,12 @@ class FolderHasher:
 
     It builds a directory tree (DirectoryNode) and computes the associated
     hash data for the folder.
+
+    Args:
+        scan_dir (str): The directory to be hashed.
+        config (FolderHasherConfig): Configuration parameters for the folder hasher.
+        scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss.
+        depth (int): How many levels to hash from the root directory (default: 1).
     """
 
     def __init__(
@@ -79,6 +84,7 @@ class FolderHasher:
         scan_dir: str,
         config: FolderHasherConfig,
         scanoss_settings: Optional[ScanossSettings] = None,
+        depth: int = DEFAULT_HFH_DEPTH,
     ):
         self.base = ScanossBase(
             debug=config.debug,
@@ -101,6 +107,7 @@ class FolderHasher:
 
         self.scan_dir = scan_dir
         self.tree = None
+        self.depth = depth
 
     def hash_directory(self, path: str) -> dict:
         """
@@ -123,7 +130,10 @@ class FolderHasher:
 
         return tree
 
-    def _build_root_node(self, path: str) -> DirectoryNode:
+    def _build_root_node(
+        self,
+        path: str,
+    ) -> DirectoryNode:
         """
         Build a directory tree from the given path with file information.
 
@@ -140,47 +150,48 @@ class FolderHasher:
         root_node = DirectoryNode(str(root))
 
         all_files = [
-            f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MAXIMUM_FILE_NAME_LENGTH
+            f for f in root.rglob('*') if f.is_file()
         ]
         filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root))
 
         # Sort the files by name to ensure the hash is the same for the same folder
         filtered_files.sort()
 
-        bar = Bar('Hashing files...', max=len(filtered_files))
-        for file_path in filtered_files:
-            try:
-                file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path
-                full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj
+        bar_ctx = Bar('Hashing files...', max=len(filtered_files))
 
-                self.base.print_debug(f'\nHashing file {str(full_file_path)}')
+        with bar_ctx as bar:
+            full_file_path = ''
+            for file_path in filtered_files:
+                try:
+                    file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path
+                    full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj
 
-                file_bytes = full_file_path.read_bytes()
-                key = CRC64.get_hash_buff(file_bytes)
-                key_str = ''.join(f'{b:02x}' for b in key)
-                rel_path = str(full_file_path.relative_to(root))
+                    self.base.print_debug(f'\nHashing file {str(full_file_path)}')
 
-                file_item = DirectoryFile(rel_path, key, key_str)
+                    file_bytes = full_file_path.read_bytes()
+                    key = CRC64.get_hash_buff(file_bytes)
+                    key_str = ''.join(f'{b:02x}' for b in key)
+                    rel_path = str(full_file_path.relative_to(root))
 
-                current_node = root_node
-                for part in Path(rel_path).parent.parts:
-                    child_path = str(Path(current_node.path) / part)
-                    if child_path not in current_node.children:
-                        current_node.children[child_path] = DirectoryNode(child_path)
-                    current_node = current_node.children[child_path]
-                    current_node.files.append(file_item)
+                    file_item = DirectoryFile(rel_path, key, key_str)
 
-                root_node.files.append(file_item)
+                    current_node = root_node
+                    for part in Path(rel_path).parent.parts:
+                        child_path = str(Path(current_node.path) / part)
+                        if child_path not in current_node.children:
+                            current_node.children[child_path] = DirectoryNode(child_path)
+                        current_node = current_node.children[child_path]
+                        current_node.files.append(file_item)
 
-            except Exception as e:
-                self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}')
+                    root_node.files.append(file_item)
 
-            bar.next()
+                except Exception as e:
+                    self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}')
 
-        bar.finish()
+                bar.next()
         return root_node
 
-    def _hash_calc_from_node(self, node: DirectoryNode) -> dict:
+    def _hash_calc_from_node(self, node: DirectoryNode, current_depth: int = 1) -> dict:
         """
         Recursively compute folder hash data for a directory node.
 
@@ -189,12 +200,13 @@ class FolderHasher:
 
         Args:
             node (DirectoryNode): The directory node to compute the hash for.
+            current_depth (int): The current depth level (1-based, root is depth 1).
 
         Returns:
             dict: The computed hash data for the node.
         """
         hash_data = self._hash_calc(node)
-        
+
         # Safely calculate relative path
         try:
             node_path = Path(node.path).resolve()
@@ -204,13 +216,18 @@ class FolderHasher:
             # If relative_to fails, use the node path as is or a fallback
             rel_path = Path(node.path).name if node.path else Path('.')
 
+        # Only process children if we haven't reached the depth limit
+        children = []
+        if current_depth < self.depth:
+            children = [self._hash_calc_from_node(child, current_depth + 1) for child in node.children.values()]
+
         return {
             'path_id': str(rel_path),
             'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None,
             'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None,
             'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None,
             'lang_extensions': hash_data['lang_extensions'],
-            'children': [self._hash_calc_from_node(child) for child in node.children.values()],
+            'children': children,
         }
 
     def _hash_calc(self, node: DirectoryNode) -> dict:
@@ -237,8 +254,6 @@ class FolderHasher:
 
         for file in node.files:
             key_str = file.key_str
-            if key_str in processed_hashes:
-                continue
 
             file_name = os.path.basename(file.path)
 

scanoss/scanners/scanner_hfh.py

@@ -29,7 +29,12 @@ from typing import Dict, Optional
 
 from progress.spinner import Spinner
 
-from scanoss.constants import DEFAULT_HFH_RANK_THRESHOLD
+from scanoss.constants import (
+    DEFAULT_HFH_DEPTH,
+    DEFAULT_HFH_MIN_ACCEPTED_SCORE,
+    DEFAULT_HFH_RANK_THRESHOLD,
+    DEFAULT_HFH_RECURSIVE_THRESHOLD,
+)
 from scanoss.cyclonedx import CycloneDx
 from scanoss.file_filters import FileFilters
 from scanoss.scanners.folder_hasher import FolderHasher
@@ -48,13 +53,17 @@ class ScannerHFH:
     and calculates simhash values based on file names and content to detect folder-level similarities.
     """
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         scan_dir: str,
         config: ScannerConfig,
         client: Optional[ScanossGrpc] = None,
         scanoss_settings: Optional[ScanossSettings] = None,
         rank_threshold: int = DEFAULT_HFH_RANK_THRESHOLD,
+        depth: int = DEFAULT_HFH_DEPTH,
+        recursive_threshold: float = DEFAULT_HFH_RECURSIVE_THRESHOLD,
+        min_accepted_score: float = DEFAULT_HFH_MIN_ACCEPTED_SCORE,
+        use_grpc: bool = False,
     ):
         """
         Initialize the ScannerHFH.
@@ -65,6 +74,9 @@ class ScannerHFH:
             client (ScanossGrpc): gRPC client for communicating with the scanning service.
             scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss.
             rank_threshold (int): Get results with rank below this threshold (default: 5).
+            depth (int): How many levels to scan (default: 1).
+            recursive_threshold (float): Minimum score threshold to consider a match (default: 0.25).
+            min_accepted_score (float): Only show results with a score at or above this threshold (default: 0.15).
         """
         self.base = ScanossBase(
             debug=config.debug,
@@ -87,12 +99,29 @@ class ScannerHFH:
             scan_dir=scan_dir,
             config=config,
             scanoss_settings=scanoss_settings,
+            depth=depth,
         )
 
         self.scan_dir = scan_dir
         self.client = client
         self.scan_results = None
         self.rank_threshold = rank_threshold
+        self.recursive_threshold = recursive_threshold
+        self.min_accepted_score = min_accepted_score
+        self.use_grpc = use_grpc
+
+    def _execute_grpc_scan(self, hfh_request: Dict) -> None:
+        """
+        Execute folder hash scan.
+
+        Args:
+            hfh_request: Request dictionary for the gRPC call
+        """
+        try:
+            self.scan_results = self.client.folder_hash_scan(hfh_request, self.use_grpc)
+        except Exception as e:
+            self.base.print_stderr(f'Error during folder hash scan: {e}')
+            self.scan_results = None
 
     def scan(self) -> Optional[Dict]:
         """
@@ -102,29 +131,23 @@ class ScannerHFH:
             Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs.
         """
         hfh_request = {
-            'root': self.folder_hasher.hash_directory(self.scan_dir),
+            'root': self.folder_hasher.hash_directory(path=self.scan_dir),
             'rank_threshold': self.rank_threshold,
+            'recursive_threshold': self.recursive_threshold,
+            'min_accepted_score': self.min_accepted_score,
         }
 
-        spinner = Spinner('Scanning folder...')
-        stop_spinner = False
+        spinner_ctx = Spinner('Scanning folder...')
 
-        def spin():
-            while not stop_spinner:
+        with spinner_ctx as spinner:
+            grpc_thread = threading.Thread(target=self._execute_grpc_scan, args=(hfh_request,))
+            grpc_thread.start()
+
+            while grpc_thread.is_alive():
                 spinner.next()
                 time.sleep(0.1)
 
-        spinner_thread = threading.Thread(target=spin)
-        spinner_thread.start()
-
-        try:
-            response = self.client.folder_hash_scan(hfh_request)
-            if response:
-                self.scan_results = response
-        finally:
-            stop_spinner = True
-            spinner_thread.join()
-            spinner.finish()
+            grpc_thread.join()
 
         return self.scan_results
 
@@ -194,7 +217,12 @@ class ScannerHFHPresenter(AbstractPresenter):
                 ]
             }
 
+            get_vulnerabilities_json_request = {
+                'components': [{'purl': purl, 'requirement': best_match_version['version']}],
+            }
+
             decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request)
+            vulnerabilities = self.scanner.client.get_vulnerabilities_json(get_vulnerabilities_json_request)
 
             cdx = CycloneDx(self.base.debug)
             scan_results = {}
@@ -205,6 +233,10 @@ class ScannerHFHPresenter(AbstractPresenter):
                 error_msg = 'ERROR: Failed to produce CycloneDX output'
                 self.base.print_stderr(error_msg)
                 return None
+
+            if vulnerabilities:
+                cdx_output = cdx.append_vulnerabilities(cdx_output, vulnerabilities, purl)
+
             return json.dumps(cdx_output, indent=2)
         except Exception as e:
             self.base.print_stderr(f'ERROR: Failed to get license information: {e}')

scanoss/scanoss_settings.py

@@ -172,7 +172,7 @@ class ScanossSettings(ScanossBase):
 
     def _get_bom(self):
         """
-        Get the Billing of Materials from the settings file
+        Get the Bill of Materials from the settings file
         Returns:
             dict: If using scanoss.json
             list: If using SBOM.json
@@ -196,6 +196,17 @@ class ScanossSettings(ScanossBase):
             return self._get_bom()
         return self._get_bom().get('include', [])
 
+
+    def get_bom_exclude(self) -> List[BomEntry]:
+        """
+        Get the list of components to exclude from the scan
+        Returns:
+            list: List of components to exclude from the scan
+        """
+        if self.settings_file_type == 'legacy':
+            return self._get_bom()
+        return self._get_bom().get('exclude', [])
+
     def get_bom_remove(self) -> List[BomEntry]:
         """
         Get the list of components to remove from the scan
@@ -225,8 +236,8 @@ class ScanossSettings(ScanossBase):
         if not self.data:
             return None
         return {
-            'scan_type': self.scan_type,
             'assets': json.dumps(self._get_sbom_assets()),
+            'scan_type': self.scan_type,
         }
 
     def _get_sbom_assets(self):
@@ -235,7 +246,18 @@ class ScanossSettings(ScanossBase):
         Returns:
             List: List of SBOM assets
         """
-        if self.scan_type == 'identify':
+
+        if self.settings_file_type == 'new':
+            if len(self.get_bom_include()):
+                self.scan_type = 'identify'
+                include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include()))
+                return {"components": include_bom_entries}
+            elif len(self.get_bom_exclude()):
+                self.scan_type = 'blacklist'
+                exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude()))
+                return {"components": exclude_bom_entries}
+
+        if self.settings_file_type == 'legacy' and self.scan_type == 'identify':            # sbom-identify.json
             include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include()))
             replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace()))
             self.print_debug(
@@ -244,6 +266,14 @@ class ScanossSettings(ScanossBase):
                 f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n'
             )
             return include_bom_entries + replace_bom_entries
+
+        if self.settings_file_type == 'legacy' and self.scan_type == 'blacklist':            # sbom-identify.json
+            exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude()))
+            self.print_debug(
+                f"Scan type set to 'blacklist'. Adding {len(exclude_bom_entries)} components as context to the scan. \n"  # noqa: E501
+                f'From Exclude list: {[entry["purl"] for entry in exclude_bom_entries]} \n')
+            return exclude_bom_entries
+
         return self.normalize_bom_entries(self.get_bom_remove())
 
     @staticmethod

scanoss/scanossapi.py

@@ -22,23 +22,23 @@ SPDX-License-Identifier: MIT
   THE SOFTWARE.
 """
 
+import http.client as http_client
 import logging
 import os
 import sys
 import time
+import uuid
 from json.decoder import JSONDecodeError
+
 import requests
-import uuid
-import http.client as http_client
 import urllib3
-
 from pypac import PACSession
 from pypac.parser import PACFile
 from urllib3.exceptions import InsecureRequestWarning
 
-from .scanossbase import ScanossBase
 from . import __version__
-
+from .constants import DEFAULT_TIMEOUT, MIN_TIMEOUT
+from .scanossbase import ScanossBase
 
 DEFAULT_URL = 'https://api.osskb.org/scan/direct'  # default free service URL
 DEFAULT_URL2 = 'https://api.scanoss.com/scan/direct'  # default premium service URL
@@ -52,7 +52,7 @@ class ScanossApi(ScanossBase):
     Currently support posting scan requests to the SCANOSS streaming API
     """
 
-    def __init__(  # noqa: PLR0913, PLR0915
+    def __init__(  # noqa: PLR0912, PLR0913, PLR0915
         self,
         scan_format: str = None,
         flags: str = None,
@@ -61,7 +61,7 @@ class ScanossApi(ScanossBase):
         debug: bool = False,
         trace: bool = False,
         quiet: bool = False,
-        timeout: int = 180,
+        timeout: int = DEFAULT_TIMEOUT,
         ver_details: str = None,
         ignore_cert_errors: bool = False,
         proxy: str = None,
@@ -78,7 +78,7 @@ class ScanossApi(ScanossBase):
         :param api_key: API Key (default None)
         :param debug: Enable debug (default False)
         :param trace: Enable trace (default False)
-        :param quiet: Enable quite mode (default False)
+        :param quiet: Enable quiet mode (default False)
 
         To set a custom certificate use:
             REQUESTS_CA_BUNDLE=/path/to/cert.pem
@@ -87,30 +87,28 @@ class ScanossApi(ScanossBase):
             HTTPS_PROXY='http://<ip>:<port>'
         """
         super().__init__(debug, trace, quiet)
-        self.url = url
-        self.api_key = api_key
         self.sbom = None
         self.scan_format = scan_format if scan_format else 'plain'
         self.flags = flags
-        self.timeout = timeout if timeout > 5 else 180
+        self.timeout = timeout if timeout > MIN_TIMEOUT else DEFAULT_TIMEOUT
         self.retry_limit = retry if retry >= 0 else 5
         self.ignore_cert_errors = ignore_cert_errors
         self.req_headers = req_headers if req_headers else {}
         self.headers = {}
-
+        # Set the correct URL/API key combination
+        self.url = url if url else SCANOSS_SCAN_URL
+        self.api_key = api_key if api_key else SCANOSS_API_KEY
+        if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'):
+            self.url = DEFAULT_URL2  # API key specific and no alternative URL, so use the default premium
         if ver_details:
             self.headers['x-scanoss-client'] = ver_details
         if self.api_key:
             self.headers['X-Session'] = self.api_key
             self.headers['x-api-key'] = self.api_key
-        self.headers['User-Agent'] = f'scanoss-py/{__version__}'
-        self.headers['user-agent'] = f'scanoss-py/{__version__}'
-        self.load_generic_headers()
-
-        self.url = url if url else SCANOSS_SCAN_URL
-        self.api_key = api_key if api_key else SCANOSS_API_KEY
-        if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'):
-            self.url = DEFAULT_URL2  # API key specific and no alternative URL, so use the default premium
+        user_agent = f'scanoss-py/{__version__}'
+        self.headers['User-Agent'] = user_agent
+        self.headers['user-agent'] = user_agent
+        self.load_generic_headers(url)
 
         if self.trace:
             logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
@@ -133,7 +131,7 @@ class ScanossApi(ScanossBase):
         if self.proxies:
             self.session.proxies = self.proxies
 
-    def scan(self, wfp: str, context: str = None, scan_id: int = None):
+    def scan(self, wfp: str, context: str = None, scan_id: int = None):  # noqa: PLR0912, PLR0915
         """
         Scan the specified WFP and return the JSON object
         :param wfp: WFP to scan
@@ -192,7 +190,7 @@ class ScanossApi(ScanossBase):
                     else:
                         self.print_stderr(f'Warning: No response received from {self.url}. Retrying...')
                         time.sleep(5)
-                elif r.status_code == 503:  # Service limits have most likely been reached
+                elif r.status_code == requests.codes.service_unavailable:  # Service limits most likely reached
                     self.print_stderr(
                         f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to '
                         f'service limits being exceeded'
@@ -202,7 +200,7 @@ class ScanossApi(ScanossBase):
                         f'ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected '
                         f'for {self.url} due to service limits being exceeded.'
                     )
-                elif r.status_code >= 400:
+                elif r.status_code >= requests.codes.bad_request:
                     if retry > self.retry_limit:  # No response retry_limit or more times, fail
                         self.save_bad_req_wfp(scan_files, request_id, scan_id)
                         raise Exception(
@@ -269,7 +267,7 @@ class ScanossApi(ScanossBase):
         self.sbom = sbom
         return self
 
-    def load_generic_headers(self):
+    def load_generic_headers(self, url):
         """
          Adds custom headers from req_headers to the headers collection.
 
@@ -279,7 +277,7 @@ class ScanossApi(ScanossBase):
         if self.req_headers:  # Load generic headers
             for key, value in self.req_headers.items():
                 if key == 'x-api-key': # Set premium URL if x-api-key header is set
-                    if not self.url and not os.environ.get('SCANOSS_SCAN_URL'):
+                    if not url and not os.environ.get('SCANOSS_SCAN_URL'):
                         self.url = DEFAULT_URL2  # API key specific and no alternative URL, so use the default premium
                     self.api_key = value
                 self.headers[key] = value

scanoss/scanossbase.py

@@ -50,7 +50,7 @@ class ScanossBase:
 
     def print_msg(self, *args, **kwargs):
         """
-        Print message if quite mode is not enabled
+        Print message if quiet mode is not enabled
         """
         if not self.quiet:
             self.print_stderr(*args, **kwargs)