scanoss 1.20.6__py3-none-any.whl → 1.23.0__py3-none-any.whl

scanoss/scanners/folder_hasher.py

@@ -0,0 +1,302 @@
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Literal, Optional
+
+from progress.bar import Bar
+
+from scanoss.file_filters import FileFilters
+from scanoss.scanoss_settings import ScanossSettings
+from scanoss.scanossbase import ScanossBase
+from scanoss.utils.abstract_presenter import AbstractPresenter
+from scanoss.utils.crc64 import CRC64
+from scanoss.utils.simhash import WordFeatureSet, fingerprint, simhash, vectorize_bytes
+
+MINIMUM_FILE_COUNT = 8
+MINIMUM_CONCATENATED_NAME_LENGTH = 32
+MINIMUM_FILE_NAME_LENGTH = 32
+
+
+class DirectoryNode:
+    """
+    Represents a node in the directory tree for folder hashing.
+    """
+
+    def __init__(self, path: str):
+        self.path = path
+        self.is_dir = True
+        self.children: Dict[str, DirectoryNode] = {}
+        self.files: List[DirectoryFile] = []
+
+
+class DirectoryFile:
+    """
+    Represents a file in the directory tree for folder hashing.
+    """
+
+    def __init__(self, path: str, key: bytes, key_str: str):
+        self.path = path
+        self.key = key
+        self.key_str = key_str
+
+
+@dataclass
+class FolderHasherConfig:
+    debug: bool = False
+    trace: bool = False
+    quiet: bool = False
+    output_file: Optional[str] = None
+    output_format: Literal['json'] = 'json'
+    settings_file: Optional[str] = None
+    skip_settings_file: bool = False
+
+
+def create_folder_hasher_config_from_args(args) -> FolderHasherConfig:
+    return FolderHasherConfig(
+        debug=args.debug,
+        trace=args.trace,
+        quiet=args.quiet,
+        output_file=getattr(args, 'output', None),
+        output_format=getattr(args, 'format', 'json'),
+        settings_file=getattr(args, 'settings', None),
+        skip_settings_file=getattr(args, 'skip_settings_file', False),
+    )
+
+
+class FolderHasher:
+    """
+    Folder Hasher.
+
+    This class is used to produce a folder hash for a given directory.
+
+    It builds a directory tree (DirectoryNode) and computes the associated
+    hash data for the folder.
+    """
+
+    def __init__(
+        self,
+        scan_dir: str,
+        config: Optional[FolderHasherConfig] = None,
+        scanoss_settings: Optional[ScanossSettings] = None,
+    ):
+        self.base = ScanossBase(
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+        )
+        self.file_filters = FileFilters(
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+            scanoss_settings=scanoss_settings,
+            is_folder_hashing_scan=True,
+        )
+        self.presenter = FolderHasherPresenter(
+            self,
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+        )
+
+        self.scan_dir = scan_dir
+        self.tree = None
+
+    def hash_directory(self, path: str) -> dict:
+        """
+        Generate the folder hashing request structure from a directory path.
+
+        This method builds a directory tree (DirectoryNode) and computes the associated
+        hash data for the folder.
+
+        Args:
+            path (str): The root directory path.
+
+        Returns:
+            dict: The folder hash request structure.
+        """
+
+        root_node = self._build_root_node(path)
+        tree = self._hash_calc_from_node(root_node)
+
+        self.tree = tree
+
+        return tree
+
+    def _build_root_node(self, path: str) -> DirectoryNode:
+        """
+        Build a directory tree from the given path with file information.
+
+        The tree includes DirectoryNode objects populated with filtered file items,
+        each containing their relative path and CRC64 hash key.
+
+        Args:
+            path (str): The directory path to build the tree from.
+
+        Returns:
+            DirectoryNode: The root node representing the directory.
+        """
+        root = Path(path).resolve()
+        root_node = DirectoryNode(str(root))
+
+        all_files = [
+            f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MINIMUM_FILE_NAME_LENGTH
+        ]
+        filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root))
+
+        # Sort the files by name to ensure the hash is the same for the same folder
+        filtered_files.sort()
+
+        bar = Bar('Hashing files...', max=len(filtered_files))
+        for file_path in filtered_files:
+            try:
+                file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path
+                full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj
+
+                self.base.print_debug(f'\nHashing file {str(full_file_path)}')
+
+                file_bytes = full_file_path.read_bytes()
+                key = CRC64.get_hash_buff(file_bytes)
+                key_str = ''.join(f'{b:02x}' for b in key)
+                rel_path = str(full_file_path.relative_to(root))
+
+                file_item = DirectoryFile(rel_path, key, key_str)
+
+                current_node = root_node
+                for part in Path(rel_path).parent.parts:
+                    child_path = str(Path(current_node.path) / part)
+                    if child_path not in current_node.children:
+                        current_node.children[child_path] = DirectoryNode(child_path)
+                    current_node = current_node.children[child_path]
+                    current_node.files.append(file_item)
+
+                root_node.files.append(file_item)
+
+            except Exception as e:
+                self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}')
+
+            bar.next()
+
+        bar.finish()
+        return root_node
+
+    def _hash_calc_from_node(self, node: DirectoryNode) -> dict:
+        """
+        Recursively compute folder hash data for a directory node.
+
+        The hash data includes the path identifier, simhash for file names,
+        simhash for file content, and children node hash information.
+
+        Args:
+            node (DirectoryNode): The directory node to compute the hash for.
+
+        Returns:
+            dict: The computed hash data for the node.
+        """
+        hash_data = self._hash_calc(node)
+
+        return {
+            'path_id': node.path,
+            'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None,
+            'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None,
+            'children': [self._hash_calc_from_node(child) for child in node.children.values()],
+        }
+
+    def _hash_calc(self, node: DirectoryNode) -> dict:
+        """
+        Compute folder hash values for a given directory node.
+
+        The method aggregates unique file keys and sorted file names to generate
+        simhash-based hash values for both file names and file contents.
+
+        The most significant byte of the name simhash is then replaced by a computed head value.
+
+        Args:
+            node (DirectoryNode): The directory node containing file items.
+
+        Returns:
+            dict: A dictionary with 'name_hash' and 'content_hash' keys.
+        """
+        processed_hashes = set()
+        file_hashes = []
+        selected_names = []
+
+        for file in node.files:
+            key_str = file.key_str
+            if key_str in processed_hashes:
+                continue
+            processed_hashes.add(key_str)
+
+            selected_names.append(os.path.basename(file.path))
+
+            file_key = bytes(file.key)
+            file_hashes.append(file_key)
+
+        if len(selected_names) < MINIMUM_FILE_COUNT:
+            return {
+                'name_hash': None,
+                'content_hash': None,
+            }
+
+        selected_names.sort()
+        concatenated_names = ''.join(selected_names)
+
+        if len(concatenated_names.encode('utf-8')) < MINIMUM_CONCATENATED_NAME_LENGTH:
+            return {
+                'name_hash': None,
+                'content_hash': None,
+            }
+
+        names_simhash = simhash(WordFeatureSet(concatenated_names.encode('utf-8')))
+        content_simhash = fingerprint(vectorize_bytes(file_hashes))
+
+        return {
+            'name_hash': names_simhash,
+            'content_hash': content_simhash,
+        }
+
+    def present(self, output_format: str = None, output_file: str = None):
+        """Present the hashed tree in the selected format"""
+        self.presenter.present(output_format=output_format, output_file=output_file)
+
+
+class FolderHasherPresenter(AbstractPresenter):
+    """
+    FolderHasher presenter class
+    Handles the presentation of the folder hashing scan results
+    """
+
+    def __init__(self, folder_hasher: FolderHasher, **kwargs):
+        super().__init__(**kwargs)
+        self.folder_hasher = folder_hasher
+
+    def _format_json_output(self) -> str:
+        """
+        Format the scan output data into a JSON object
+
+        Returns:
+            str: The formatted JSON string
+        """
+        return json.dumps(self.folder_hasher.tree, indent=2)
+
+    def _format_plain_output(self) -> str:
+        """
+        Format the scan output data into a plain text string
+        """
+        return (
+            json.dumps(self.folder_hasher.tree, indent=2)
+            if isinstance(self.folder_hasher.tree, dict)
+            else str(self.folder_hasher.tree)
+        )
+
+    def _format_cyclonedx_output(self) -> str:
+        raise NotImplementedError('CycloneDX output is not implemented')
+
+    def _format_spdxlite_output(self) -> str:
+        raise NotImplementedError('SPDXlite output is not implemented')
+
+    def _format_csv_output(self) -> str:
+        raise NotImplementedError('CSV output is not implemented')
+
+    def _format_raw_output(self) -> str:
+        raise NotImplementedError('Raw output is not implemented')

scanoss/scanners/scanner_config.py

@@ -0,0 +1,73 @@
+"""
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2025, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+
+from pypac.parser import PACFile
+
+from scanoss.constants import (
+    DEFAULT_NB_THREADS,
+    DEFAULT_POST_SIZE,
+    DEFAULT_SC_TIMEOUT,
+    DEFAULT_TIMEOUT,
+)
+
+
+@dataclass
+class ScannerConfig:
+    debug: bool = False
+    trace: bool = False
+    quiet: bool = False
+    api_key: Optional[str] = None
+    url: Optional[str] = None
+    grpc_url: Optional[str] = None
+    post_size: int = DEFAULT_POST_SIZE
+    timeout: int = DEFAULT_TIMEOUT
+    sc_timeout: int = DEFAULT_SC_TIMEOUT
+    nb_threads: int = DEFAULT_NB_THREADS
+    proxy: Optional[str] = None
+    grpc_proxy: Optional[str] = None
+
+    ca_cert: Optional[str] = None
+    pac: Optional[PACFile] = None
+
+
+def create_scanner_config_from_args(args) -> ScannerConfig:
+    return ScannerConfig(
+        debug=args.debug,
+        trace=args.trace,
+        quiet=args.quiet,
+        api_key=getattr(args, 'key', None),
+        url=getattr(args, 'api_url', None),
+        grpc_url=getattr(args, 'grpc_url', None),
+        post_size=getattr(args, 'post_size', DEFAULT_POST_SIZE),
+        timeout=getattr(args, 'timeout', DEFAULT_TIMEOUT),
+        sc_timeout=getattr(args, 'sc_timeout', DEFAULT_SC_TIMEOUT),
+        nb_threads=getattr(args, 'nb_threads', DEFAULT_NB_THREADS),
+        proxy=getattr(args, 'proxy', None),
+        grpc_proxy=getattr(args, 'grpc_proxy', None),
+        ca_cert=getattr(args, 'ca_cert', None),
+        pac=getattr(args, 'pac', None),
+    )

scanoss/scanners/scanner_hfh.py

@@ -0,0 +1,173 @@
+"""
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2025, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+"""
+
+import json
+import threading
+import time
+from typing import Dict, Optional
+
+from progress.spinner import Spinner
+
+from scanoss.file_filters import FileFilters
+from scanoss.scanners.folder_hasher import FolderHasher
+from scanoss.scanners.scanner_config import ScannerConfig
+from scanoss.scanoss_settings import ScanossSettings
+from scanoss.scanossbase import ScanossBase
+from scanoss.scanossgrpc import ScanossGrpc
+from scanoss.utils.abstract_presenter import AbstractPresenter
+
+
+class ScannerHFH:
+    """
+    Folder Hashing Scanner.
+
+    This scanner processes a directory, computes CRC64 hashes for the files,
+    and calculates simhash values based on file names and content to detect folder-level similarities.
+    """
+
+    def __init__(
+        self,
+        scan_dir: str,
+        config: ScannerConfig,
+        client: Optional[ScanossGrpc] = None,
+        scanoss_settings: Optional[ScanossSettings] = None,
+    ):
+        """
+        Initialize the ScannerHFH.
+
+        Args:
+            scan_dir (str): The directory to be scanned.
+            config (ScannerConfig): Configuration parameters for the scanner.
+            client (ScanossGrpc): gRPC client for communicating with the scanning service.
+            scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss.
+        """
+        self.base = ScanossBase(
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+        )
+        self.presenter = ScannerHFHPresenter(
+            self,
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+        )
+        self.file_filters = FileFilters(
+            debug=config.debug,
+            trace=config.trace,
+            quiet=config.quiet,
+            scanoss_settings=scanoss_settings,
+        )
+        self.folder_hasher = FolderHasher(
+            scan_dir=scan_dir,
+            config=config,
+            scanoss_settings=scanoss_settings,
+        )
+
+        self.scan_dir = scan_dir
+        self.client = client
+        self.scan_results = None
+        self.best_match = False
+        self.threshold = 100
+
+    def scan(self) -> Optional[Dict]:
+        """
+        Scan the provided directory using the folder hashing algorithm.
+
+        Returns:
+            Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs.
+        """
+        hfh_request = {
+            'root': self.folder_hasher.hash_directory(self.scan_dir),
+            'threshold': self.threshold,
+            'best_match': self.best_match,
+        }
+
+        spinner = Spinner('Scanning folder...')
+        stop_spinner = False
+
+        def spin():
+            while not stop_spinner:
+                spinner.next()
+                time.sleep(0.1)
+
+        spinner_thread = threading.Thread(target=spin)
+        spinner_thread.start()
+
+        try:
+            response = self.client.folder_hash_scan(hfh_request)
+            if response:
+                self.scan_results = response
+        finally:
+            stop_spinner = True
+            spinner_thread.join()
+            spinner.finish()
+
+        return self.scan_results
+
+    def present(self, output_format: str = None, output_file: str = None):
+        """Present the results in the selected format"""
+        self.presenter.present(output_format=output_format, output_file=output_file)
+
+
+class ScannerHFHPresenter(AbstractPresenter):
+    """
+    ScannerHFH presenter class
+    Handles the presentation of the folder hashing scan results
+    """
+
+    def __init__(self, scanner: ScannerHFH, **kwargs):
+        super().__init__(**kwargs)
+        self.scanner = scanner
+
+    def _format_json_output(self) -> str:
+        """
+        Format the scan output data into a JSON object
+
+        Returns:
+            str: The formatted JSON string
+        """
+        return json.dumps(self.scanner.scan_results, indent=2)
+
+    def _format_plain_output(self) -> str:
+        """
+        Format the scan output data into a plain text string
+        """
+        return (
+            json.dumps(self.scanner.scan_results, indent=2)
+            if isinstance(self.scanner.scan_results, dict)
+            else str(self.scanner.scan_results)
+        )
+
+    def _format_cyclonedx_output(self) -> str:
+        raise NotImplementedError('CycloneDX output is not implemented')
+
+    def _format_spdxlite_output(self) -> str:
+        raise NotImplementedError('SPDXlite output is not implemented')
+
+    def _format_csv_output(self) -> str:
+        raise NotImplementedError('CSV output is not implemented')
+
+    def _format_raw_output(self) -> str:
+        raise NotImplementedError('Raw output is not implemented')

scanoss/scanoss_settings.py

@@ -24,13 +24,17 @@ SPDX-License-Identifier: MIT
 
 import json
 from pathlib import Path
-from typing import List, TypedDict
+from typing import List, Optional, TypedDict
 
 import importlib_resources
 from jsonschema import validate
 
 from .scanossbase import ScanossBase
-from .utils.file import JSON_ERROR_FILE_NOT_FOUND, JSON_ERROR_FILE_EMPTY, validate_json_file
+from .utils.file import (
+    JSON_ERROR_FILE_EMPTY,
+    JSON_ERROR_FILE_NOT_FOUND,
+    validate_json_file,
+)
 
 DEFAULT_SCANOSS_JSON_FILE = Path('scanoss.json')
 
@@ -96,7 +100,7 @@ class ScanossSettings(ScanossBase):
         if filepath:
             self.load_json_file(filepath)
 
-    def load_json_file(self, filepath: 'str | None' = None, scan_root: 'str | None' = None) -> 'ScanossSettings':
+    def load_json_file(self, filepath: Optional[str] = None, scan_root: Optional[str] = None) -> 'ScanossSettings':
         """
         Load the scan settings file. If no filepath is provided, scanoss.json will be used as default.
 
@@ -118,7 +122,7 @@ class ScanossSettings(ScanossBase):
 
         result = validate_json_file(json_file)
         if not result.is_valid:
-            if result.error_code == JSON_ERROR_FILE_NOT_FOUND or result.error_code == JSON_ERROR_FILE_EMPTY:
+            if result.error_code in (JSON_ERROR_FILE_NOT_FOUND, JSON_ERROR_FILE_EMPTY):
                 self.print_msg(
                     f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...'
                 )
@@ -235,7 +239,7 @@ class ScanossSettings(ScanossBase):
             include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include()))
             replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace()))
             self.print_debug(
-                f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n"
+                f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n"  # noqa: E501
                 f'From Include list: {[entry["purl"] for entry in include_bom_entries]} \n'
                 f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n'
             )

scanoss/scanossbase.py

@@ -80,20 +80,26 @@ class ScanossBase:
             **kwargs,
         )
 
-    def print_to_file_or_stdout(self, msg: str, file: str = None):
+    def print_to_file_or_stdout(self, content: str, file: str = None):
         """
         Print message to file if provided or stdout
         """
+        if not content:
+            return
+
         if file:
             with open(file, 'w') as f:
-                f.write(msg)
+                f.write(content)
         else:
-            self.print_stdout(msg)
+            self.print_stdout(content)
 
     def print_to_file_or_stderr(self, msg: str, file: str = None):
         """
         Print message to file if provided or stderr
         """
+        if not msg:
+            return
+
         if file:
             with open(file, 'w') as f:
                 f.write(msg)