scanoss 1.41.0__py3-none-any.whl → 1.42.0__py3-none-any.whl

scanoss/__init__.py

@@ -22,4 +22,4 @@ SPDX-License-Identifier: MIT
   THE SOFTWARE.
 """
 
-__version__ = '1.41.0'
+__version__ = '1.42.0'

scanoss/cli.py

@@ -1096,6 +1096,19 @@ def setup_args() -> None:  # noqa: PLR0912, PLR0915
         p.add_argument('--skip-md5', '-5', type=str, action='append', help='Skip files matching MD5.')
         p.add_argument('--strip-hpsm', '-G', type=str, action='append', help='Strip HPSM string from WFP.')
         p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.')
+        p.add_argument(
+            '--skip-headers',
+            '-skh',
+            action='store_true',
+            help='Skip license headers, comments and imports at the beginning of files.',
+        )
+        p.add_argument(
+            '--skip-headers-limit',
+            '-shl',
+            type=int,
+            default=0,
+            help='Maximum number of lines to skip when filtering headers (default: 0 = no limit).',
+        )
 
     # Global Scan/GRPC options
     for p in [
@@ -1388,6 +1401,8 @@ def wfp(parser, args):
         strip_hpsm_ids=args.strip_hpsm,
         strip_snippet_ids=args.strip_snippet,
         scan_settings=scan_settings,
+        skip_headers=args.skip_headers,
+        skip_headers_limit=args.skip_headers_limit,
     )
     if args.stdin:
         contents = sys.stdin.buffer.read()
@@ -1583,6 +1598,8 @@ def scan(parser, args):  # noqa: PLR0912, PLR0915
         scan_settings=scan_settings,
         req_headers=process_req_headers(args.header),
         use_grpc=args.grpc,
+        skip_headers=args.skip_headers,
+        skip_headers_limit=args.skip_headers_limit,
     )
     if args.wfp:
         if not scanner.is_file_or_snippet_scan():

scanoss/data/build_date.txt

@@ -1 +1 @@
-date: 20251117160705, utime: 1763395625
+date: 20251218123159, utime: 1766061119

scanoss/header_filter.py

@@ -0,0 +1,563 @@
+"""
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2025, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+
+  Line Filter Module - Identifies where real source code implementation begins.
+
+  This module analyzes source code files and determines which lines are:
+  - License headers
+  - Documentation comments
+  - Imports/includes
+  - Blank lines
+
+  And returns the content from where the real implementation begins.
+"""
+
+import re
+from pathlib import Path
+from typing import Optional, Tuple
+
+from .scanossbase import ScanossBase
+
+
+class LanguagePatterns:
+    """
+    Regex patterns for different programming languages.
+
+    This class provides a collection of regex patterns for identifying different
+    programming constructs, handling imports, comments, and license statements
+    across various programming languages. The main purpose of this class is to
+    assist in parsing or analysing code written in different languages efficiently.
+
+    :ivar COMMENT_PATTERNS: A dictionary containing regex patterns to identify
+        single-line and multi-line comments in various programming languages.
+    :ivar IMPORT_PATTERNS: A dictionary mapping programming languages to their
+        respective regex patterns for identifying import statements or package
+        includes it.
+    :ivar LICENSE_KEYWORDS: A list of keywords commonly found in license texts
+        or statements, often used to detect the presence of licensing information.
+    """
+    # Comment patterns (single-line and multi-line start/end)
+    COMMENT_PATTERNS = {
+        # C-style languages: C, C++, Java, JavaScript, TypeScript, Go,
+        # Rust, C#, PHP, Kotlin, Scala, Dart, Objective-C
+        'c_style': {
+            'single_line': r'^\s*//.*$',
+            'multi_start': r'^\s*/\*',
+            'multi_end': r'\*/\s*$',
+            'multi_single': r'^\s*/\*.*\*/\s*$',
+        },
+        # Python, shell scripts, Ruby, Perl, R, Julia, YAML
+        'python_style': {
+            'single_line': r'^\s*#.*$',
+            'doc_string_start': r'^\s*"""',
+            'doc_string_end': r'"""\s*$',
+        },
+        # Lua, SQL, Haskell
+        'lua_style': {
+            'single_line': r'^\s*--.*$',
+            'multi_start': r'^\s*--\[\[',
+            'multi_end': r'\]\]\s*$',
+        },
+        # HTML, XML
+        'html_style': {
+            'multi_start': r'^\s*<!--',
+            'multi_end': r'-->\s*$',
+            'multi_single': r'^\s*<!--.*-->\s*$',
+        },
+    }
+    # Import/include patterns by language
+    IMPORT_PATTERNS = {
+        'python': [
+            r'^\s*import\s+',
+            r'^\s*from\s+.*\s+import\s+',
+        ],
+        'javascript': [
+            r'^\s*import\s+.*\s+from\s+',
+            r'^\s*import\s+["\']',
+            r'^\s*import\s+type\s+',
+            r'^\s*export\s+\*\s+from\s+',
+            r'^\s*export\s+\{.*\}\s+from\s+',
+            r'^\s*const\s+.*\s*=\s*require\(',
+            r'^\s*var\s+.*\s*=\s*require\(',
+            r'^\s*let\s+.*\s*=\s*require\(',
+        ],
+        'typescript': [
+            r'^\s*import\s+',
+            r'^\s*export\s+.*\s+from\s+',
+            r'^\s*import\s+type\s+',
+            r'^\s*import\s+\{.*\}\s+from\s+',
+        ],
+        'java': [
+            r'^\s*import\s+',
+            r'^\s*package\s+',
+        ],
+        'kotlin': [
+            r'^\s*import\s+',
+            r'^\s*package\s+',
+        ],
+        'scala': [
+            r'^\s*import\s+',
+            r'^\s*package\s+',
+        ],
+        'go': [
+            r'^\s*import\s+\(',
+            r'^\s*import\s+"',
+            r'^\s*package\s+',
+            r'^\s*"[^"]*"\s*$',  # Imports inside import () block
+            # Imports with alias: name "package"
+            r'^\s*[a-zA-Z_][a-zA-Z0-9_]*\s+"[^"]*"\s*$',
+            r'^\s*_\s+"[^"]*"\s*$',  # _ "package" imports
+        ],
+        'rust': [
+            r'^\s*use\s+',
+            r'^\s*extern\s+crate\s+',
+            r'^\s*mod\s+',
+        ],
+        'cpp': [
+            r'^\s*#include\s+',
+            r'^\s*#pragma\s+',
+            r'^\s*#ifndef\s+.*_H.*',  # Header guards: #ifndef FOO_H
+            r'^\s*#define\s+.*_H.*',  # Header guards: #define FOO_H
+            # #endif at end of file (may have comment)
+            r'^\s*#endif\s+(//.*)?\s*$',
+        ],
+        'csharp': [
+            r'^\s*using\s+',
+            r'^\s*namespace\s+',
+        ],
+        'php': [
+            r'^\s*use\s+',
+            r'^\s*require\s+',
+            r'^\s*require_once\s+',
+            r'^\s*include\s+',
+            r'^\s*include_once\s+',
+            r'^\s*namespace\s+',
+        ],
+        'swift': [
+            r'^\s*import\s+',
+        ],
+        'ruby': [
+            r'^\s*require\s+',
+            r'^\s*require_relative\s+',
+            r'^\s*load\s+',
+        ],
+        'perl': [
+            r'^\s*use\s+',
+            r'^\s*require\s+',
+        ],
+        'r': [
+            r'^\s*library\(',
+            r'^\s*require\(',
+            r'^\s*source\(',
+        ],
+        'lua': [
+            r'^\s*require\s+',
+            r'^\s*local\s+.*\s*=\s*require\(',
+        ],
+        'dart': [
+            r'^\s*import\s+',
+            r'^\s*export\s+',
+            r'^\s*part\s+',
+        ],
+        'haskell': [
+            r'^\s*import\s+',
+            r'^\s*module\s+',
+        ],
+        'elixir': [
+            r'^\s*import\s+',
+            r'^\s*alias\s+',
+            r'^\s*require\s+',
+            r'^\s*use\s+',
+        ],
+        'clojure': [
+            r'^\s*\(\s*ns\s+',
+            r'^\s*\(\s*require\s+',
+            r'^\s*\(\s*import\s+',
+        ],
+    }
+    # Keywords that indicate licenses
+    LICENSE_KEYWORDS = [
+        'copyright', 'license', 'licensed', 'all rights reserved',
+        'permission', 'redistribution', 'warranty', 'liability',
+        'apache', 'mit', 'gpl', 'bsd', 'mozilla', 'author:',
+        'spdx-license', 'contributors', 'licensee'
+    ]
+
+COMPLETE_DOCSTRING_QUOTE_COUNT = 2
+LICENSE_HEADER_MAX_LINES = 50
+# Map of file extensions to programming languages
+EXT_MAP = {
+    '.py': 'python',
+    '.js': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+    '.ts': 'typescript',
+    '.tsx': 'typescript',
+    '.jsx': 'javascript',
+    '.java': 'java',
+    '.kt': 'kotlin',
+    '.kts': 'kotlin',
+    '.scala': 'scala',
+    '.sc': 'scala',
+    '.go': 'go',
+    '.rs': 'rust',
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.cxx': 'cpp',
+    '.c': 'cpp',
+    '.h': 'cpp',
+    '.hpp': 'cpp',
+    '.hxx': 'cpp',
+    '.cs': 'csharp',
+    '.php': 'php',
+    '.swift': 'swift',
+    '.rb': 'ruby',
+    '.pl': 'perl',
+    '.pm': 'perl',
+    '.r': 'r',
+    '.R': 'r',
+    '.lua': 'lua',
+    '.dart': 'dart',
+    '.hs': 'haskell',
+    '.ex': 'elixir',
+    '.exs': 'elixir',
+    '.clj': 'clojure',
+    '.cljs': 'clojure',
+    '.m': 'cpp',  # Objective-C
+    '.mm': 'cpp',  # Objective-C++
+    # Shell scripts share Python's # comment style, but lack dedicated
+    # import patterns (source/. commands won't be filtered)
+    '.sh': 'python',
+    '.bash': 'python',
+    '.zsh': 'python',
+    '.fish': 'python',
+}
+
+
+def is_blank_line(stripped_line: str) -> bool:
+    """
+    Check if a line is blank.
+
+    This method determines whether a given string `line` is blank by checking
+    if it consists entirely of whitespace or is empty.
+
+    :param stripped_line: The string to be evaluated.
+    :return: True if the string is blank, otherwise False.
+    """
+    return len(stripped_line) == 0
+
+
+def is_shebang(stripped_line: str) -> bool:
+    """
+    Check if the given line is a shebang line.
+
+    This function determines if the provided string is a shebang line,
+    which indicates the path to the interpreter that should execute the
+    script.
+
+    :param stripped_line: The string to check if it's a shebang line.
+    :return: True if the given line starts with '#!', otherwise False.
+    """
+    return stripped_line.startswith('#!')
+
+
+class HeaderFilter(ScanossBase):
+    """
+    Source code file analyser that filters headers, comments, and imports.
+
+    This class processes code files and returns only the real
+    implementation content, omitting licenses, documentation comments,
+    and imports.
+    """
+
+    def __init__(
+        self,
+        debug: bool = False,
+        trace: bool = False,
+        quiet: bool = False,
+        skip_limit: Optional[int] = None
+    ):
+        """
+        Initialise HeaderFilter
+        Parameters
+        ----------
+            skip_limit: int
+                Maximum number of lines to skip when analysing a file.
+                If set, then stop stripping data after this number of lines.
+                (None/0 = unlimited by default)
+        """
+        super().__init__(debug, trace, quiet)
+        self.patterns = LanguagePatterns()
+        self.max_lines = skip_limit
+
+    def filter(self, file: str, decoded_contents: str) -> int:
+        """
+        Main method that filters file content
+        Parameters
+        ----------
+            :param file: File path (used to detect extension)
+            :param decoded_contents: File contents in utf-8 encoding
+        Return
+        ------
+            - line_offset: Number of lines skipped from the beginning
+              (0 if no filtering)
+        """
+        if not decoded_contents or not file:
+            self.print_msg(f'No file or contents provided, skipping line filter for: {file}')
+            return 0
+        self.print_debug(f'HeaderFilter processing file: {file}')
+        # Detect language
+        language = self.detect_language(file)
+        # If language is not supported, return original content
+        if not language:
+            self.print_debug(f'Skipping line filter for unsupported language: {file}')
+            return 0
+        lines = decoded_contents.splitlines(keepends=True)
+        num_lines = len(lines)
+        if num_lines == 0:
+            self.print_msg(f'No lines in file: {file}')
+            return 0
+        self.print_debug(f'Analysing {num_lines} lines for file: {file}')
+
+        # Find the first implementation line (optimised - stops at first match)
+        implementation_start = self.find_first_implementation_line(lines, language)
+        # If no implementation, return empty
+        if implementation_start is None:
+            self.print_debug(f'No implementation found in file: {file}')
+            return 0
+        # Calculate how many lines were filtered out (line_offset)
+        line_offset = implementation_start - 1
+        # Apply max_lines limit if configured
+        if self.max_lines is not None and 0 < self.max_lines < line_offset:
+            self.print_trace(
+                f'Line offset {line_offset} exceeds max_lines {self.max_lines}, '
+                f'capping at {self.max_lines} for: {file}'
+            )
+            line_offset = self.max_lines
+
+        if line_offset > 0:
+            self.print_debug(f'Filtered out {line_offset} lines from beginning of {file} (language: {language})')
+        return line_offset
+
+    def detect_language(self, file_path: str) -> Optional[str]:
+        """
+        Detects the programming language based on the provided file extension.
+
+        This function uses a predefined mapping between file extensions and programming
+        languages to determine the language associated with the file. If the file extension
+        is found in the mapping, the corresponding language is returned. Otherwise, it
+        returns None.
+
+        :param file_path: Path to the file whose programming language needs to be detected.
+        :return: The programming language corresponding to the file extension if mapped,
+                 otherwise None.
+        """
+        path = Path(file_path)
+        extension = path.suffix.lower()
+        if extension:
+            detected_language = EXT_MAP.get(extension)
+            if detected_language:
+                self.print_debug(f'Detected language "{detected_language}" for extension "{extension}"')
+            else:
+                self.print_debug(f'No language mapping found for extension "{extension}"')
+        else:
+            self.print_debug(f'No file extension found, skipping language detection for: {file_path}')
+            detected_language = None
+        return detected_language
+
+    def is_license_header(self, line: str) -> bool:
+        """
+        Check if the line appears to be part of a license header.
+
+        This method evaluates a given line of text to determine whether it
+        contains keywords that suggest it is part of a license header. It
+        performs a case-insensitive check against a predefined set of license
+        keywords.
+
+        :param line: The line of text to check.
+        :return: True if the line contains keywords indicating it is part of a
+            license header; False otherwise.
+        """
+        line_lower = line.lower()
+        return any(keyword in line_lower for keyword in self.patterns.LICENSE_KEYWORDS)
+
+    def get_comment_style(self, language: str) -> str:
+        """
+        Return the comment style associated with a given programming language.
+
+        This method determines the appropriate comment style to use based on the
+        specified programming language. Supported languages include those with C-style
+        comments, Python-style comments, and Lua-style comments. If the language does
+        not match any of the explicitly defined groups, a default of `c_style` is
+        returned.
+
+        :param language: The name of the programming language for which the comment
+            style needs to be determined.
+        :return: The comment style for the provided programming language. Possible
+            values are 'c_style', 'python_style', or 'lua_style'.
+        """
+        if language:
+            if language in ['cpp', 'java', 'kotlin', 'scala', 'javascript', 'typescript',
+                            'go', 'rust', 'csharp', 'php', 'swift', 'dart']:
+                return 'c_style'
+            if language in ['python', 'ruby', 'perl', 'r']:
+                return 'python_style'
+            if language in ['lua', 'haskell']:
+                return 'lua_style'
+        self.print_debug(f'No comment style defined for language "{language}", using default: "c_style"')
+        return 'c_style'  # Default
+
+    def is_comment(self, line: str, in_multiline: bool, patterns: dict) -> Tuple[bool, bool]:  # noqa: PLR0911
+        """
+        Check if a line is a comment
+
+        :param patterns: comment patterns
+        :param line: Line to check
+        :param in_multiline: Whether we're currently in a multiline comment
+        :return: Tuple of (is_comment, still_in_multiline)
+        """
+        if not patterns:
+            self.print_msg('No comment patterns defined, skipping comment check')
+            return False, in_multiline
+        # If we're in a multiline comment
+        if in_multiline:
+            # Check if the comment ends
+            if 'multi_end' in patterns and re.search(patterns['multi_end'], line):
+                return True, False
+            if 'doc_string_end' in patterns and re.search(patterns['doc_string_end'], line):
+                return True, False
+            return True, True
+        # Single-line comment
+        if 'single_line' in patterns and re.match(patterns['single_line'], line):
+            return True, False
+        # Multiline comment complete in one line
+        if 'multi_single' in patterns and re.match(patterns['multi_single'], line):
+            return True, False
+        # Start of multiline comment (C-style)
+        if 'multi_start' in patterns and re.search(patterns['multi_start'], line):
+            # If it also ends on the same line
+            if 'multi_end' in patterns and re.search(patterns['multi_end'], line):
+                return True, False
+            return True, True
+        # Start of docstring (Python)
+        if 'doc_string_start' in patterns and '"""' in line:
+            # Count how many quotes there are
+            count = line.count('"""')
+            if count == COMPLETE_DOCSTRING_QUOTE_COUNT:  # Complete docstring in one line
+                return True, False
+            if count == 1:  # Start of a multiline docstring
+                return True, True
+        # Default response: not a comment
+        return False, in_multiline
+
+    def is_import(self, line: str, patterns: dict) -> bool:
+        """
+        Check if a line of code is an import or include statement for a given programming language.
+
+        This function determines whether a specific line of code matches any
+        import/include patterns defined for the provided programming language.
+        It relies on predefined regular expression patterns.
+
+        :param patterns: import patterns for the given language.
+        :param line: A single line of code to check.
+        :return: True if the line matches any import/include pattern for the given language,
+            otherwise False.
+        """
+        if not patterns:
+            self.print_debug('No import patterns defined, skipping import check')
+        return any(re.match(pattern, line) for pattern in patterns)
+
+    def find_first_implementation_line(self, lines: list[str], language: str) -> Optional[int]:  # noqa: PLR0912
+        """
+        Find the line number where the implementation begins (optimised version).
+        Returns as soon as the first implementation line is found.
+
+        :param lines: List of code lines
+        :param language: Programming language
+        :return: Line number (1-indexed) where implementation starts, or None if not found
+        """
+        if not lines or not language:
+            self.print_debug('No lines or language provided, skipping implementation line detection')
+            return None
+        in_multiline_comment = False
+        in_license_section = False
+        in_import_block = False  # To handle import blocks in Go
+        consecutive_imports_count = 0
+        # Get comment & import patterns for the language
+        comment_patterns = self.patterns.COMMENT_PATTERNS[self.get_comment_style(language)]
+        import_patterns = self.patterns.IMPORT_PATTERNS[language]
+        # Iterate through lines trying to find the first implementation line
+        for i, line in enumerate(lines):
+            line_number = i + 1
+            stripped = line.strip()
+            # Shebang (only first line) or blank line
+            if (i == 0 and is_shebang(stripped)) or is_blank_line(stripped):
+                continue
+            # Check if it's a comment
+            is_a_comment, in_multiline_comment = self.is_comment(line, in_multiline_comment, comment_patterns)
+            if is_a_comment:
+                # Check if it's part of the license header
+                if self.is_license_header(line):
+                    if not in_license_section:
+                        self.print_trace(f'Line {line_number}: Detected license header section')
+                    in_license_section = True
+                # If still in the license section (first lines)
+                elif in_license_section and line_number < LICENSE_HEADER_MAX_LINES:
+                    pass  # Still in the license section. Keep looking.
+                else:
+                    if in_license_section:
+                        self.print_trace(f'Line {line_number}: End of license header section')
+                    in_license_section = False
+                continue
+            # If not a comment but we find a non-empty line, end license section
+            if not is_a_comment:
+                in_license_section = False
+            # Handle import blocks in Go
+            if language == 'go':
+                if stripped.startswith('import ('):
+                    self.print_trace(f'Line {line_number}: Detected Go import block start')
+                    in_import_block = True
+                    continue
+                if in_import_block:
+                    if stripped == ')':
+                        self.print_trace(f'Line {line_number}: Detected Go import block end')
+                        in_import_block = False
+                        continue
+                    if (stripped.startswith('"') or stripped.startswith('_') or
+                            re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s+"', stripped)):
+                        # It's part of the import block
+                        continue
+            # Check if it's an import
+            if self.is_import(line, import_patterns):
+                if consecutive_imports_count == 0:
+                    self.print_trace(f'Line {line_number}: Detected import section')
+                consecutive_imports_count += 1
+                continue
+            # If we get here, it's implementation code - return immediately!
+            self.print_trace(f'Line {line_number}: First implementation line detected')
+            return line_number
+        # End for loop?
+        return None
+#
+# End of HeaderFilter Class
+#

scanoss/scanner.py

@@ -109,6 +109,8 @@ class Scanner(ScanossBase):
         scan_settings: 'ScanossSettings | None' = None,
         req_headers: dict = None,
         use_grpc: bool = False,
+        skip_headers: bool = False,
+        skip_headers_limit: int = 0,
     ):
         """
         Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning
@@ -137,6 +139,7 @@ class Scanner(ScanossBase):
 
         self.winnowing = Winnowing(
             debug=debug,
+            trace=trace,
             quiet=quiet,
             skip_snippets=self._skip_snippets,
             all_extensions=all_extensions,
@@ -145,6 +148,8 @@ class Scanner(ScanossBase):
             strip_hpsm_ids=strip_hpsm_ids,
             strip_snippet_ids=strip_snippet_ids,
             skip_md5_ids=skip_md5_ids,
+            skip_headers=skip_headers,
+            skip_headers_limit=skip_headers_limit,
         )
         self.scanoss_api = ScanossApi(
             debug=debug,

scanoss/scanners/scanner_hfh.py

@@ -218,7 +218,7 @@ class ScannerHFHPresenter(AbstractPresenter):
             }
 
             get_vulnerabilities_json_request = {
-                'purls': [{'purl': purl, 'requirement': best_match_version['version']}],
+                'components': [{'purl': purl, 'requirement': best_match_version['version']}],
             }
 
             decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request)

scanoss/scanossapi.py

@@ -78,7 +78,7 @@ class ScanossApi(ScanossBase):
         :param api_key: API Key (default None)
         :param debug: Enable debug (default False)
         :param trace: Enable trace (default False)
-        :param quiet: Enable quite mode (default False)
+        :param quiet: Enable quiet mode (default False)
 
         To set a custom certificate use:
             REQUESTS_CA_BUNDLE=/path/to/cert.pem

scanoss/scanossbase.py

@@ -50,7 +50,7 @@ class ScanossBase:
 
     def print_msg(self, *args, **kwargs):
         """
-        Print message if quite mode is not enabled
+        Print message if quiet mode is not enabled
         """
         if not self.quiet:
             self.print_stderr(*args, **kwargs)

scanoss/winnowing.py

@@ -37,6 +37,7 @@ from typing import Tuple
 from binaryornot.check import is_binary
 from crc32c import crc32c
 
+from .header_filter import HeaderFilter
 from .scanossbase import ScanossBase
 
 # Winnowing configuration. DO NOT CHANGE.
@@ -172,6 +173,8 @@ class Winnowing(ScanossBase):
         strip_hpsm_ids=None,
         strip_snippet_ids=None,
         skip_md5_ids=None,
+        skip_headers: bool = False,
+        skip_headers_limit: int = 0,
     ):
         """
         Instantiate Winnowing class
@@ -198,7 +201,9 @@ class Winnowing(ScanossBase):
         self.strip_hpsm_ids = strip_hpsm_ids
         self.strip_snippet_ids = strip_snippet_ids
         self.hpsm = hpsm
+        self.skip_headers = skip_headers
         self.is_windows = platform.system() == 'Windows'
+        self.header_filter = HeaderFilter(debug=debug, trace=trace, quiet=quiet, skip_limit=skip_headers_limit)
         if hpsm:
             self.crc8_maxim_dow_table = []
             self.crc8_generate_table()
@@ -353,6 +358,48 @@ class Winnowing(ScanossBase):
             self.print_debug(f'Stripped snippet ids from {file}')
         return wfp
 
+    def __strip_lines_until_offset(self, file: str, wfp: str, line_offset: int) -> str:
+        """
+        Strip lines from the WFP up to and including the line_offset
+
+        :param file: name of fingerprinted file
+        :param wfp: WFP to clean
+        :param line_offset: line number offset to strip up to
+        :return: updated WFP
+        """
+        # No offset specified, return original WFP
+        if line_offset <= 0:
+            return wfp
+        lines = wfp.split('\n')
+        filtered_lines = []
+        start_line_added = False
+        for line in lines:
+            # Check if a line contains snippet data (format: line_number=hash,hash,...)
+            line_details = line.split('=')
+            if line_details[0].isdigit():
+                try:
+                    line_num = int(line_details[0])
+                    # Keep lines that are after the offset
+                    # (line_offset is the last line previous to real code)
+                    if line_num > line_offset:
+                        # Add the start_line tag before the first snippet line
+                        if not start_line_added:
+                            filtered_lines.append(f'start_line={line_offset}')
+                            start_line_added = True
+                        filtered_lines.append(line)
+                except (ValueError, IndexError) as e:
+                    self.print_stderr(f'Error decoding line number from line {line} in {file}: {e}')
+                    # Keep non-snippet lines (like file=, hpsm=, etc.)
+                    filtered_lines.append(line)
+            else:
+                # Keep non-snippet lines (like file=, hpsm=, etc.)
+                filtered_lines.append(line)
+        # End for loop comment
+        wfp = '\n'.join(filtered_lines)
+        if start_line_added:
+            self.print_debug(f'Stripped lines up to offset {line_offset} from {file}')
+        return wfp
+
     def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool]:
         """Detect the types of line endings present in file contents.
 
@@ -362,13 +409,14 @@ class Winnowing(ScanossBase):
         Returns:
             Tuple of (has_crlf, has_lf_only, has_cr_only, has_mixed) indicating which line ending types are present.
         """
+        if not contents:
+            self.print_debug('Warning: No file contents provided')
         has_crlf = b'\r\n' in contents
         # For LF detection, we need to find LF that's not part of CRLF
         content_without_crlf = contents.replace(b'\r\n', b'')
         has_standalone_lf = b'\n' in content_without_crlf
         # For CR detection, we need to find CR that's not part of CRLF
         has_standalone_cr = b'\r' in content_without_crlf
-
         return has_crlf, has_standalone_lf, has_standalone_cr
 
     def __calculate_opposite_line_ending_hash(self, contents: bytes):
@@ -384,13 +432,11 @@ class Winnowing(ScanossBase):
             Hash with opposite line endings as hex string, or None if no line endings detected.
         """
         has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents)
-
         if not has_crlf and not has_standalone_lf and not has_standalone_cr:
+            self.print_debug('No line endings detected in file contents')
             return None
-
-        # Normalize all line endings to LF first
+        # Normalise all line endings to LF first
         normalized = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
-
         # Determine the dominant line ending type
         if has_crlf and not has_standalone_lf and not has_standalone_cr:
             # File is Windows (CRLF) - produce Unix (LF) hash
@@ -398,7 +444,7 @@ class Winnowing(ScanossBase):
         else:
             # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash
             opposite_contents = normalized.replace(b'\n', b'\r\n')
-
+        # Return the MD5 hash of the opposite contents
         return hashlib.md5(opposite_contents).hexdigest()
 
     def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str:  # noqa: PLR0912, PLR0915
@@ -420,27 +466,26 @@ class Winnowing(ScanossBase):
         # Print file line
         content_length = len(contents)
         original_filename = file
-
         if self.is_windows:
             original_filename = file.replace('\\', '/')
         wfp_filename = repr(original_filename).strip("'")  # return a utf-8 compatible version of the filename
-        if self.obfuscate:  # hide the real size of the file and its name, but keep the suffix
+        # hide the real size of the file and its name but keep the suffix
+        if self.obfuscate:
             wfp_filename = f'{self.ob_count}{pathlib.Path(original_filename).suffix}'
             self.ob_count = self.ob_count + 1
             self.file_map[wfp_filename] = original_filename  # Save the file name map for later (reverse lookup)
-
+        # Construct the WFP header
         wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename)
-
-        # Add opposite line ending hash based on line ending analysis
+        # Add the opposite line ending hash based on line ending analysis
         if not bin_file:
             opposite_hash = self.__calculate_opposite_line_ending_hash(contents)
             if opposite_hash is not None:
                 wfp += f'fh2={opposite_hash}\n'
-
         # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip
-        if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')):
+        decoded_contents = contents.decode('utf-8', 'ignore')
+        if bin_file or self.skip_snippets or self.__skip_snippets(file, decoded_contents):
             return wfp
-        # Add HPSM
+        # Add HPSM (calculated from original contents, not filtered)
         if self.hpsm:
             hpsm = self.__strip_hpsm(file, self.calc_hpsm(contents))
             if len(hpsm) > 0:
@@ -448,7 +493,7 @@ class Winnowing(ScanossBase):
         # Initialize variables
         gram = ''
         window = []
-        line = 1
+        line = 1  # Line counter for WFP generation
         last_hash = MAX_CRC32
         last_line = 0
         output = ''
@@ -503,12 +548,19 @@ class Winnowing(ScanossBase):
                 wfp += output + '\n'
             else:
                 self.print_debug(f'Warning: skipping output in WFP for {file} - "{output}"')
-
+        # Warn if we don't have any WFP content
         if wfp is None or wfp == '':
             self.print_stderr(f'Warning: No WFP content data for {file}')
-        elif self.strip_snippet_ids:
-            wfp = self.__strip_snippets(file, wfp)
-
+        else:
+            # Apply line filter to remove headers, comments, and imports from the beginning (if enabled)
+            if self.skip_headers:
+                line_offset = self.header_filter.filter(file, decoded_contents)
+                if line_offset > 0:
+                    wfp = self.__strip_lines_until_offset(file, wfp, line_offset)
+            # Strip snippet IDs from the WFP (if enabled)
+            if self.strip_snippet_ids:
+                wfp = self.__strip_snippets(file, wfp)
+        # Return the WFP contents
         return wfp
 
     def calc_hpsm(self, content):

{scanoss-1.41.0.dist-info → scanoss-1.42.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scanoss
-Version: 1.41.0
+Version: 1.42.0
 Summary: Simple Python library to leverage the SCANOSS APIs
 Home-page: https://scanoss.com
 Author: SCANOSS

{scanoss-1.41.0.dist-info → scanoss-1.42.0.dist-info}/RECORD

@@ -6,8 +6,8 @@ protoc_gen_swagger/options/annotations_pb2_grpc.py,sha256=KZOW9Ciio-f9iL42FuLFnS
 protoc_gen_swagger/options/openapiv2_pb2.py,sha256=w0xDs63uyrWGgzRaQZXfJpfI7Jpyvh-i9ay_uzOR-aM,16475
 protoc_gen_swagger/options/openapiv2_pb2.pyi,sha256=hYOV6uQ2yqhP89042_V3GuAsvoBBiXf5CGuYmnFnfv4,54665
 protoc_gen_swagger/options/openapiv2_pb2_grpc.py,sha256=sje9Nh3yE7CHCUWZwtjTgwsKB4GvyGz5vOrGTnRXJfc,917
-scanoss/__init__.py,sha256=gwiLnpxvm3FTkfCt7CRIdD8To8AjOpOG_5oXLMvgpOk,1146
-scanoss/cli.py,sha256=YAE9ihA2LswSLHBRHnpyqM-e2tcE7lPd8-ghABbnyU8,103984
+scanoss/__init__.py,sha256=9ZAHE5MaLo7fYIxovUewqgv-BFsZ7YIIgJalSwDtGc4,1146
+scanoss/cli.py,sha256=cHkms9nVYniZMAT184VkKQTcd7SL1SEnByxPSAsBVwo,104610
 scanoss/components.py,sha256=NFyt_w3aoMotr_ZaFU-ng00_89sruc0kgY7ERnJXkmM,15891
 scanoss/constants.py,sha256=vurzLNIfP_dnRMwOdZsUWvr5XAVuGoj98XZ0yjXNOjQ,632
 scanoss/cryptography.py,sha256=lOoD_dW16ARQxYiYyb5R8S7gx0FqWIsnGkKfsB0nGaU,10627
@@ -17,20 +17,21 @@ scanoss/delta.py,sha256=slmgnD7SsUOmfSE2zb0zdRAGo-JcjPJAtxyzuCSzO3I,9455
 scanoss/file_filters.py,sha256=QcLqunaBKQIafjNZ9_Snh9quBX5_-fsTusVmxwjC1q8,18511
 scanoss/filecount.py,sha256=icWaKN_xapMrH3ZZ-D3nldx7hWiguIOjoKg4gCeKDOM,6678
 scanoss/gitlabqualityreport.py,sha256=_VG0Xoh8wYF3lsXGJvjoj-Ty58OS_-H1Domiq9OpQEo,8830
+scanoss/header_filter.py,sha256=-Dqore9coROLMWWw9yP3nz8dpCB7jYAVm842hoRTmeE,21879
 scanoss/osadl.py,sha256=VWalcHpshWxtRDGje2cK32SfFeSBAO62knfSW9pyYqc,4558
 scanoss/results.py,sha256=47ZXXuU2sDjYa5vhtbWTmikit9jHhA0rsYKwkvZFI5w,9252
 scanoss/scancodedeps.py,sha256=JbpoGW1POtPMmowzfwa4oh8sSBeeQCqaW9onvc4UFYM,11517
-scanoss/scanner.py,sha256=P2gswXapSVYC-tH6_oY763DYpe9Awi2g9xW1U9WGgQI,46374
+scanoss/scanner.py,sha256=E71i7sni3hc22b0Mpd2NyifajmX0YOkRyUGBJiiN58o,46562
 scanoss/scanoss_settings.py,sha256=W8uFQ6uRIqtE-DXXA56bO8I4GsbJ-aA1c84hQ_qBel4,12161
-scanoss/scanossapi.py,sha256=U3IytJXA0mQHYsylzjwCuFakuhD_dJhstY37NmyvtB8,13243
-scanoss/scanossbase.py,sha256=Dkpwxa8NH8XN1iRl03NM_Mkvby0JQ4qfvCiiUrJ5ul0,3163
+scanoss/scanossapi.py,sha256=O1ZNH9Kt8JzhLVBxfOSmJdEwSJTDP-rA54DulYdE8e4,13243
+scanoss/scanossbase.py,sha256=tKlHPAi50ZarGaPXsNi1XrowQBynsSqSSst-NuG2ScI,3163
 scanoss/scanossgrpc.py,sha256=9UuVPUjBLUhqim_tSntyoRZW-OAtiz5iP_VjjNr5RPY,41715
 scanoss/scanpostprocessor.py,sha256=-JsThlxrU70r92GHykTMERnicdd-6jmwNsE4PH0MN2o,11063
 scanoss/scantype.py,sha256=gFmyVmKQpHWogN2iCmMj032e_sZo4T92xS3_EH5B3Tc,1310
 scanoss/spdxlite.py,sha256=4JMxmyNmvcL6fjScihk8toWfSuQ-Pj1gzaT3SIn1fXA,29425
 scanoss/threadeddependencies.py,sha256=aN8E43iKS1pWJLJP3xCle5ewlfR5DE2-ljUzI_29Xwk,9851
 scanoss/threadedscanning.py,sha256=Y-OYamD3xJvFiqwCn5y_4QD5gk_rJ5xs2jI1DxNtJlc,9661
-scanoss/winnowing.py,sha256=RsR9jRTR3TzS1pEeKQ2RuYlIG8Q7RnUQFfgPLog6B-A,21679
+scanoss/winnowing.py,sha256=py4gFKVHI5ZsLNyQIvNtnO6k3tBbvEXYNw24yuMgoTc,24751
 scanoss/api/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
 scanoss/api/common/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
 scanoss/api/common/v2/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
@@ -66,7 +67,7 @@ scanoss/api/vulnerabilities/__init__.py,sha256=IFrDk_DTJgKSZmmU-nuLXuq_s8sQZlrSC
 scanoss/api/vulnerabilities/v2/__init__.py,sha256=IFrDk_DTJgKSZmmU-nuLXuq_s8sQZlrSCHhIDMJT4r0,1122
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py,sha256=pmm0MSiXkdf8e4rCIIDRcsNRixR2vGvD1Xak4l-wdwI,16550
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py,sha256=BNxT5kUKQ-mgtOt5QYBM1Qrg5LNDqSpWKpfEZquIlsM,19127
-scanoss/data/build_date.txt,sha256=9AXzlvNjRrd9YhQ9pyrxQftmN0vVNptg1mAwlPWlkuY,40
+scanoss/data/build_date.txt,sha256=e-9ZOSAGNgP3bfrxejydpbf1_yVzmVCB3S0XOeP7nOs,40
 scanoss/data/osadl-copyleft.json,sha256=O9b2XAfpjQY0TL0fYzO6kwMcp5IwQbF6f_YWbB10MhQ,4761
 scanoss/data/scanoss-settings-schema.json,sha256=ClkRYAkjAN0Sk704G8BE_Ok006oQ6YnIGmX84CF8h9w,8798
 scanoss/data/spdx-exceptions.json,sha256=s7UTYxC7jqQXr11YBlIWYCNwN6lRDFTR33Y8rpN_dA4,17953
@@ -93,7 +94,7 @@ scanoss/scanners/__init__.py,sha256=D4C0lWLuNp8k_BjQZEc07WZcUgAvriVwQWOk063b0ZU,
 scanoss/scanners/container_scanner.py,sha256=fOrb64owrstX7LnTuxiIan059YgLeKXeBS6g2QaCyq0,16346
 scanoss/scanners/folder_hasher.py,sha256=UzOmtYTLMqeL3Jf4CpvT-L4qaPNCy9-7xV0BwYhAuRc,12973
 scanoss/scanners/scanner_config.py,sha256=egG7cw3S2akU-D9M1aLE5jLrfz_c8e7_DIotMnnpM84,2601
-scanoss/scanners/scanner_hfh.py,sha256=8CAYZ0FGFSW8Dcp6ci0ZV6bxphR9D_WxMY17fvalICk,9507
+scanoss/scanners/scanner_hfh.py,sha256=gn2DnWD0J1_EPAKm5Zmu6ZjbGBPPSdz0YjqcPESPE-c,9512
 scanoss/services/dependency_track_service.py,sha256=JIpqev4I-x_ZajMxD5W2Y3OAUvEJ_4nstzAPV90vfP8,5070
 scanoss/utils/__init__.py,sha256=0hjb5ktavp7utJzFhGMPImPaZiHWgilM2HwvTp5lXJE,1122
 scanoss/utils/abstract_presenter.py,sha256=teiDTxBj5jBMCk2T8i4l1BJPf_u4zBLWrtCTFHSSECM,3148
@@ -101,9 +102,9 @@ scanoss/utils/crc64.py,sha256=TMrwQimSdE6imhFOUL7oAG6Kxu-8qMpGWMuMg8QpSVs,3169
 scanoss/utils/file.py,sha256=62cA9a17TU9ZvfA3FY5HY4-QOajJeSrc8S6xLA_f-3M,2980
 scanoss/utils/scanoss_scan_results_utils.py,sha256=ho9-DKefHFJlVZkw4gXOmMI-mgPIbV9Y2ftkI83fC1k,1727
 scanoss/utils/simhash.py,sha256=6iu8DOcecPAY36SZjCOzrrLMT9oIE7-gI6QuYwUQ7B0,5793
-scanoss-1.41.0.dist-info/licenses/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
-scanoss-1.41.0.dist-info/METADATA,sha256=PlxJlKB07rRGpvxVduJV_McBVlNdWWtBGcoFl50d2Xc,6181
-scanoss-1.41.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-scanoss-1.41.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
-scanoss-1.41.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
-scanoss-1.41.0.dist-info/RECORD,,
+scanoss-1.42.0.dist-info/licenses/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
+scanoss-1.42.0.dist-info/METADATA,sha256=qIMoqDLLS5apaE0c3zuoEFb5NYY9qwupeH5pbshUkc4,6181
+scanoss-1.42.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scanoss-1.42.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
+scanoss-1.42.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
+scanoss-1.42.0.dist-info/RECORD,,