scanoss 1.24.0__py3-none-any.whl → 1.25.0__py3-none-any.whl

scanoss/__init__.py

@@ -22,4 +22,4 @@ SPDX-License-Identifier: MIT
   THE SOFTWARE.
 """
 
-__version__ = '1.24.0'
+__version__ = '1.25.0'

scanoss/data/build_date.txt

@@ -1 +1 @@
-date: 20250528081438, utime: 1748420078
+date: 20250610161304, utime: 1749571984

scanoss/inspection/policy_check.py

@@ -26,9 +26,10 @@ import json
 import os.path
 from abc import abstractmethod
 from enum import Enum
-from typing import Callable, List, Dict, Any
-from .utils.license_utils import LicenseUtil
+from typing import Any, Callable, Dict, List
+
 from ..scanossbase import ScanossBase
+from .utils.license_utils import LicenseUtil
 
 
 class PolicyStatus(Enum):
@@ -87,7 +88,7 @@ class PolicyCheck(ScanossBase):
 
     VALID_FORMATS = {'md', 'json', 'jira_md'}
 
-    def __init__(
+    def __init__( # noqa: PLR0913
         self,
         debug: bool = False,
         trace: bool = True,
@@ -181,10 +182,9 @@ class PolicyCheck(ScanossBase):
         :param status: The new component status
         :return: The updated components dictionary
         """
-
         # Determine the component key and purl based on component type
         if id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]:
-            purl = new_component['purl'][0]  # Take first purl for these component types
+            purl = new_component['purl'][0]  # Take the first purl for these component types
         else:
             purl = new_component['purl']
 
@@ -195,14 +195,13 @@ class PolicyCheck(ScanossBase):
             'licenses': {},
             'status': status,
         }
-
         if not new_component.get('licenses'):
-            self.print_stderr(f'WARNING: Results missing licenses. Skipping.')
+            self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}')
             return components
         # Process licenses for this component
-        for l in new_component['licenses']:
-            if l.get('name'):
-                spdxid = l['name']
+        for license_item in new_component['licenses']:
+            if license_item.get('name'):
+                spdxid = license_item['name']
                 components[component_key]['licenses'][spdxid] = {
                     'spdxid': spdxid,
                     'copyleft': self.license_util.is_copyleft(spdxid),
@@ -210,71 +209,103 @@ class PolicyCheck(ScanossBase):
                 }
         return components
 
-    def _get_components_from_results(self, results: Dict[str, Any]) -> list or None:
+    def _get_components_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]:
         """
-        Process the results dictionary to extract and format component information.
-
-        This function iterates through the results dictionary, identifying components from
-        different sources (files, snippets, and dependencies). It consolidates this information
-        into a list of unique components, each with its associated licenses and other details.
+        Extract and process file and snippet components from results.
 
         :param results: A dictionary containing the raw results of a component scan
-        :return: A list of dictionaries, each representing a unique component with its details
+        :param components: Existing components dictionary to update
+        :return: Updated components dictionary with file and snippet data
         """
-        if results is None:
-            self.print_stderr(f'ERROR: Results cannot be empty')
-            return None
-        components = {}
         for component in results.values():
             for c in component:
                 component_id = c.get('id')
                 if not component_id:
-                    self.print_stderr(f'WARNING: Result missing id. Skipping.')
+                    self.print_debug(f'WARNING: Result missing id. Skipping: {c}')
                     continue
                 status = c.get('status')
-                if not component_id:
-                    self.print_stderr(f'WARNING: Result missing status. Skipping.')
+                if not status:
+                    self.print_debug(f'WARNING: Result missing status. Skipping: {c}')
                     continue
                 if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]:
                     if not c.get('purl'):
-                        self.print_stderr(f'WARNING: Result missing purl. Skipping.')
+                        self.print_debug(f'WARNING: Result missing purl. Skipping: {c}')
                         continue
                     if len(c.get('purl')) <= 0:
-                        self.print_stderr(f'WARNING: Result missing purls. Skipping.')
+                        self.print_debug(f'WARNING: Result missing purls. Skipping: {c}')
                         continue
                     if not c.get('version'):
-                        self.print_stderr(f'WARNING: Result missing version. Skipping.')
+                        self.print_msg(f'WARNING: Result missing version. Skipping: {c}')
                         continue
                     component_key = f'{c["purl"][0]}@{c["version"]}'
-                    # Initialize or update the component entry
                     if component_key not in components:
                         components = self._append_component(components, c, component_id, status)
+            # End component loop
+        # End components loop
+        return components
 
-                if c['id'] == ComponentID.DEPENDENCY.value:
+    def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Extract and process dependency components from results.
+
+        :param results: A dictionary containing the raw results of a component scan
+        :param components: Existing components dictionary to update
+        :return: Updated components dictionary with dependency data
+        """
+        for component in results.values():
+            for c in component:
+                component_id = c.get('id')
+                if not component_id:
+                    self.print_debug(f'WARNING: Result missing id. Skipping: {c}')
+                    continue
+                status = c.get('status')
+                if not status:
+                    self.print_debug(f'WARNING: Result missing status. Skipping: {c}')
+                    continue
+                if component_id == ComponentID.DEPENDENCY.value:
                     if c.get('dependencies') is None:
                         continue
-                    for d in c['dependencies']:
-                        if not d.get('purl'):
-                            self.print_stderr(f'WARNING: Result missing purl. Skipping.')
-                            continue
-                        if len(d.get('purl')) <= 0:
-                            self.print_stderr(f'WARNING: Result missing purls. Skipping.')
+                    for dependency in c['dependencies']:
+                        if not dependency.get('purl'):
+                            self.print_debug(f'WARNING: Dependency result missing purl. Skipping: {dependency}')
                             continue
-                        if not d.get('version'):
-                            self.print_stderr(f'WARNING: Result missing version. Skipping.')
+                        if not dependency.get('version'):
+                            self.print_msg(f'WARNING: Dependency result missing version. Skipping: {dependency}')
                             continue
-                        component_key = f'{d["purl"]}@{d["version"]}'
+                        component_key = f'{dependency["purl"]}@{dependency["version"]}'
                         if component_key not in components:
-                            components = self._append_component(components, d, component_id, status)
-                    # End of dependencies loop
-                # End if
-            # End of component loop
-        # End of results loop
-        results = list(components.values())
-        for component in results:
+                            components = self._append_component(components, dependency, component_id, status)
+                    # End dependency loop
+            # End component loop
+        # End of result loop
+        return components
+
+    def _get_components_from_results(self, results: Dict[str, Any]) -> list or None:
+        """
+        Process the results dictionary to extract and format component information.
+
+        This function iterates through the results dictionary, identifying components from
+        different sources (files, snippets, and dependencies). It consolidates this information
+        into a list of unique components, each with its associated licenses and other details.
+
+        :param results: A dictionary containing the raw results of a component scan
+        :return: A list of dictionaries, each representing a unique component with its details
+        """
+        if results is None:
+            self.print_stderr('ERROR: Results cannot be empty')
+            return None
+        
+        components = {}
+        # Extract file and snippet components
+        components = self._get_components_data(results, components)
+        # Extract dependency components
+        components = self._get_dependencies_data(results, components)
+        # Convert to list and process licenses
+        results_list = list(components.values())
+        for component in results_list:
             component['licenses'] = list(component['licenses'].values())
 
-        return results
+        return results_list
 
     def generate_table(self, headers, rows, centered_columns=None):
         """
@@ -403,7 +434,6 @@ class PolicyCheck(ScanossBase):
         components = self._get_components_from_results(self.results)
         return components
 
-
 #
 # End of PolicyCheck Class
 #

scanoss/winnowing.py

@@ -32,9 +32,10 @@ import hashlib
 import pathlib
 import platform
 import re
+from typing import Tuple
 
-from crc32c import crc32c
 from binaryornot.check import is_binary
+from crc32c import crc32c
 
 from .scanossbase import ScanossBase
 
@@ -157,7 +158,7 @@ class Winnowing(ScanossBase):
     a list of WFP fingerprints with their corresponding line numbers.
     """
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         size_limit: bool = False,
         debug: bool = False,
@@ -197,6 +198,7 @@ class Winnowing(ScanossBase):
         self.strip_hpsm_ids = strip_hpsm_ids
         self.strip_snippet_ids = strip_snippet_ids
         self.hpsm = hpsm
+        self.is_windows = platform.system() == 'Windows'
         if hpsm:
             self.crc8_maxim_dow_table = []
             self.crc8_generate_table()
@@ -218,11 +220,11 @@ class Winnowing(ScanossBase):
             return byte
         if byte >= ASCII_a:
             return byte
-        if (byte >= 65) and (byte <= 90):
+        if (byte >= ASCII_A) and (byte <= ASCII_Z):
             return byte + 32
         return 0
 
-    def __skip_snippets(self, file: str, src: str) -> bool:
+    def __skip_snippets(self, file: str, src: str) -> bool:  # noqa: PLR0911
         """
         Determine files that are not of interest based on their content or file extension
         Parameters
@@ -351,7 +353,55 @@ class Winnowing(ScanossBase):
             self.print_debug(f'Stripped snippet ids from {file}')
         return wfp
 
-    def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str:
+    def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool]:
+        """Detect the types of line endings present in file contents.
+
+        Args:
+            contents: File contents as bytes.
+
+        Returns:
+            Tuple of (has_crlf, has_lf_only, has_cr_only, has_mixed) indicating which line ending types are present.
+        """
+        has_crlf = b'\r\n' in contents
+        # For LF detection, we need to find LF that's not part of CRLF
+        content_without_crlf = contents.replace(b'\r\n', b'')
+        has_standalone_lf = b'\n' in content_without_crlf
+        # For CR detection, we need to find CR that's not part of CRLF
+        has_standalone_cr = b'\r' in content_without_crlf
+
+        return has_crlf, has_standalone_lf, has_standalone_cr
+
+    def __calculate_opposite_line_ending_hash(self, contents: bytes):
+        """Calculate hash for contents with opposite line endings.
+
+        If the file is primarily Unix (LF), calculates Windows (CRLF) hash.
+        If the file is primarily Windows (CRLF), calculates Unix (LF) hash.
+
+        Args:
+            contents: File contents as bytes.
+
+        Returns:
+            Hash with opposite line endings as hex string, or None if no line endings detected.
+        """
+        has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents)
+
+        if not has_crlf and not has_standalone_lf and not has_standalone_cr:
+            return None
+
+        # Normalize all line endings to LF first
+        normalized = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
+
+        # Determine the dominant line ending type
+        if has_crlf and not has_standalone_lf and not has_standalone_cr:
+            # File is Windows (CRLF) - produce Unix (LF) hash
+            opposite_contents = normalized
+        else:
+            # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash
+            opposite_contents = normalized.replace(b'\n', b'\r\n')
+
+        return hashlib.md5(opposite_contents).hexdigest()
+
+    def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str:  # noqa: PLR0912, PLR0915
         """
         Generate a Winnowing fingerprint (WFP) for the given file contents
         Parameters
@@ -371,7 +421,7 @@ class Winnowing(ScanossBase):
         content_length = len(contents)
         original_filename = file
 
-        if platform.system() == 'Windows':
+        if self.is_windows:
             original_filename = file.replace('\\', '/')
         wfp_filename = repr(original_filename).strip("'")  # return a utf-8 compatible version of the filename
         if self.obfuscate:  # hide the real size of the file and its name, but keep the suffix
@@ -380,6 +430,13 @@ class Winnowing(ScanossBase):
             self.file_map[wfp_filename] = original_filename  # Save the file name map for later (reverse lookup)
 
         wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename)
+
+        # Add opposite line ending hash based on line ending analysis
+        if not bin_file:
+            opposite_hash = self.__calculate_opposite_line_ending_hash(contents)
+            if opposite_hash is not None:
+                wfp += f'fh2={opposite_hash}\n'
+
         # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip
         if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')):
             return wfp
@@ -467,7 +524,7 @@ class Winnowing(ScanossBase):
         for i, byte in enumerate(content):
             c = byte
             if c == ASCII_LF:  # When there is a new line
-                if len(list_normalized):
+                if list_normalized:
                     crc_lines.append(self.crc8_buffer(list_normalized))
                     list_normalized = []
                 elif last_line + 1 == i:

{scanoss-1.24.0.dist-info → scanoss-1.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scanoss
-Version: 1.24.0
+Version: 1.25.0
 Summary: Simple Python library to leverage the SCANOSS APIs
 Home-page: https://scanoss.com
 Author: SCANOSS

{scanoss-1.24.0.dist-info → scanoss-1.25.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ protoc_gen_swagger/options/annotations_pb2.py,sha256=b25EDD6gssUWnFby9gxgcpLIROT
 protoc_gen_swagger/options/annotations_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 protoc_gen_swagger/options/openapiv2_pb2.py,sha256=vYElGp8E1vGHszvWqX97zNG9GFJ7u2QcdK9ouq0XdyI,14939
 protoc_gen_swagger/options/openapiv2_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
-scanoss/__init__.py,sha256=eyFaTBLEaia158sB-9Qg2uEmdiJFZKEB4ow7f4e5wVw,1146
+scanoss/__init__.py,sha256=T3J1XVw8Hyutct8ClOLn1tqQ_NnHlMfxUSVYfLSuVJg,1146
 scanoss/cli.py,sha256=SAB0xuHjEEw20YtYAPSZwrQaVf4JUsm8NRcVArMzd4U,69099
 scanoss/components.py,sha256=b0R9DdKuXqyQiw5nZZwjQ6NJXBr1U9gyx1RI2FP9ozA,14511
 scanoss/constants.py,sha256=FWCZG8gQputKwV7XwvW1GuwDXL4wDLQyVRGdwygg578,320
@@ -25,7 +25,7 @@ scanoss/scantype.py,sha256=gFmyVmKQpHWogN2iCmMj032e_sZo4T92xS3_EH5B3Tc,1310
 scanoss/spdxlite.py,sha256=MQqFgQhIO-yrbRwEAQS77HmRgP5GDxff-2JYLVoceA0,28946
 scanoss/threadeddependencies.py,sha256=CAeZnoYd3d1ayoRvfm_aVjYbaTDLsk6DMWDxkoBPvq0,9866
 scanoss/threadedscanning.py,sha256=38ryN_kZGpzmrd_hkuiY9Sb3tOG248canGCDQDmGEwI,9317
-scanoss/winnowing.py,sha256=68_AL3iI-yR8GMfOD1LemToA_rvbNHeghKTX5-AK698,19254
+scanoss/winnowing.py,sha256=RsR9jRTR3TzS1pEeKQ2RuYlIG8Q7RnUQFfgPLog6B-A,21679
 scanoss/api/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
 scanoss/api/common/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
 scanoss/api/common/v2/__init__.py,sha256=hx-P78xbDsh6WQIigewkJ7Y7y1fqc_eYnyHC5IZTKmo,1122
@@ -57,13 +57,13 @@ scanoss/api/vulnerabilities/__init__.py,sha256=IFrDk_DTJgKSZmmU-nuLXuq_s8sQZlrSC
 scanoss/api/vulnerabilities/v2/__init__.py,sha256=IFrDk_DTJgKSZmmU-nuLXuq_s8sQZlrSCHhIDMJT4r0,1122
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py,sha256=CFhF80av8tenGvn9AIsGEtRJPuV2dC_syA5JLZb2lDw,5464
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py,sha256=HlS4k4Zmx6RIAqaO9I96jD-eyF5yU6Xx04pVm7pdqOg,6864
-scanoss/data/build_date.txt,sha256=PDQnZE2-WtpYAXPgkmN0EbClN0u6qP3jO1XlbqtFxSk,40
+scanoss/data/build_date.txt,sha256=3dG3QmOR7re5yDaUAaCa2noLbeaUxqlhd4ZYGn2-eo0,40
 scanoss/data/scanoss-settings-schema.json,sha256=ClkRYAkjAN0Sk704G8BE_Ok006oQ6YnIGmX84CF8h9w,8798
 scanoss/data/spdx-exceptions.json,sha256=s7UTYxC7jqQXr11YBlIWYCNwN6lRDFTR33Y8rpN_dA4,17953
 scanoss/data/spdx-licenses.json,sha256=A6Z0q82gaTLtnopBfzeIVZjJFxkdRW1g2TuumQc-lII,228794
 scanoss/inspection/__init__.py,sha256=0hjb5ktavp7utJzFhGMPImPaZiHWgilM2HwvTp5lXJE,1122
 scanoss/inspection/copyleft.py,sha256=VynluuCUVxR7uQUz026sp4_yE0Eh3dwpPK0YJ6FNFFw,7579
-scanoss/inspection/policy_check.py,sha256=NYBGKElaeHXKsMOFfSS4IKCiS71h3CKoL20uwhdjaoU,15685
+scanoss/inspection/policy_check.py,sha256=wUUQD2WTBibjf3MOPXKOmB8jFGIX0cd1K3flU0pjTjc,17295
 scanoss/inspection/undeclared_component.py,sha256=0YEiWQ4Q1xu7j4YHLPsS3b5Mf9fplHJdHRXgUoQyF2w,10102
 scanoss/inspection/utils/license_utils.py,sha256=Zb6QLmVJb86lKCwZyBsmwakyAtY1SXa54kUyyKmWMqA,5093
 scanoss/scanners/__init__.py,sha256=D4C0lWLuNp8k_BjQZEc07WZcUgAvriVwQWOk063b0ZU,1122
@@ -76,9 +76,9 @@ scanoss/utils/abstract_presenter.py,sha256=teiDTxBj5jBMCk2T8i4l1BJPf_u4zBLWrtCTF
 scanoss/utils/crc64.py,sha256=TMrwQimSdE6imhFOUL7oAG6Kxu-8qMpGWMuMg8QpSVs,3169
 scanoss/utils/file.py,sha256=62cA9a17TU9ZvfA3FY5HY4-QOajJeSrc8S6xLA_f-3M,2980
 scanoss/utils/simhash.py,sha256=6iu8DOcecPAY36SZjCOzrrLMT9oIE7-gI6QuYwUQ7B0,5793
-scanoss-1.24.0.dist-info/licenses/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
-scanoss-1.24.0.dist-info/METADATA,sha256=wQSSRp3ABZrmz68RkCS802sgrqTAylb9XzXmMGt_7TI,6060
-scanoss-1.24.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-scanoss-1.24.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
-scanoss-1.24.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
-scanoss-1.24.0.dist-info/RECORD,,
+scanoss-1.25.0.dist-info/licenses/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
+scanoss-1.25.0.dist-info/METADATA,sha256=mvajvjxQSlBpII6nFUq44pp7-Kedgf9hXGblfCjqjWU,6060
+scanoss-1.25.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scanoss-1.25.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
+scanoss-1.25.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
+scanoss-1.25.0.dist-info/RECORD,,