scanoss 1.27.1__py3-none-any.whl → 1.43.1__py3-none-any.whl

scanoss/scanner.py

@@ -26,11 +26,11 @@ import datetime
 import json
 import os
 import sys
+from contextlib import nullcontext
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 import importlib_resources
-from progress.bar import Bar
 from progress.spinner import Spinner
 from pypac.parser import PACFile
 
@@ -71,7 +71,6 @@ class Scanner(ScanossBase):
 
     def __init__( # noqa: PLR0913, PLR0915
         self,
-        wfp: str = None,
         scan_output: str = None,
         output_format: str = 'plain',
         debug: bool = False,
@@ -83,7 +82,6 @@ class Scanner(ScanossBase):
         nb_threads: int = 5,
         post_size: int = 32,
         timeout: int = 180,
-        no_wfp_file: bool = False,
         all_extensions: bool = False,
         all_folders: bool = False,
         hidden_files_folders: bool = False,
@@ -107,6 +105,9 @@ class Scanner(ScanossBase):
         skip_md5_ids=None,
         scan_settings: 'ScanossSettings | None' = None,
         req_headers: dict = None,
+        use_grpc: bool = False,
+        skip_headers: bool = False,
+        skip_headers_limit: int = 0,
     ):
         """
         Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning
@@ -116,10 +117,8 @@ class Scanner(ScanossBase):
             skip_folders = []
         if skip_extensions is None:
             skip_extensions = []
-        self.wfp = wfp if wfp else 'scanner_output.wfp'
         self.scan_output = scan_output
         self.output_format = output_format
-        self.no_wfp_file = no_wfp_file
         self.isatty = sys.stderr.isatty()
         self.all_extensions = all_extensions
         self.all_folders = all_folders
@@ -135,6 +134,7 @@ class Scanner(ScanossBase):
 
         self.winnowing = Winnowing(
             debug=debug,
+            trace=trace,
             quiet=quiet,
             skip_snippets=self._skip_snippets,
             all_extensions=all_extensions,
@@ -143,6 +143,8 @@ class Scanner(ScanossBase):
             strip_hpsm_ids=strip_hpsm_ids,
             strip_snippet_ids=strip_snippet_ids,
             skip_md5_ids=skip_md5_ids,
+            skip_headers=skip_headers,
+            skip_headers_limit=skip_headers_limit,
         )
         self.scanoss_api = ScanossApi(
             debug=debug,
@@ -173,6 +175,8 @@ class Scanner(ScanossBase):
             pac=pac,
             grpc_proxy=grpc_proxy,
             req_headers=self.req_headers,
+            ignore_cert_errors=ignore_cert_errors,
+            use_grpc=use_grpc
         )
         self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace)
         self.nb_threads = nb_threads
@@ -360,70 +364,62 @@ class Scanner(ScanossBase):
             operation_type='scanning',
         )
         self.print_msg(f'Searching {scan_dir} for files to fingerprint...')
-        spinner = None
-        if not self.quiet and self.isatty:
-            spinner = Spinner('Fingerprinting ')
-        save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan
-        wfp_list = []
-        scan_block = ''
-        scan_size = 0
-        queue_size = 0
-        file_count = 0  # count all files fingerprinted
-        wfp_file_count = 0  # count number of files in each queue post
-        scan_started = False
-
-        to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir)
-        for to_scan_file in to_scan_files:
-            if self.threaded_scan and self.threaded_scan.stop_scanning():
-                self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
-                break
-            self.print_debug(f'Fingerprinting {to_scan_file}...')
-            if spinner:
-                spinner.next()
-            abs_path = Path(scan_dir, to_scan_file).resolve()
-            wfp = self.winnowing.wfp_for_file(str(abs_path), to_scan_file)
-            if wfp is None or wfp == '':
-                self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.')
-                continue
-            if save_wfps_for_print:
-                wfp_list.append(wfp)
-            file_count += 1
-            if self.threaded_scan:
-                wfp_size = len(wfp.encode('utf-8'))
-                # If the WFP is bigger than the max post size and we already have something stored in the scan block,
-                # add it to the queue
-                if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
-                    self.threaded_scan.queue_add(scan_block)
-                    queue_size += 1
-                    scan_block = ''
-                    wfp_file_count = 0
-                scan_block += wfp
-                scan_size = len(scan_block.encode('utf-8'))
-                wfp_file_count += 1
-                # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue  # noqa: E501
-                if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
-                    self.threaded_scan.queue_add(scan_block)
-                    queue_size += 1
-                    scan_block = ''
-                    wfp_file_count = 0
-                if not scan_started and queue_size > self.nb_threads:  # Start scanning if we have something to do
-                    scan_started = True
-                    if not self.threaded_scan.run(wait=False):
-                        self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.')
-                        success = False
-        # End for loop
-        if self.threaded_scan and scan_block != '':
-            self.threaded_scan.queue_add(scan_block)  # Make sure all files have been submitted
-        if spinner:
-            spinner.finish()
+        spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext()
+
+        with spinner_ctx as spinner:
+            scan_block = ''
+            scan_size = 0
+            queue_size = 0
+            file_count = 0  # count all files fingerprinted
+            wfp_file_count = 0  # count number of files in each queue post
+            scan_started = False
+
+            to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir)
+            for to_scan_file in to_scan_files:
+                if self.threaded_scan and self.threaded_scan.stop_scanning():
+                    self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
+                    break
+                self.print_debug(f'Fingerprinting {to_scan_file}...')
+                if spinner:
+                    spinner.next()
+                abs_path = Path(scan_dir, to_scan_file).resolve()
+                wfp = self.winnowing.wfp_for_file(str(abs_path), to_scan_file)
+                if wfp is None or wfp == '':
+                    self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.')
+                    continue
+                file_count += 1
+                if self.threaded_scan:
+                    wfp_size = len(wfp.encode('utf-8'))
+                    # If the WFP is bigger than the max post size and we already have something
+                    # stored in the scan block, add it to the queue
+                    if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    scan_block += wfp
+                    scan_size = len(scan_block.encode('utf-8'))
+                    wfp_file_count += 1
+                    # If the scan request block (group of WFPs) is larger than the POST size
+                    # or we have reached the file limit, add it to the queue
+                    if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    if not scan_started and queue_size > self.nb_threads:  # Start scanning if we have something to do
+                        scan_started = True
+                        if not self.threaded_scan.run(wait=False):
+                            self.print_stderr(
+                                'Warning: Some errors encountered while scanning. '
+                                'Results might be incomplete.'
+                            )
+                            success = False
+            # End for loop
+            if self.threaded_scan and scan_block != '':
+                self.threaded_scan.queue_add(scan_block)  # Make sure all files have been submitted
 
         if file_count > 0:
-            if save_wfps_for_print:  # Write a WFP file if no threading is requested
-                self.print_debug(f'Writing fingerprints to {self.wfp}')
-                with open(self.wfp, 'w') as f:
-                    f.write(''.join(wfp_list))
-            else:
-                self.print_debug(f'Skipping writing WFP file {self.wfp}')
             if self.threaded_scan:
                 success = self.__run_scan_threaded(scan_started, file_count)
         else:
@@ -628,71 +624,62 @@ class Scanner(ScanossBase):
             skip_extensions=self.skip_extensions,
             operation_type='scanning',
         )
-        spinner = None
-        if not self.quiet and self.isatty:
-            spinner = Spinner('Fingerprinting ')
-        save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan
-        wfp_list = []
-        scan_block = ''
-        scan_size = 0
-        queue_size = 0
-        file_count = 0  # count all files fingerprinted
-        wfp_file_count = 0  # count number of files in each queue post
-        scan_started = False
+        spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext()
+
+        with spinner_ctx as spinner:
+            scan_block = ''
+            scan_size = 0
+            queue_size = 0
+            file_count = 0  # count all files fingerprinted
+            wfp_file_count = 0  # count number of files in each queue post
+            scan_started = False
+
+            to_scan_files = file_filters.get_filtered_files_from_files(files)
+            for file in to_scan_files:
+                if self.threaded_scan and self.threaded_scan.stop_scanning():
+                    self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
+                    break
+                self.print_debug(f'Fingerprinting {file}...')
+                if spinner:
+                    spinner.next()
+                wfp = self.winnowing.wfp_for_file(file, file)
+                if wfp is None or wfp == '':
+                    self.print_debug(f'No WFP returned for {file}. Skipping.')
+                    continue
+                file_count += 1
+                if self.threaded_scan:
+                    wfp_size = len(wfp.encode('utf-8'))
+                    # If the WFP is bigger than the max post size and we already have something
+                    # stored in the scan block, add it to the queue
+                    if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    scan_block += wfp
+                    scan_size = len(scan_block.encode('utf-8'))
+                    wfp_file_count += 1
+                    # If the scan request block (group of WFPs) is larger than the POST size
+                    # or we have reached the file limit, add it to the queue
+                    if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    if not scan_started and queue_size > self.nb_threads:  # Start scanning if we have something to do
+                        scan_started = True
+                        if not self.threaded_scan.run(wait=False):
+                            self.print_stderr(
+                                'Warning: Some errors encountered while scanning. '
+                                'Results might be incomplete.'
+                            )
+                            success = False
 
-        to_scan_files = file_filters.get_filtered_files_from_files(files)
-        for file in to_scan_files:
-            if self.threaded_scan and self.threaded_scan.stop_scanning():
-                self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
-                break
-            self.print_debug(f'Fingerprinting {file}...')
-            if spinner:
-                spinner.next()
-            wfp = self.winnowing.wfp_for_file(file, file)
-            if wfp is None or wfp == '':
-                self.print_debug(f'No WFP returned for {file}. Skipping.')
-                continue
-            if save_wfps_for_print:
-                wfp_list.append(wfp)
-            file_count += 1
-            if self.threaded_scan:
-                wfp_size = len(wfp.encode('utf-8'))
-                # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue  # noqa: E501
-                if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
-                    self.threaded_scan.queue_add(scan_block)
-                    queue_size += 1
-                    scan_block = ''
-                    wfp_file_count = 0
-                scan_block += wfp
-                scan_size = len(scan_block.encode('utf-8'))
-                wfp_file_count += 1
-                # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue  # noqa: E501
-                if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
-                    self.threaded_scan.queue_add(scan_block)
-                    queue_size += 1
-                    scan_block = ''
-                    wfp_file_count = 0
-                if not scan_started and queue_size > self.nb_threads:  # Start scanning if we have something to do
-                    scan_started = True
-                    if not self.threaded_scan.run(wait=False):
-                        self.print_stderr(
-                            'Warning: Some errors encounted while scanning. Results might be incomplete.'
-                        )
-                        success = False
-
-        # End for loop
-        if self.threaded_scan and scan_block != '':
-            self.threaded_scan.queue_add(scan_block)  # Make sure all files have been submitted
-        if spinner:
-            spinner.finish()
+            # End for loop
+            if self.threaded_scan and scan_block != '':
+                self.threaded_scan.queue_add(scan_block)  # Make sure all files have been submitted
 
         if file_count > 0:
-            if save_wfps_for_print:  # Write a WFP file if no threading is requested
-                self.print_debug(f'Writing fingerprints to {self.wfp}')
-                with open(self.wfp, 'w') as f:
-                    f.write(''.join(wfp_list))
-            else:
-                self.print_debug(f'Skipping writing WFP file {self.wfp}')
             if self.threaded_scan:
                 success = self.__run_scan_threaded(scan_started, file_count)
         else:
@@ -755,125 +742,22 @@ class Scanner(ScanossBase):
                 success = False
         return success
 
-    def scan_wfp_file(self, file: str = None) -> bool:  # noqa: PLR0912, PLR0915
-        """
-        Scan the contents of the specified WFP file (in the current process)
-        :param file: Scan the contents of the specified WFP file (in the current process)
-        :return: True if successful, False otherwise
-        """
-        success = True
-        wfp_file = file if file else self.wfp  # If a WFP file is specified, use it, otherwise us the default
-        if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
-            raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
-        file_count = Scanner.__count_files_in_wfp_file(wfp_file)
-        cur_files = 0
-        cur_size = 0
-        batch_files = 0
-        wfp = ''
-        max_component = {'name': '', 'hits': 0}
-        components = {}
-        self.print_debug(f'Found {file_count} files to process.')
-        raw_output = '{\n'
-        file_print = ''
-        bar = None
-        if not self.quiet and self.isatty:
-            bar = Bar('Scanning', max=file_count)
-            bar.next(0)
-        with open(wfp_file) as f:
-            for line in f:
-                if line.startswith(WFP_FILE_START):
-                    if file_print:
-                        wfp += file_print  # Store the WFP for the current file
-                        cur_size = len(wfp.encode('utf-8'))
-                    file_print = line  # Start storing the next file
-                    cur_files += 1
-                    batch_files += 1
-                else:
-                    file_print += line  # Store the rest of the WFP for this file
-                l_size = cur_size + len(file_print.encode('utf-8'))
-                # Hit the max post size, so sending the current batch and continue processing
-                if l_size >= self.max_post_size and wfp:
-                    self.print_debug(
-                        f'Sending {batch_files} ({cur_files}) of'
-                        f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.'
-                    )
-                    if self.debug and cur_size > self.max_post_size:
-                        Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}')
-                    scan_resp = self.scanoss_api.scan(wfp, max_component['name'])  # Scan current WFP and store
-                    if bar:
-                        bar.next(batch_files)
-                    if scan_resp is not None:
-                        for key, value in scan_resp.items():
-                            raw_output += '  "%s":%s,' % (key, json.dumps(value, indent=2))
-                            for v in value:
-                                if hasattr(v, 'get'):
-                                    if v.get('id') != 'none':
-                                        vcv = '%s:%s:%s' % (v.get('vendor'), v.get('component'), v.get('version'))
-                                        components[vcv] = components[vcv] + 1 if vcv in components else 1
-                                        if max_component['hits'] < components[vcv]:
-                                            max_component['name'] = v.get('component')
-                                            max_component['hits'] = components[vcv]
-                                else:
-                                    Scanner.print_stderr(f'Warning: Unknown value: {v}')
-                    else:
-                        success = False
-                    batch_files = 0
-                    wfp = ''
-        if file_print:
-            wfp += file_print  # Store the WFP for the current file
-        if wfp:
-            self.print_debug(
-                f'Sending {batch_files} ({cur_files}) of'
-                f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.'
-            )
-            scan_resp = self.scanoss_api.scan(wfp, max_component['name'])  # Scan current WFP and store
-            if bar:
-                bar.next(batch_files)
-            first = True
-            if scan_resp is not None:
-                for key, value in scan_resp.items():
-                    if first:
-                        raw_output += '  "%s":%s' % (key, json.dumps(value, indent=2))
-                        first = False
-                    else:
-                        raw_output += ',\n  "%s":%s' % (key, json.dumps(value, indent=2))
-            else:
-                success = False
-        raw_output += '\n}'
-        if bar:
-            bar.finish()
-        if self.output_format == 'plain':
-            self.__log_result(raw_output)
-        elif self.output_format == 'cyclonedx':
-            cdx = CycloneDx(self.debug, self.scan_output)
-            cdx.produce_from_str(raw_output)
-        elif self.output_format == 'spdxlite':
-            spdxlite = SpdxLite(self.debug, self.scan_output)
-            success = spdxlite.produce_from_str(raw_output)
-        elif self.output_format == 'csv':
-            csvo = CsvOutput(self.debug, self.scan_output)
-            csvo.produce_from_str(raw_output)
-        else:
-            self.print_stderr(f'ERROR: Unknown output format: {self.output_format}')
-            success = False
-
-        return success
-
-    def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) -> bool:
+    def scan_wfp_with_options(self, wfp_file: str, deps_file: str, file_map: dict = None) -> bool:
         """
         Scan the given WFP file for whatever scaning options that have been configured
-        :param wfp: WFP file to scan
+        :param wfp_file: WFP file to scan
         :param deps_file: pre-parsed dependency file to decorate
         :param file_map: mapping of obfuscated files back into originals
         :return: True if successful, False otherwise
         """
         success = True
-        wfp_file = wfp if wfp else self.wfp  # If a WFP file is specified, use it, otherwise us the default
+        if not wfp_file:
+            raise Exception('ERROR: Please specify a WFP file to scan')
         if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
             raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
 
         if not self.is_file_or_snippet_scan() and not self.is_dependency_scan():
-            raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp}')
+            raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp_file}')
 
         if self.scan_output:
             self.print_msg(f'Writing results to {self.scan_output}...')
@@ -888,14 +772,15 @@ class Scanner(ScanossBase):
                 success = False
         return success
 
-    def scan_wfp_file_threaded(self, file: str = None) -> bool:
+    def scan_wfp_file_threaded(self, wfp_file: str) -> bool:  # noqa: PLR0912
         """
         Scan the contents of the specified WFP file (threaded)
-        :param file: WFP file to scan (optional)
+        :param wfp_file: WFP file to scan
         return: True if successful, False otherwise
         """
         success = True
-        wfp_file = file if file else self.wfp  # If a WFP file is specified, use it, otherwise us the default
+        if not wfp_file:
+            raise Exception('ERROR: Please specify a WFP file to scan')
         if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
             raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
         cur_size = 0
@@ -1049,19 +934,16 @@ class Scanner(ScanossBase):
         )
         wfps = ''
         self.print_msg(f'Searching {scan_dir} for files to fingerprint...')
-        spinner = None
-        if not self.quiet and self.isatty:
-            spinner = Spinner('Fingerprinting ')
-
-        to_fingerprint_files = file_filters.get_filtered_files_from_folder(scan_dir)
-        for file in to_fingerprint_files:
-            if spinner:
-                spinner.next()
-            abs_path = Path(scan_dir, file).resolve()
-            self.print_debug(f'Fingerprinting {file}...')
-            wfps += self.winnowing.wfp_for_file(str(abs_path), file)
-        if spinner:
-            spinner.finish()
+        spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext()
+
+        with spinner_ctx as spinner:
+            to_fingerprint_files = file_filters.get_filtered_files_from_folder(scan_dir)
+            for file in to_fingerprint_files:
+                if spinner:
+                    spinner.next()
+                abs_path = Path(scan_dir, file).resolve()
+                self.print_debug(f'Fingerprinting {file}...')
+                wfps += self.winnowing.wfp_for_file(str(abs_path), file)
         if wfps:
             if wfp_file:
                 self.print_stderr(f'Writing fingerprints to {wfp_file}')