scanoss 1.8.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

scanoss/__init__.py

@@ -22,4 +22,4 @@
    THE SOFTWARE.
 """
 
-__version__ = '1.8.0'
+__version__ = '1.10.0'

scanoss/cli.py

@@ -86,7 +86,6 @@ def setup_args() -> None:
                              '256: disable best match only, 512: hide identified files, '
                              '1024: enable download_url, 2048: enable GitHub full path, '
                              '4096: disable extended server stats)')
-    p_scan.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets')
     p_scan.add_argument('--post-size', '-P', type=int, default=32,
                         help='Number of kilobytes to limit the post to while scanning (optional - default 32)')
     p_scan.add_argument('--timeout', '-M', type=int, default=180,
@@ -94,17 +93,12 @@ def setup_args() -> None:
     p_scan.add_argument('--retry', '-R', type=int, default=5,
                         help='Retry limit for API communication (optional - default 5)')
     p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation')
-    p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions')
-    p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders')
-    p_scan.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders')
-    p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate file paths and names')
     p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning')
     p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only')
     p_scan.add_argument('--sc-command', type=str,
                         help='Scancode command and path if required (optional - default scancode).')
     p_scan.add_argument('--sc-timeout', type=int, default=600,
                         help='Timeout (in seconds) for scancode to complete (optional - default 600)')
-    p_scan.add_argument('--hpsm', '-H', action='store_true', help='Scan using High Precision Snippet Matching')
 
     # Sub-command: fingerprint
     p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'],
@@ -116,12 +110,6 @@ def setup_args() -> None:
     p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME',  type=str,
                        help='Fingerprint the file contents supplied via STDIN (optional)')
     p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
-    p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints')
-    p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets')
-    p_wfp.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions')
-    p_wfp.add_argument('--all-folders', action='store_true', help='Fingerprint all folders')
-    p_wfp.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders')
-    p_wfp.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.')
 
     # Sub-command: dependency
     p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'],
@@ -260,6 +248,19 @@ def setup_args() -> None:
                        help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)')
         p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors')
 
+    # Global Scan/Fingerprint filter options
+    for p in [p_scan, p_wfp]:
+        p.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints')
+        p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions')
+        p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders')
+        p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders')
+        p.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.')
+        p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets')
+        p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.')
+        p.add_argument('--skip-folder', '-O', type=str, action='append', help='Folder to skip.')
+        p.add_argument('--skip-size', '-Z', type=int, default=0,
+                       help='Minimum file size to consider for fingerprinting (optional - default 0 bytes [unlimited])')
+
     # Global Scan/GRPC options
     for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]:
         p.add_argument('--key', '-k', type=str,
@@ -374,8 +375,9 @@ def wfp(parser, args):
     scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value  # Skip snippet generation or not
     scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate,
                       scan_options=scan_options, all_extensions=args.all_extensions,
-                      all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm)
-
+                      all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm,
+                      skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder
+                      )
     if args.stdin:
         contents = sys.stdin.buffer.read()
         scanner.wfp_contents(args.stdin, contents, scan_output)
@@ -406,7 +408,7 @@ def get_scan_options(args):
     scan_dependencies = 0
     if args.skip_snippets:
         scan_snippets = 0
-    if args.dependencies:
+    if args.dependencies or args.dep:
         scan_dependencies = ScanType.SCAN_DEPENDENCIES.value
     if args.dependencies_only:
         scan_files = scan_snippets = 0
@@ -437,8 +439,8 @@ def scan(parser, args):
         args: Namespace
             Parsed arguments
     """
-    if not args.scan_dir and not args.wfp and not args.stdin:
-        print_stderr('Please specify a file/folder, fingerprint (--wfp) or STDIN (--stdin)')
+    if not args.scan_dir and not args.wfp and not args.stdin and not args.dep:
+        print_stderr('Please specify a file/folder, fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)')
         parser.parse_args([args.subparser, '-h'])
         exit(1)
     if args.pac and args.proxy:
@@ -530,16 +532,17 @@ def scan(parser, args):
                       scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command,
                       grpc_url=args.api2url, obfuscate=args.obfuscate,
                       ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy,
-                      pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm
+                      pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm,
+                      skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder
                       )
     if args.wfp:
         if not scanner.is_file_or_snippet_scan():
             print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})')
             exit(1)
-        if args.threads > 1:
-            scanner.scan_wfp_file_threaded(args.wfp)
-        else:
-            scanner.scan_wfp_file(args.wfp)
+        if scanner.is_dependency_scan() and not args.dep:
+            print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)')
+            exit(1)
+        scanner.scan_wfp_with_options(args.wfp, args.dep)
     elif args.stdin:
         contents = sys.stdin.buffer.read()
         if not scanner.scan_contents(args.stdin, contents):
@@ -549,14 +552,20 @@ def scan(parser, args):
             print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.')
             exit(1)
         if os.path.isdir(args.scan_dir):
-            if not scanner.scan_folder_with_options(args.scan_dir, scanner.winnowing.file_map):
+            if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map):
                 exit(1)
         elif os.path.isfile(args.scan_dir):
-            if not scanner.scan_file_with_options(args.scan_dir, scanner.winnowing.file_map):
+            if not scanner.scan_file_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map):
                 exit(1)
         else:
             print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.')
             exit(1)
+    elif args.dep:
+        if not args.dependencies_only:
+            print_stderr(f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.')
+            exit(1)
+        if not scanner.scan_folder_with_options(".", args.dep, scanner.winnowing.file_map):
+            exit(1)
     else:
         print_stderr('No action found to process')
         exit(1)

scanoss/data/build_date.txt

@@ -1 +1 @@
-date: 20231114111907, utime: 1699960747
+date: 20240209161953, utime: 1707495593

scanoss/scancodedeps.py

@@ -215,6 +215,26 @@ class ScancodeDeps(ScanossBase):
             self.print_stderr(f'ERROR: Issue running scancode dependency scan on {what_to_scan}: {e}')
             return False
         return True
+
+    def load_from_file(self, json_file: str = None) -> json:
+        """
+        Load the parsed JSON dependencies file and return the json object
+        :param json_file: dependency json file
+        :return: SCANOSS dependency JSON
+        """
+        if not json_file:
+            self.print_stderr('ERROR: No parsed JSON file provided to load.')
+            return None
+        if not os.path.isfile(json_file):
+            self.print_stderr(f'ERROR: parsed JSON file does not exist or is not a file: {json_file}')
+            return None
+        with open(json_file, 'r') as f:
+            try:
+                return json.loads(f.read())
+            except Exception as e:
+                self.print_stderr(f'ERROR: Problem loading input JSON: {e}')
+        return None
+
 #
 # End of ScancodeDeps Class
 #

scanoss/scanner.py

@@ -58,7 +58,7 @@ FILTERED_DIRS = {  # Folders to skip
 FILTERED_DIR_EXT = {  # Folder endings to skip
     ".egg-info"
 }
-FILTERED_EXT = {  # File extensions to skip
+FILTERED_EXT = [  # File extensions to skip
     ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am",
     ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf",
     ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css",
@@ -78,7 +78,7 @@ FILTERED_EXT = {  # File extensions to skip
     # File endings
     "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice",
     "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"
-}
+]
 FILTERED_FILES = {  # Files to skip
     "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar",
     "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile"
@@ -100,12 +100,17 @@ class Scanner(ScanossBase):
                  all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False,
                  scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None,
                  obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None,
-                 ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False
+                 ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False,
+                 skip_size: int = 0, skip_extensions=None, skip_folders=None
                  ):
         """
         Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning
         """
         super().__init__(debug, trace, quiet)
+        if skip_folders is None:
+            skip_folders = []
+        if skip_extensions is None:
+            skip_extensions = []
         self.wfp = wfp if wfp else "scanner_output.wfp"
         self.scan_output = scan_output
         self.output_format = output_format
@@ -117,6 +122,8 @@ class Scanner(ScanossBase):
         self.scan_options = scan_options
         self._skip_snippets = True if not scan_options & ScanType.SCAN_SNIPPETS.value else False
         self.hpsm = hpsm
+        self.skip_folders = skip_folders
+        self.skip_size = skip_size
         ver_details = Scanner.version_details()
 
         self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets,
@@ -143,6 +150,9 @@ class Scanner(ScanossBase):
         self.post_file_count = post_size if post_size > 0 else 32  # Max number of files for any given POST (default 32)
         if self._skip_snippets:
             self.max_post_size = 8 * 1024  # 8k Max post size if we're skipping snippets
+        self.skip_extensions = FILTERED_EXT
+        if skip_extensions:  # Append extra file extensions to skip
+            self.skip_extensions.extend(skip_extensions)
 
     def __filter_files(self, files: list) -> list:
         """
@@ -160,8 +170,8 @@ class Scanner(ScanossBase):
                 if f_lower in FILTERED_FILES:  # Check for exact files to ignore
                     ignore = True
                 if not ignore:
-                    for ending in FILTERED_EXT:  # Check for file endings to ignore
-                        if f_lower.endswith(ending):
+                    for ending in self.skip_extensions:  # Check for file endings to ignore (static and user supplied)
+                        if ending and f_lower.endswith(ending):
                             ignore = True
                             break
             if not ignore:
@@ -181,10 +191,12 @@ class Scanner(ScanossBase):
                 ignore = True
             if not ignore and not self.all_folders:  # Skip this check if we're allowing all folders
                 d_lower = d.lower()
-                if d_lower in FILTERED_DIRS:  # Ignore specific folders
+                if d_lower in FILTERED_DIRS:  # Ignore specific folders (case insensitive)
+                    ignore = True
+                elif self.skip_folders and d in self.skip_folders:  # Ignore user-supplied folders (case sensitive)
                     ignore = True
                 if not ignore:
-                    for de in FILTERED_DIR_EXT:  # Ignore specific folder endings
+                    for de in FILTERED_DIR_EXT:  # Ignore specific folder endings (case insensitive)
                         if d_lower.endswith(de):
                             ignore = True
                             break
@@ -313,10 +325,11 @@ class Scanner(ScanossBase):
             return True
         return False
 
-    def scan_folder_with_options(self, scan_dir: str, file_map: dict = None) -> bool:
+    def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_map: dict = None) -> bool:
         """
         Scan the given folder for whatever scaning options that have been configured
         :param scan_dir: directory to scan
+        :param deps_file: pre-parsed dependency file to decorate
         :param file_map: mapping of obfuscated files back into originals
         :return: True if successful, False otherwise
         """
@@ -331,7 +344,7 @@ class Scanner(ScanossBase):
         if self.scan_output:
             self.print_msg(f'Writing results to {self.scan_output}...')
         if self.is_dependency_scan():
-            if not self.threaded_deps.run(what_to_scan=scan_dir, wait=False):  # Kick off a background dependency scan
+            if not self.threaded_deps.run(what_to_scan=scan_dir, deps_file=deps_file, wait=False):  # Kick off a background dependency scan
                 success = False
         if self.is_file_or_snippet_scan():
             if not self.scan_folder(scan_dir):
@@ -384,7 +397,8 @@ class Scanner(ScanossBase):
                 except Exception as e:
                     self.print_trace(
                         f'Ignoring missing symlink file: {file} ({e})')  # Can fail if there is a broken symlink
-                if f_size > 0:  # Ignore broken links and empty files
+                # Ignore broken links and empty files or if a user-specified size limit is supplied
+                if f_size > 0 and (self.skip_size <= 0 or f_size > self.skip_size):
                     self.print_trace(f'Fingerprinting {path}...')
                     if spinner:
                         spinner.next()
@@ -542,10 +556,11 @@ class Scanner(ScanossBase):
             success = False
         return success
 
-    def scan_file_with_options(self, file: str, file_map: dict = None) -> bool:
+    def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None) -> bool:
         """
         Scan the given file for whatever scaning options that have been configured
         :param file: file to scan
+        :param deps_file: pre-parsed dependency file to decorate
         :param file_map: mapping of obfuscated files back into originals
         :return: True if successful, False otherwise
         """
@@ -560,7 +575,7 @@ class Scanner(ScanossBase):
         if self.scan_output:
             self.print_msg(f'Writing results to {self.scan_output}...')
         if self.is_dependency_scan():
-            if not self.threaded_deps.run(what_to_scan=file, wait=False):  # Kick off a background dependency scan
+            if not self.threaded_deps.run(what_to_scan=file, deps_file=deps_file, wait=False):  # Kick off a background dependency scan
                 success = False
         if self.is_file_or_snippet_scan():
             if not self.scan_file(file):
@@ -596,6 +611,117 @@ class Scanner(ScanossBase):
             success = False
         return success
 
+    def scan_files(self, files: []) -> bool:
+        """
+        Scan the specified list of files, producing fingerprints, send to the SCANOSS API and return results
+        Please note that by providing an explicit list you bypass any exclusions that may be defined on the scanner
+        :param files: list[str]
+                    List of filenames to scan
+        :return True if successful, False otherwise
+        """
+        success = True
+        if not files:
+            raise Exception(f"ERROR: Please provide a non-empty list of filenames to scan")
+        self.print_msg(f'Scanning {len(files)} files...')
+        spinner = None
+        if not self.quiet and self.isatty:
+            spinner = Spinner('Fingerprinting ')
+        save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan
+        wfp_list = []
+        scan_block = ''
+        scan_size = 0
+        queue_size = 0
+        file_count = 0  # count all files fingerprinted
+        wfp_file_count = 0  # count number of files in each queue post
+        scan_started = False
+        for file in files:
+            if self.threaded_scan and self.threaded_scan.stop_scanning():
+                self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
+                break
+            f_size = 0
+            try:
+                f_size = os.stat(file).st_size
+            except Exception as e:
+                self.print_trace(
+                    f'Ignoring missing symlink file: {file} ({e})')  # Can fail if there is a broken symlink
+            if f_size > 0:  # Ignore broken links and empty files
+                self.print_trace(f'Fingerprinting {file}...')
+                if spinner:
+                    spinner.next()
+                wfp = self.winnowing.wfp_for_file(file, file)
+                if wfp is None or wfp == '':
+                        self.print_stderr(f'Warning: No WFP returned for {file}')
+                        continue
+                if save_wfps_for_print:
+                        wfp_list.append(wfp)
+                file_count += 1
+                if self.threaded_scan:
+                    wfp_size = len(wfp.encode("utf-8"))
+                    # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue
+                    if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    scan_block += wfp
+                    scan_size = len(scan_block.encode("utf-8"))
+                    wfp_file_count += 1
+                    # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue
+                    if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
+                        self.threaded_scan.queue_add(scan_block)
+                        queue_size += 1
+                        scan_block = ''
+                        wfp_file_count = 0
+                    if not scan_started and queue_size > self.nb_threads:  # Start scanning if we have something to do
+                        scan_started = True
+                        if not self.threaded_scan.run(wait=False):
+                            self.print_stderr(
+                                f'Warning: Some errors encounted while scanning. Results might be incomplete.')
+                            success = False
+        # End for loop
+        if self.threaded_scan and scan_block != '':
+            self.threaded_scan.queue_add(scan_block)  # Make sure all files have been submitted
+        if spinner:
+            spinner.finish()
+
+        if file_count > 0:
+            if save_wfps_for_print:  # Write a WFP file if no threading is requested
+                self.print_debug(f'Writing fingerprints to {self.wfp}')
+                with open(self.wfp, 'w') as f:
+                    f.write(''.join(wfp_list))
+            else:
+                self.print_debug(f'Skipping writing WFP file {self.wfp}')
+            if self.threaded_scan:
+                success = self.__run_scan_threaded(scan_started, file_count)
+        else:
+            Scanner.print_stderr(f'Warning: No files found to scan from: {files}')
+        return success
+    
+    def scan_files_with_options(self, files: [], deps_file: str = None, file_map: dict = None) -> bool:
+        """
+        Scan the given list of files for whatever scaning options that have been configured
+        :param files: list of files to scan
+        :param deps_file: pre-parsed dependency file to decorate
+        :param file_map: mapping of obfuscated files back into originals
+        :return: True if successful, False otherwise
+        """
+        success = True
+        if not files:
+            raise Exception(f"ERROR: Please specify a list of files to scan")
+        if not self.is_file_or_snippet_scan():
+            raise Exception(f"ERROR: file or snippet scan options have to be set to scan files: {files}")
+        if self.is_dependency_scan() or deps_file:
+            raise Exception(f"ERROR: The dependency scan option is currently not supported when scanning a list of files")
+        if self.scan_output:
+            self.print_msg(f'Writing results to {self.scan_output}...')
+        if self.is_file_or_snippet_scan():
+            if not self.scan_files(files):
+                success = False
+        if self.threaded_scan:
+            if not self.__finish_scan_threaded(file_map):
+                success = False
+        return success
+
     def scan_contents(self, filename: str, contents: bytes) -> bool:
         """
         Scan the given contents as a file
@@ -725,11 +851,39 @@ class Scanner(ScanossBase):
 
         return success
 
-    def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> bool:
+    def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) -> bool:
+        """
+        Scan the given WFP file for whatever scaning options that have been configured
+        :param wfp: WFP file to scan
+        :param deps_file: pre-parsed dependency file to decorate
+        :param file_map: mapping of obfuscated files back into originals
+        :return: True if successful, False otherwise
+        """
+        success = True
+        wfp_file = wfp if wfp else self.wfp  # If a WFP file is specified, use it, otherwise us the default
+        if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
+            raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}")
+
+        if not self.is_file_or_snippet_scan() and not self.is_dependency_scan():
+            raise Exception(f"ERROR: No scan options defined to scan WFP: {wfp}")
+
+        if self.scan_output:
+            self.print_msg(f'Writing results to {self.scan_output}...')
+        if self.is_dependency_scan():
+            if not self.threaded_deps.run(deps_file=deps_file, wait=False):  # Kick off a background dependency scan
+                success = False
+        if self.is_file_or_snippet_scan():
+            if not self.scan_wfp_file_threaded(wfp_file):
+                success = False
+        if self.threaded_scan:
+            if not self.__finish_scan_threaded(file_map):
+                success = False
+        return success
+
+    def scan_wfp_file_threaded(self, file: str = None) -> bool:
         """
         Scan the contents of the specified WFP file (threaded)
         :param file: WFP file to scan (optional)
-        :param file_map: mapping of obfuscated files back into originals (optional)
         return: True if successful, False otherwise
         """
         success = True
@@ -778,8 +932,6 @@ class Scanner(ScanossBase):
 
         if not self.__run_scan_threaded(scan_started, file_count):
             success = False
-        elif not self.__finish_scan_threaded(file_map):
-            success = False
         return success
 
     def scan_wfp(self, wfp: str) -> bool:

scanoss/scanossgrpc.py

@@ -43,7 +43,8 @@ from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, Dep
 from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest
 from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse
 from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse
-from .api.components.v2.scanoss_components_pb2 import CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse
+from .api.components.v2.scanoss_components_pb2 import (CompSearchRequest, CompSearchResponse,
+                                                       CompVersionRequest, CompVersionResponse)
 from .scanossbase import ScanossBase
 from . import __version__
 

scanoss/threadeddependencies.py

@@ -31,6 +31,8 @@ from .scancodedeps import ScancodeDeps
 from .scanossbase import ScanossBase
 from .scanossgrpc import ScanossGrpc
 
+DEP_FILE_PREFIX = "file="  # Default prefix to signify an existing parsed dependency file
+
 
 @dataclass
 class ThreadedDependencies(ScanossBase):
@@ -64,18 +66,23 @@ class ThreadedDependencies(ScanossBase):
                 return resp
         return None
 
-    def run(self, what_to_scan: str = None, wait: bool = True) -> bool:
+    def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True) -> bool:
         """
         Initiate a background scan for the specified file/dir
         :param what_to_scan: file/folder to scan
+        :param deps_file: file to decorate instead of scan (overrides what_to_scan option)
         :param wait: wait for completion
         :return: True if successful, False if error encountered
         """
         what_to_scan = what_to_scan if what_to_scan else self.what_to_scan
         self._errors = False
         try:
-            self.print_msg(f'Searching {what_to_scan} for dependencies...')
-            self.inputs.put(what_to_scan)   # Set up an input queue to enable the parent to wait for completion
+            if deps_file:                                                  # Decorate the given dependencies file
+                self.print_msg(f'Decorating {deps_file} dependencies...')
+                self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}')           # Add to queue and have parent wait on it
+            else:                                                          # Search for dependencies to decorate
+                self.print_msg(f'Searching {what_to_scan} for dependencies...')
+                self.inputs.put(what_to_scan)                              # Add to queue and have parent wait on it
             self._thread = threading.Thread(target=self.scan_dependencies, daemon=True)
             self._thread.start()
         except Exception as e:
@@ -87,22 +94,27 @@ class ThreadedDependencies(ScanossBase):
 
     def scan_dependencies(self) -> None:
         """
-        Scan for dependencies from the given file/dir (from the input queue)
+        Scan for dependencies from the given file/dir or from an input file (from the input queue).
         """
         current_thread = threading.get_ident()
         self.print_trace(f'Starting dependency worker {current_thread}...')
         try:
-            what_to_scan = self.inputs.get(timeout=5)                # Begin processing the dependency request
-            if not self.sc_deps.run_scan(what_to_scan=what_to_scan):
-                self._errors = True
-            else:
-                deps = self.sc_deps.produce_from_file()
+            what_to_scan = self.inputs.get(timeout=5)            # Begin processing the dependency request
+            deps = None
+            if what_to_scan.startswith(DEP_FILE_PREFIX):         # We have a pre-parsed dependency file, load it
+                deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX))
+            else:                                                # Search the file/folder for dependency files to parse
+                if not self.sc_deps.run_scan(what_to_scan=what_to_scan):
+                    self._errors = True
+                else:
+                    deps = self.sc_deps.produce_from_file()
+            if not self._errors:
                 if deps is None:
                     self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}')
                     self._errors = True
-                elif not deps:
-                    self.print_trace(f'No dependencies found to decorate for: {what_to_scan}')
-                else:                         # TODO add API call to get dep data
+                elif not deps or len(deps.get("files", [])) == 0:
+                    self.print_debug(f'No dependencies found to decorate for: {what_to_scan}')
+                else:
                     decorated_deps = self.grpc_api.get_dependencies(deps)
                     if decorated_deps:
                         self.output.put(decorated_deps)

{scanoss-1.8.0.dist-info → scanoss-1.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scanoss
-Version: 1.8.0
+Version: 1.10.0
 Summary: Simple Python library to leverage the SCANOSS APIs
 Home-page: https://scanoss.com
 Author: SCANOSS

{scanoss-1.8.0.dist-info → scanoss-1.10.0.dist-info}/RECORD

@@ -4,20 +4,20 @@ protoc_gen_swagger/options/annotations_pb2.py,sha256=b25EDD6gssUWnFby9gxgcpLIROT
 protoc_gen_swagger/options/annotations_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 protoc_gen_swagger/options/openapiv2_pb2.py,sha256=vYElGp8E1vGHszvWqX97zNG9GFJ7u2QcdK9ouq0XdyI,14939
 protoc_gen_swagger/options/openapiv2_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
-scanoss/__init__.py,sha256=N3xlYVsAadBxzi2AhmMaGQgaO7FkR3AT8296M7P8svI,1162
-scanoss/cli.py,sha256=M2MODqfg8MMCbpwqBbLEx0W5xI8lnJZyoYRxjY7qLWA,41016
+scanoss/__init__.py,sha256=kNW4sX9iGCu_4Qlaeuymupek7DpOaKBVIfL8PoLvQVE,1163
+scanoss/cli.py,sha256=V-T9_tuG-2GfjULxPjtw_d-mEVZgBlOs3KbAL-tVCv4,41637
 scanoss/components.py,sha256=ZHZ1KA69shxOASZK7USD9yPTITpAc_RXL5q5zpDK23o,12590
 scanoss/csvoutput.py,sha256=hBwr_Fc6mBdOdXgyQcdFrockYH-PJ0jblowlExJ6OPg,9925
 scanoss/cyclonedx.py,sha256=dPhj6sdwl2P8viC-sicAOLZzyklUR82NGFHaEeGYpeA,12065
 scanoss/filecount.py,sha256=o7xb6m387ucnsU4H1OXGzf_AdWsudhAHe49T8uX4Ieo,6660
-scanoss/scancodedeps.py,sha256=e2FTCxBEkABrOmHlz7TNajn4EMPRXxBjHI_9GAjAodU,9996
-scanoss/scanner.py,sha256=-amQOrsf-ZnUQRaEe8i6uHEu_BbC4gSjhC4LaFk7OAc,42141
+scanoss/scancodedeps.py,sha256=dPJsv9BmEsaM1IEzceJCnwLyu6Z0JwPposxdY4q0DAg,10775
+scanoss/scanner.py,sha256=04cVbQI1K4Twwdu4kORKqACtvx4xjHOcOPEzUW_snpU,50149
 scanoss/scanossapi.py,sha256=JU5B_TgaFs1hQn0W7RaHm9jBmZXFpFC89kwyKNDA1PA,12562
 scanoss/scanossbase.py,sha256=WxYlWl6WxRArho4VKGFxEla8qYnjOXtF6EnwsHTrKm4,2319
-scanoss/scanossgrpc.py,sha256=rN4P-iDk9QKTukdqLjXGJ0PnEkSkiZmGB0fcWhZ_zkk,20426
+scanoss/scanossgrpc.py,sha256=lf5LJ8FFzF6OAu0zNUvCvLD6-7bIzQNR5pn3XkRiyRo,20483
 scanoss/scantype.py,sha256=R2-ExLGOrYxaJFtIK2AEo2caD0XrN1zpF5q1qT9Zsyc,1326
 scanoss/spdxlite.py,sha256=ZAJlkgW5U9WUT35D1ZEwIJ-eLkbVLBv3lU_XIArfoik,15441
-scanoss/threadeddependencies.py,sha256=jGSU7QTwmclcltbXuyqPS1fpmcYApnqL0d65wtejc9M,5264
+scanoss/threadeddependencies.py,sha256=JotQC9X3nnviblKe--OPS-7rr1W-cZjuxsxSPL-tbPg,6284
 scanoss/threadedscanning.py,sha256=T0tL8W1IEX_hLY5ksrAl_iQqtxT_KbyDhTDHo6a7xFE,9387
 scanoss/winnowing.py,sha256=CC4hB0iPHh8CmftnM2w8bmJlS8lAroi3kmJa6hHyfgk,15222
 scanoss/api/__init__.py,sha256=KlDD87JmyZP-10T-fuJo0_v2zt1gxWfTgs70wjky9xg,1139
@@ -47,12 +47,12 @@ scanoss/api/vulnerabilities/__init__.py,sha256=FLQtiDiv85Q1Chk-sJ9ky9WOV1mulZhEK
 scanoss/api/vulnerabilities/v2/__init__.py,sha256=FLQtiDiv85Q1Chk-sJ9ky9WOV1mulZhEKjiBihlwiaM,1139
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py,sha256=CFhF80av8tenGvn9AIsGEtRJPuV2dC_syA5JLZb2lDw,5464
 scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py,sha256=HlS4k4Zmx6RIAqaO9I96jD-eyF5yU6Xx04pVm7pdqOg,6864
-scanoss/data/build_date.txt,sha256=_uu7KfZJ5zvx0G35HZ-4Te-QZXVpJsvoTRQtXI9QvYA,40
+scanoss/data/build_date.txt,sha256=BzICu2lLBdWOULEC4j0yzP_SrZ4mkTzDp5dZUMyD5CI,40
 scanoss/data/spdx-exceptions.json,sha256=s7UTYxC7jqQXr11YBlIWYCNwN6lRDFTR33Y8rpN_dA4,17953
 scanoss/data/spdx-licenses.json,sha256=A6Z0q82gaTLtnopBfzeIVZjJFxkdRW1g2TuumQc-lII,228794
-scanoss-1.8.0.dist-info/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
-scanoss-1.8.0.dist-info/METADATA,sha256=56es3JKCrJNkX0A6u57We6ppuVk2cXQSfbQdJzc9DHI,5905
-scanoss-1.8.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-scanoss-1.8.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
-scanoss-1.8.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
-scanoss-1.8.0.dist-info/RECORD,,
+scanoss-1.10.0.dist-info/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
+scanoss-1.10.0.dist-info/METADATA,sha256=W1j9E0Pg4ISCu4EUYsQFEwnkvtvMC5lqDOai-zhIGi8,5906
+scanoss-1.10.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+scanoss-1.10.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
+scanoss-1.10.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
+scanoss-1.10.0.dist-info/RECORD,,

{scanoss-1.8.0.dist-info → scanoss-1.10.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any