scanoss 1.20.0__py3-none-any.whl → 1.20.1__py3-none-any.whl

scanoss/threadeddependencies.py

@@ -1,25 +1,25 @@
 """
- SPDX-License-Identifier: MIT
-
-   Copyright (c) 2021, SCANOSS
-
-   Permission is hereby granted, free of charge, to any person obtaining a copy
-   of this software and associated documentation files (the "Software"), to deal
-   in the Software without restriction, including without limitation the rights
-   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-   copies of the Software, and to permit persons to whom the Software is
-   furnished to do so, subject to the following conditions:
-
-   The above copyright notice and this permission notice shall be included in
-   all copies or substantial portions of the Software.
-
-   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-   THE SOFTWARE.
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2021, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
 """
 
 import threading
@@ -33,9 +33,21 @@ from .scancodedeps import ScancodeDeps
 from .scanossbase import ScanossBase
 from .scanossgrpc import ScanossGrpc
 
-DEP_FILE_PREFIX = "file="  # Default prefix to signify an existing parsed dependency file
+DEP_FILE_PREFIX = 'file='  # Default prefix to signify an existing parsed dependency file
 
-DEV_DEPENDENCIES = { "dev", "test", "development", "provided", "runtime", "devDependencies", "dev-dependencies", "testImplementation", "testCompile", "Test", "require-dev" }
+DEV_DEPENDENCIES = {
+    'dev',
+    'test',
+    'development',
+    'provided',
+    'runtime',
+    'devDependencies',
+    'dev-dependencies',
+    'testImplementation',
+    'testCompile',
+    'Test',
+    'require-dev',
+}
 
 
 # Define an enum class
@@ -46,17 +58,21 @@ class SCOPE(Enum):
 
 @dataclass
 class ThreadedDependencies(ScanossBase):
-    """
+    """ """
 
-    """
     inputs: queue.Queue = queue.Queue()
     output: queue.Queue = queue.Queue()
 
-    def __init__(self, sc_deps: ScancodeDeps, grpc_api: ScanossGrpc, what_to_scan: str = None, debug: bool = False,
-                 trace: bool = False, quiet: bool = False) -> None:
-        """
-
-        """
+    def __init__(
+        self,
+        sc_deps: ScancodeDeps,
+        grpc_api: ScanossGrpc,
+        what_to_scan: str = None,
+        debug: bool = False,
+        trace: bool = False,
+        quiet: bool = False,
+    ) -> None:
+        """ """
         super().__init__(debug, trace, quiet)
         self.sc_deps = sc_deps
         self.grpc_api = grpc_api
@@ -76,8 +92,15 @@ class ThreadedDependencies(ScanossBase):
                 return resp
         return None
 
-    def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True, dep_scope: SCOPE =  None,
-            dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool:
+    def run(
+        self,
+        what_to_scan: str = None,
+        deps_file: str = None,
+        wait: bool = True,
+        dep_scope: SCOPE = None,
+        dep_scope_include: str = None,
+        dep_scope_exclude: str = None,
+    ) -> bool:
         """
         Initiate a background scan for the specified file/dir
         :param dep_scope_exclude: comma separated list of dependency scopes to exclude
@@ -91,23 +114,25 @@ class ThreadedDependencies(ScanossBase):
         what_to_scan = what_to_scan if what_to_scan else self.what_to_scan
         self._errors = False
         try:
-            if deps_file:                                                  # Decorate the given dependencies file
+            if deps_file:  # Decorate the given dependencies file
                 self.print_msg(f'Decorating {deps_file} dependencies...')
-                self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}')           # Add to queue and have parent wait on it
-            else:                                                          # Search for dependencies to decorate
+                self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}')  # Add to queue and have parent wait on it
+            else:  # Search for dependencies to decorate
                 self.print_msg(f'Searching {what_to_scan} for dependencies...')
                 self.inputs.put(what_to_scan)
                 # Add to queue and have parent wait on it
-            self._thread = threading.Thread(target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True)
+            self._thread = threading.Thread(
+                target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True
+            )
             self._thread.start()
         except Exception as e:
             self.print_stderr(f'ERROR: Problem running threaded dependencies: {e}')
             self._errors = True
-        if wait and not self._errors:               # Wait for all inputs to complete
+        if wait and not self._errors:  # Wait for all inputs to complete
             self.complete()
         return False if self._errors else True
 
-    def filter_dependencies(self,deps ,filter_dep)-> json:
+    def filter_dependencies(self, deps, filter_dep) -> json:
         files = deps.get('files', [])
         # Iterate over files and their purls
         for file in files:
@@ -120,52 +145,56 @@ class ThreadedDependencies(ScanossBase):
                 ]
         # End of for loop
 
-        return {
-            'files': [
-                file for file in deps.get('files', [])
-                if file.get('purls')
-            ]
-        }
+        return {'files': [file for file in deps.get('files', []) if file.get('purls')]}
 
-    def filter_dependencies_by_scopes(self,deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None,
-                                      dep_scope_exclude: str = None) -> json:
+    def filter_dependencies_by_scopes(
+        self, deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None
+    ) -> json:
         # Predefined set of scopes to filter
 
         # Include all scopes
-        include_all = (dep_scope is None or dep_scope == "") and dep_scope_include is None and dep_scope_exclude is None
+        include_all = (dep_scope is None or dep_scope == '') and dep_scope_include is None and dep_scope_exclude is None
         ## All dependencies, remove scope key
         if include_all:
-            return self.filter_dependencies(deps, lambda purl:True)
+            return self.filter_dependencies(deps, lambda purl: True)
 
         # Use default list of scopes if a custom list is not set
-        if (dep_scope is not None and dep_scope != "") and dep_scope_include is None and dep_scope_exclude is None:
-            return self.filter_dependencies(deps, lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES) or
-            dep_scope == SCOPE.DEVELOPMENT and purl in DEV_DEPENDENCIES)
+        if (dep_scope is not None and dep_scope != '') and dep_scope_include is None and dep_scope_exclude is None:
+            return self.filter_dependencies(
+                deps,
+                lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES)
+                or dep_scope == SCOPE.DEVELOPMENT
+                and purl in DEV_DEPENDENCIES,
+            )
 
-        if ((dep_scope_include is not None and dep_scope_include != "")
-                or  dep_scope_exclude is not None and dep_scope_exclude != ""):
+        if (
+            (dep_scope_include is not None and dep_scope_include != '')
+            or dep_scope_exclude is not None
+            and dep_scope_exclude != ''
+        ):
             # Create sets from comma-separated strings, if provided
             exclude = set(dep_scope_exclude.split(',')) if dep_scope_exclude else set()
             include = set(dep_scope_include.split(',')) if dep_scope_include else set()
 
             # Define a lambda function that checks the inclusion/exclusion logic
             return self.filter_dependencies(
-                deps,
-                lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include)
+                deps, lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include)
             )
 
-    def scan_dependencies(self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None) -> None:
+    def scan_dependencies(
+        self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None
+    ) -> None:
         """
         Scan for dependencies from the given file/dir or from an input file (from the input queue).
         """
         current_thread = threading.get_ident()
         self.print_trace(f'Starting dependency worker {current_thread}...')
         try:
-            what_to_scan = self.inputs.get(timeout=5)            # Begin processing the dependency request
+            what_to_scan = self.inputs.get(timeout=5)  # Begin processing the dependency request
             deps = None
-            if what_to_scan.startswith(DEP_FILE_PREFIX):         # We have a pre-parsed dependency file, load it
+            if what_to_scan.startswith(DEP_FILE_PREFIX):  # We have a pre-parsed dependency file, load it
                 deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX))
-            else:                                                # Search the file/folder for dependency files to parse
+            else:  # Search the file/folder for dependency files to parse
                 if not self.sc_deps.run_scan(what_to_scan=what_to_scan):
                     self._errors = True
                 else:
@@ -176,13 +205,13 @@ class ThreadedDependencies(ScanossBase):
                         self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes")
                     if dep_scope_exclude is not None:
                         self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes")
-                    deps = self.filter_dependencies_by_scopes(deps, dep_scope,dep_scope_include, dep_scope_exclude)
+                    deps = self.filter_dependencies_by_scopes(deps, dep_scope, dep_scope_include, dep_scope_exclude)
 
             if not self._errors:
                 if deps is None:
                     self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}')
                     self._errors = True
-                elif not deps or len(deps.get("files", [])) == 0:
+                elif not deps or len(deps.get('files', [])) == 0:
                     self.print_debug(f'No dependencies found to decorate for: {what_to_scan}')
                 else:
                     decorated_deps = self.grpc_api.get_dependencies(deps)
@@ -210,6 +239,7 @@ class ThreadedDependencies(ScanossBase):
             self._errors = True
         return True if not self._errors else False
 
+
 #
 # End of ThreadedDependencies Class
 #

scanoss/threadedscanning.py

@@ -1,26 +1,27 @@
 """
- SPDX-License-Identifier: MIT
-
-   Copyright (c) 2021, SCANOSS
-
-   Permission is hereby granted, free of charge, to any person obtaining a copy
-   of this software and associated documentation files (the "Software"), to deal
-   in the Software without restriction, including without limitation the rights
-   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-   copies of the Software, and to permit persons to whom the Software is
-   furnished to do so, subject to the following conditions:
-
-   The above copyright notice and this permission notice shall be included in
-   all copies or substantial portions of the Software.
-
-   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-   THE SOFTWARE.
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2021, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
 """
+
 import os
 import sys
 import threading
@@ -34,8 +35,10 @@ from progress.bar import Bar
 from .scanossapi import ScanossApi
 from .scanossbase import ScanossBase
 
-WFP_FILE_START = "file="
-MAX_ALLOWED_THREADS = int(os.environ.get("SCANOSS_MAX_ALLOWED_THREADS")) if os.environ.get("SCANOSS_MAX_ALLOWED_THREADS") else 30
+WFP_FILE_START = 'file='
+MAX_ALLOWED_THREADS = (
+    int(os.environ.get('SCANOSS_MAX_ALLOWED_THREADS')) if os.environ.get('SCANOSS_MAX_ALLOWED_THREADS') else 30
+)
 
 
 @dataclass
@@ -45,13 +48,14 @@ class ThreadedScanning(ScanossBase):
     WFP scan requests are loaded into the input queue.
     Multiple threads pull messages off this queue, process the request and put the results into an output queue
     """
+
     inputs: queue.Queue = queue.Queue()
     output: queue.Queue = queue.Queue()
     bar: Bar = None
 
-    def __init__(self, scanapi: ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False,
-                 nb_threads: int = 5
-                 ) -> None:
+    def __init__(
+        self, scanapi: ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False, nb_threads: int = 5
+    ) -> None:
         """
         Initialise the ThreadedScanning class
         :param scanapi: SCANOSS API to send scan requests to
@@ -158,8 +162,9 @@ class ThreadedScanning(ScanossBase):
         """
         qsize = self.inputs.qsize()
         if qsize < self.nb_threads:
-            self.print_debug(f'Input queue ({qsize}) smaller than requested threads: {self.nb_threads}. '
-                             f'Reducing to queue size.')
+            self.print_debug(
+                f'Input queue ({qsize}) smaller than requested threads: {self.nb_threads}. Reducing to queue size.'
+            )
             self.nb_threads = qsize
         else:
             self.print_debug(f'Starting {self.nb_threads} threads to process {qsize} requests...')
@@ -171,7 +176,7 @@ class ThreadedScanning(ScanossBase):
         except Exception as e:
             self.print_stderr(f'ERROR: Problem running threaded scanning: {e}')
             self._errors = True
-        if wait:                    # Wait for all inputs to complete
+        if wait:  # Wait for all inputs to complete
             self.complete()
         return False if self._errors else True
 
@@ -180,7 +185,7 @@ class ThreadedScanning(ScanossBase):
         Wait for input queue to complete processing and complete the worker threads
         """
         self.inputs.join()
-        self._stop_event.set()       # Tell the worker threads to stop
+        self._stop_event.set()  # Tell the worker threads to stop
         try:
             for t in self._threads:  # Complete the threads
                 t.join(timeout=5)
@@ -199,7 +204,7 @@ class ThreadedScanning(ScanossBase):
         api_error = False
         while not self._stop_event.is_set():
             wfp = None
-            if not self.inputs.empty():          # Only try to get a message if there is one on the queue
+            if not self.inputs.empty():  # Only try to get a message if there is one on the queue
                 try:
                     wfp = self.inputs.get(timeout=5)
                     if api_error:  # API error encountered, so stop processing anymore requests
@@ -228,6 +233,7 @@ class ThreadedScanning(ScanossBase):
                 time.sleep(1)  # Sleep while waiting for the queue depth to build up
         self.print_trace(f'Thread complete ({current_thread}).')
 
+
 #
 # End of ThreadedScanning Class
 #

scanoss/utils/file.py

@@ -52,24 +52,24 @@ def validate_json_file(json_file_path: str) -> JsonValidation:
         Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message
     """
     if not json_file_path:
-        return JsonValidation(is_valid=False, error="No JSON file specified")
+        return JsonValidation(is_valid=False, error='No JSON file specified')
     if not os.path.isfile(json_file_path):
         return JsonValidation(
             is_valid=False,
-            error=f"File not found: {json_file_path}",
+            error=f'File not found: {json_file_path}',
             error_code=JSON_ERROR_FILE_NOT_FOUND,
         )
     try:
         if os.stat(json_file_path).st_size == 0:
             return JsonValidation(
                 is_valid=False,
-                error=f"File is empty: {json_file_path}",
+                error=f'File is empty: {json_file_path}',
                 error_code=JSON_ERROR_FILE_EMPTY,
             )
     except OSError as e:
         return JsonValidation(
             is_valid=False,
-            error=f"Problem checking file size: {json_file_path}: {e}",
+            error=f'Problem checking file size: {json_file_path}: {e}',
             error_code=JSON_ERROR_FILE_SIZE,
         )
     try:

scanoss/winnowing.py

@@ -1,32 +1,33 @@
 """
- SPDX-License-Identifier: MIT
+SPDX-License-Identifier: MIT
 
-   Copyright (c) 2021, SCANOSS
+  Copyright (c) 2021, SCANOSS
 
-   Permission is hereby granted, free of charge, to any person obtaining a copy
-   of this software and associated documentation files (the "Software"), to deal
-   in the Software without restriction, including without limitation the rights
-   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-   copies of the Software, and to permit persons to whom the Software is
-   furnished to do so, subject to the following conditions:
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
 
-   The above copyright notice and this permission notice shall be included in
-   all copies or substantial portions of the Software.
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
 
-   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-   THE SOFTWARE.
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
 
-   Winnowing Algorithm implementation for SCANOSS.
+  Winnowing Algorithm implementation for SCANOSS.
 
-   This module implements an adaptation of the original winnowing algorithm by S. Schleimer, D. S. Wilkerson and
-   A. Aiken as described in their seminal article which can be found here:
-   https://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf
+  This module implements an adaptation of the original winnowing algorithm by S. Schleimer, D. S. Wilkerson and
+  A. Aiken as described in their seminal article which can be found here:
+  https://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf
 """
+
 import hashlib
 import pathlib
 import platform
@@ -55,11 +56,56 @@ MAX_POST_SIZE = 64 * 1024  # 64k Max post size
 MIN_FILE_SIZE = 256
 
 SKIP_SNIPPET_EXT = {  # File extensions to ignore snippets for
-    ".exe", ".zip", ".tar", ".tgz", ".gz", ".7z", ".rar", ".jar", ".war", ".ear", ".class", ".pyc",
-    ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin",
-    ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt",
-    ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers",
-    ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2", '.xsd', ".pom", ".whl",
+    '.exe',
+    '.zip',
+    '.tar',
+    '.tgz',
+    '.gz',
+    '.7z',
+    '.rar',
+    '.jar',
+    '.war',
+    '.ear',
+    '.class',
+    '.pyc',
+    '.o',
+    '.a',
+    '.so',
+    '.obj',
+    '.dll',
+    '.lib',
+    '.out',
+    '.app',
+    '.bin',
+    '.lst',
+    '.dat',
+    '.json',
+    '.htm',
+    '.html',
+    '.xml',
+    '.md',
+    '.txt',
+    '.doc',
+    '.docx',
+    '.xls',
+    '.xlsx',
+    '.ppt',
+    '.pptx',
+    '.odt',
+    '.ods',
+    '.odp',
+    '.pages',
+    '.key',
+    '.numbers',
+    '.pdf',
+    '.min.js',
+    '.mf',
+    '.sum',
+    '.woff',
+    '.woff2',
+    '.xsd',
+    '.pom',
+    '.whl',
 }
 
 CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
@@ -111,11 +157,21 @@ class Winnowing(ScanossBase):
     a list of WFP fingerprints with their corresponding line numbers.
     """
 
-    def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = False, quiet: bool = False,
-                 skip_snippets: bool = False, post_size: int = 32, all_extensions: bool = False,
-                 obfuscate: bool = False, hpsm: bool = False,
-                 strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None
-                 ):
+    def __init__(
+        self,
+        size_limit: bool = False,
+        debug: bool = False,
+        trace: bool = False,
+        quiet: bool = False,
+        skip_snippets: bool = False,
+        post_size: int = 32,
+        all_extensions: bool = False,
+        obfuscate: bool = False,
+        hpsm: bool = False,
+        strip_hpsm_ids=None,
+        strip_snippet_ids=None,
+        skip_md5_ids=None,
+    ):
         """
         Instantiate Winnowing class
         Parameters
@@ -190,12 +246,16 @@ class Winnowing(ScanossBase):
         if src_len == 0 or src_len <= MIN_FILE_SIZE:  # Ignore empty or files that are too small
             self.print_trace(f'Skipping snippets as the file is too small: {file} - {src_len}')
             return True
-        prefix = src[0:(MIN_FILE_SIZE - 1)].lower().strip()
-        if len(prefix) > 0 and (prefix[0] == "{" or prefix[0] == "["):  # Ignore json
+        prefix = src[0 : (MIN_FILE_SIZE - 1)].lower().strip()
+        if len(prefix) > 0 and (prefix[0] == '{' or prefix[0] == '['):  # Ignore json
             self.print_trace(f'Skipping snippets as the file appears to be JSON: {file}')
             return True
-        if prefix.startswith("<?xml") or prefix.startswith("<html") or prefix.startswith("<ac3d") or prefix.startswith(
-                "<!doc"):
+        if (
+            prefix.startswith('<?xml')
+            or prefix.startswith('<html')
+            or prefix.startswith('<ac3d')
+            or prefix.startswith('<!doc')
+        ):
             self.print_trace(f'Skipping snippets as the file appears to be xml/html/binary: {file}')
             return True  # Ignore xml & html & ac3d
         index = src.index('\n') if '\n' in src else (src_len - 1)  # TODO still necessary if we have a binary check?
@@ -258,11 +318,12 @@ class Winnowing(ScanossBase):
                 elif hpsm_id_len % 2 == 1:
                     hpsm_id_len = hpsm_id_len + 1
 
-                to_remove = hpsm[hpsm_id_index:hpsm_id_index + hpsm_id_len]
+                to_remove = hpsm[hpsm_id_index : hpsm_id_index + hpsm_id_len]
                 self.print_debug(f'HPSM ID {to_remove} to replace')
                 # Calculate the XOR of each byte to produce the correct ignore sequence.
                 replacement = ''.join(
-                    [format(int(to_remove[i:i + 2], 16) ^ 0xFF, '02x') for i in range(0, len(to_remove), 2)])
+                    [format(int(to_remove[i : i + 2], 16) ^ 0xFF, '02x') for i in range(0, len(to_remove), 2)]
+                )
 
                 self.print_debug(f'HPSM ID replacement {replacement}')
                 # Overwrite HPSM bytes to be removed.
@@ -309,7 +370,7 @@ class Winnowing(ScanossBase):
         # Print file line
         content_length = len(contents)
         original_filename = file
-        
+
         if platform.system() == 'Windows':
             original_filename = file.replace('\\', '/')
         wfp_filename = repr(original_filename).strip("'")  # return a utf-8 compatible version of the filename
@@ -361,14 +422,16 @@ class Winnowing(ScanossBase):
                             crc_hex = '{:08x}'.format(crc)
                             if last_line != line:
                                 if output != '':
-                                    if self.size_limit and \
-                                            (len(wfp.encode("utf-8")) + len(
-                                                output.encode("utf-8"))) > self.max_post_size:
+                                    if (
+                                        self.size_limit
+                                        and (len(wfp.encode('utf-8')) + len(output.encode('utf-8')))
+                                        > self.max_post_size
+                                    ):
                                         self.print_debug(f'Truncating WFP ({self.max_post_size} limit) for: {file}')
                                         output = ''
                                         break  # Stop collecting snippets as it's over 64k
                                     wfp += output + '\n'
-                                output = "%d=%s" % (line, crc_hex)
+                                output = '%d=%s' % (line, crc_hex)
                             else:
                                 output += ',' + crc_hex
 
@@ -379,7 +442,7 @@ class Winnowing(ScanossBase):
                     # Shift gram
                     gram = gram[1:]
         if output != '':
-            if not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size:
+            if not self.size_limit or (len(wfp.encode('utf-8')) + len(output.encode('utf-8'))) < self.max_post_size:
                 wfp += output + '\n'
             else:
                 self.print_debug(f'Warning: skipping output in WFP for {file} - "{output}"')
@@ -403,13 +466,13 @@ class Winnowing(ScanossBase):
         last_line = 0
         for i, byte in enumerate(content):
             c = byte
-            if c == ASCII_LF:   # When there is a new line
-                if len(list_normalized): 
+            if c == ASCII_LF:  # When there is a new line
+                if len(list_normalized):
                     crc_lines.append(self.crc8_buffer(list_normalized))
                     list_normalized = []
-                elif last_line+1 == i:
+                elif last_line + 1 == i:
                     crc_lines.append(0xFF)
-                elif i-last_line > 1:
+                elif i - last_line > 1:
                     crc_lines.append(0x00)
                 last_line = i
             else:
@@ -470,6 +533,7 @@ class Winnowing(ScanossBase):
         crc ^= CRC8_MAXIM_DOW_FINAL  # Bitwise OR (XOR) of crc in Maxim Dow Final
         return crc
 
+
 #
 # End of Winnowing Class
 #

{scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: scanoss
-Version: 1.20.0
+Version: 1.20.1
 Summary: Simple Python library to leverage the SCANOSS APIs
 Home-page: https://scanoss.com
 Author: SCANOSS