scanoss 1.20.6__py3-none-any.whl → 1.23.0__py3-none-any.whl

scanoss/scanossgrpc.py

@@ -26,6 +26,9 @@ import concurrent.futures
 import json
 import os
 import uuid
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Dict, Optional
 from urllib.parse import urlparse
 
 import grpc
@@ -33,6 +36,9 @@ from google.protobuf.json_format import MessageToDict, ParseDict
 from pypac.parser import PACFile
 from pypac.resolver import ProxyResolver
 
+from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub
+from scanoss.constants import DEFAULT_TIMEOUT
+
 from . import __version__
 from .api.common.v2.scanoss_common_pb2 import (
     EchoRequest,
@@ -52,8 +58,9 @@ from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse
 from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub
 from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest
 from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub
-from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse
-from .api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub
+from .api.geoprovenance.v2.scanoss_geoprovenance_pb2 import ContributorResponse
+from .api.geoprovenance.v2.scanoss_geoprovenance_pb2_grpc import GeoProvenanceStub
+from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest
 from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse
 from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub
 from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse
@@ -68,6 +75,23 @@ SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_A
 MAX_CONCURRENT_REQUESTS = 5
 
 
+class ScanossGrpcError(Exception):
+    """
+    Custom exception for SCANOSS gRPC errors
+    """
+
+    pass
+
+
+class ScanossGrpcStatusCode(IntEnum):
+    """Status codes for SCANOSS gRPC responses"""
+
+    SUCCESS = 1
+    SUCCESS_WITH_WARNINGS = 2
+    FAILED_WITH_WARNINGS = 3
+    FAILED = 4
+
+
 class ScanossGrpc(ScanossBase):
     """
     Client for gRPC functionality
@@ -113,7 +137,6 @@ class ScanossGrpc(ScanossBase):
         self.req_headers = req_headers
         self.metadata = []
 
-
         if self.api_key:
             self.metadata.append(('x-api-key', api_key))  # Set API key if we have one
         if ver_details:
@@ -146,7 +169,8 @@ class ScanossGrpc(ScanossBase):
             self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url))
             self.semgrep_stub = SemgrepStub(grpc.insecure_channel(self.url))
             self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url))
-            self.provenance_stub = ProvenanceStub(grpc.insecure_channel(self.url))
+            self.provenance_stub = GeoProvenanceStub(grpc.insecure_channel(self.url))
+            self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url))
         else:
             if ca_cert is not None:
                 credentials = grpc.ssl_channel_credentials(cert_data)  # secure with specified certificate
@@ -157,7 +181,8 @@ class ScanossGrpc(ScanossBase):
             self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials))
             self.semgrep_stub = SemgrepStub(grpc.secure_channel(self.url, credentials))
             self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials))
-            self.provenance_stub = ProvenanceStub(grpc.secure_channel(self.url, credentials))
+            self.provenance_stub = GeoProvenanceStub(grpc.secure_channel(self.url, credentials))
+            self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials))
 
     @classmethod
     def _load_cert(cls, cert_file: str) -> bytes:
@@ -437,6 +462,59 @@ class ScanossGrpc(ScanossBase):
                 return resp_dict
         return None
 
+    def folder_hash_scan(self, request: Dict) -> Optional[Dict]:
+        """
+        Client function to call the rpc for Folder Hashing Scan
+
+        Args:
+            request (Dict): Folder Hash Request
+
+        Returns:
+            Optional[Dict]: Folder Hash Response, or None if the request was not succesfull
+        """
+        return self._call_rpc(
+            self.scanning_stub.FolderHashScan,
+            request,
+            HFHRequest,
+            'Sending folder hash scan data (rqId: {rqId})...',
+        )
+
+    def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> Optional[Dict]:
+        """
+        Call a gRPC method and return the response as a dictionary
+
+        Args:
+            rpc_method (): The gRPC stub method
+            request_input (): Either a dict or a gRPC request object.
+            request_type (): The type of the gRPC request object.
+            debug_msg (str, optional): Debug message template that can include {rqId} placeholder.
+
+        Returns:
+            dict: The parsed gRPC response as a dictionary, or None if something went wrong
+        """
+        request_id = str(uuid.uuid4())
+
+        if isinstance(request_input, dict):
+            request_obj = ParseDict(request_input, request_type())
+        else:
+            request_obj = request_input
+
+        metadata = self.metadata[:] + [('x-request-id', request_id)]
+
+        self.print_debug(debug_msg.format(rqId=request_id))
+        try:
+            resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout)
+        except grpc.RpcError as e:
+            raise ScanossGrpcError(
+                f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}'
+            )
+
+        if resp and not self._check_status_response(resp.status, request_id):
+            return None
+
+        resp_dict = MessageToDict(resp, preserving_proto_field_name=True)
+        return resp_dict
+
     def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool:
         """
         Check the response object to see if the command was successful or not
@@ -444,21 +522,18 @@ class ScanossGrpc(ScanossBase):
         :return: True if successful, False otherwise
         """
 
-        SUCCEDED_WITH_WARNINGS_STATUS_CODE = 2
-        FAILED_STATUS_CODE = 3
-
         if not status_response:
             self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.')
             return True
         self.print_debug(f'Checking response status (rqId: {request_id}): {status_response}')
         status_code: StatusCode = status_response.status
-        if status_code > 1:
+        if status_code > ScanossGrpcStatusCode.SUCCESS:
             ret_val = False  # default to failed
             msg = 'Unsuccessful'
-            if status_code == SUCCEDED_WITH_WARNINGS_STATUS_CODE:
+            if status_code == ScanossGrpcStatusCode.SUCCESS_WITH_WARNINGS:
                 msg = 'Succeeded with warnings'
                 ret_val = True  # No need to fail as it succeeded with warnings
-            elif status_code == FAILED_STATUS_CODE:
+            elif status_code == ScanossGrpcStatusCode.FAILED_WITH_WARNINGS:
                 msg = 'Failed with warnings'
             self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}')
             return ret_val
@@ -496,13 +571,13 @@ class ScanossGrpc(ScanossBase):
             self.print_stderr('ERROR: No message supplied to send to gRPC service.')
             return None
         request_id = str(uuid.uuid4())
-        resp: ProvenanceResponse
+        resp: ContributorResponse
         try:
             request = ParseDict(purls, PurlRequest())  # Parse the JSON/Dict into the purl request object
             metadata = self.metadata[:]
             metadata.append(('x-request-id', request_id))  # Set a Request ID
             self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...')
-            resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout)
+            resp = self.provenance_stub.GetComponentContributors(request, metadata=metadata, timeout=self.timeout)
         except Exception as e:
             self.print_stderr(
                 f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
@@ -515,20 +590,70 @@ class ScanossGrpc(ScanossBase):
                 return resp_dict
         return None
 
+    def get_provenance_origin(self, request: Dict) -> Optional[Dict]:
+        """
+        Client function to call the rpc for GetComponentOrigin
+
+        Args:
+            request (Dict): GetComponentOrigin Request
+
+        Returns:
+            Optional[Dict]: OriginResponse, or None if the request was not successfull
+        """
+        return self._call_rpc(
+            self.provenance_stub.GetComponentOrigin,
+            request,
+            PurlRequest,
+            'Sending data for provenance origin decoration (rqId: {rqId})...',
+        )
+
     def load_generic_headers(self):
         """
-           Adds custom headers from req_headers to metadata.
+        Adds custom headers from req_headers to metadata.
 
-           If x-api-key is present and no URL is configured (directly or via
-           environment), sets URL to the premium endpoint (DEFAULT_URL2).
-           """
+        If x-api-key is present and no URL is configured (directly or via
+        environment), sets URL to the premium endpoint (DEFAULT_URL2).
+        """
         if self.req_headers:  # Load generic headers
             for key, value in self.req_headers.items():
-                if key == 'x-api-key': # Set premium URL if x-api-key header is set
+                if key == 'x-api-key':  # Set premium URL if x-api-key header is set
                     if not self.url and not os.environ.get('SCANOSS_GRPC_URL'):
                         self.url = DEFAULT_URL2  # API key specific and no alternative URL, so use the default premium
                     self.api_key = value
                 self.metadata.append((key, value))
+
+
 #
 # End of ScanossGrpc Class
 #
+
+
+@dataclass
+class GrpcConfig:
+    url: str = DEFAULT_URL
+    api_key: Optional[str] = SCANOSS_API_KEY
+    debug: Optional[bool] = False
+    trace: Optional[bool] = False
+    quiet: Optional[bool] = False
+    ver_details: Optional[str] = None
+    ca_cert: Optional[str] = None
+    pac: Optional[PACFile] = None
+    timeout: Optional[int] = DEFAULT_TIMEOUT
+    proxy: Optional[str] = None
+    grpc_proxy: Optional[str] = None
+
+
+def create_grpc_config_from_args(args) -> GrpcConfig:
+    return GrpcConfig(
+        url=getattr(args, 'api2url', DEFAULT_URL),
+        api_key=getattr(args, 'key', SCANOSS_API_KEY),
+        debug=getattr(args, 'debug', False),
+        trace=getattr(args, 'trace', False),
+        quiet=getattr(args, 'quiet', False),
+        ver_details=getattr(args, 'ver_details', None),
+        ca_cert=getattr(args, 'ca_cert', None),
+        pac=getattr(args, 'pac', None),
+        timeout=getattr(args, 'timeout', DEFAULT_TIMEOUT),
+        proxy=getattr(args, 'proxy', None),
+        grpc_proxy=getattr(args, 'grpc_proxy', None),
+    )

scanoss/threadedscanning.py

@@ -23,13 +23,13 @@ SPDX-License-Identifier: MIT
 """
 
 import os
+import queue
 import sys
 import threading
-import queue
 import time
-
-from typing import Dict, List
 from dataclasses import dataclass
+from typing import Dict, List
+
 from progress.bar import Bar
 
 from .scanossapi import ScanossApi
@@ -49,8 +49,6 @@ class ThreadedScanning(ScanossBase):
     Multiple threads pull messages off this queue, process the request and put the results into an output queue
     """
 
-    inputs: queue.Queue = queue.Queue()
-    output: queue.Queue = queue.Queue()
     bar: Bar = None
 
     def __init__(
@@ -65,6 +63,8 @@ class ThreadedScanning(ScanossBase):
         :param nb_threads: Number of thread to run (default 5)
         """
         super().__init__(debug, trace, quiet)
+        self.inputs = queue.Queue()
+        self.output = queue.Queue()
         self.scanapi = scanapi
         self.nb_threads = nb_threads
         self._isatty = sys.stderr.isatty()
@@ -134,7 +134,7 @@ class ThreadedScanning(ScanossBase):
         :param wfp: WFP to add to queue
         """
         if wfp is None or wfp == '':
-            self.print_stderr(f'Warning: empty WFP. Skipping from scan...')
+            self.print_stderr('Warning: empty WFP. Skipping from scan...')
         else:
             self.inputs.put(wfp)
 

scanoss/utils/abstract_presenter.py

@@ -0,0 +1,103 @@
+from abc import ABC, abstractmethod
+
+from scanoss.scanossbase import ScanossBase
+
+
+class AbstractPresenter(ABC):
+    """
+    Abstract presenter class for presenting output in a given format.
+    Subclasses must implement the _format_json_output and _format_plain_output methods.
+    """
+
+    def __init__(
+        self,
+        debug: bool = False,
+        trace: bool = False,
+        quiet: bool = False,
+        output_file: str = None,
+        output_format: str = None,
+    ):
+        """
+        Initialize the presenter with the given output file and format.
+        """
+        self.AVAILABLE_OUTPUT_FORMATS = ['json', 'plain', 'cyclonedx', 'spdxlite', 'csv', 'raw']
+        self.base = ScanossBase(debug=debug, trace=trace, quiet=quiet)
+        self.output_file = output_file
+        self.output_format = output_format
+
+    def present(self, output_format: str = None, output_file: str = None):
+        """
+        Present the formatted output to a file if provided; otherwise, print to stdout.
+        """
+        file_path = output_file or self.output_file
+        fmt = output_format or self.output_format
+
+        if fmt and fmt not in self.AVAILABLE_OUTPUT_FORMATS:
+            raise ValueError(
+                f"ERROR: Invalid output format '{fmt}'. Valid values are: {', '.join(self.AVAILABLE_OUTPUT_FORMATS)}"
+            )
+
+        if fmt == 'json':
+            content = self._format_json_output()
+        elif fmt == 'plain':
+            content = self._format_plain_output()
+        elif fmt == 'cyclonedx':
+            content = self._format_cyclonedx_output()
+        elif fmt == 'spdxlite':
+            content = self._format_spdxlite_output()
+        elif fmt == 'csv':
+            content = self._format_csv_output()
+        elif fmt == 'raw':
+            content = self._format_raw_output()
+        else:
+            content = self._format_plain_output()
+
+        self._present_output(content, file_path)
+
+    def _present_output(self, content: str, file_path: str = None):
+        """
+        If a file path is provided, write to that file; otherwise, print the content to stdout.
+        """
+        self.base.print_to_file_or_stdout(content, file_path)
+
+    @abstractmethod
+    def _format_cyclonedx_output(self) -> str:
+        """
+        Return a CycloneDX string representation of the data.
+        """
+        pass
+
+    @abstractmethod
+    def _format_spdxlite_output(self) -> str:
+        """
+        Return a SPDX-Lite string representation of the data.
+        """
+        pass
+
+    @abstractmethod
+    def _format_csv_output(self) -> str:
+        """
+        Return a CSV string representation of the data.
+        """
+        pass
+
+    @abstractmethod
+    def _format_json_output(self) -> str:
+        """
+        Return a JSON string representation of the data.
+        """
+        pass
+
+    @abstractmethod
+    def _format_plain_output(self) -> str:
+        """
+        Return a plain text string representation of the data.
+        """
+        pass
+
+    @abstractmethod
+    def _format_raw_output(self) -> str:
+        """
+        Return a raw string representation of the data.
+        """
+        pass

scanoss/utils/crc64.py

@@ -0,0 +1,96 @@
+"""
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2025, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+"""
+
+import struct
+from typing import List
+
+
+class CRC64:
+    """
+    CRC64 ECMA implementation matching Go's hash/crc64 package.
+    Uses polynomial: 0xC96C5795D7870F42
+    """
+
+    POLY = 0xC96C5795D7870F42
+    _TABLE = None
+
+    def __init__(self):
+        if CRC64._TABLE is None:
+            CRC64._TABLE = self._make_table()
+        self.crc = 0xFFFFFFFFFFFFFFFF  # Initial value
+
+    def _make_table(self) -> list:
+        """Generate the CRC64 lookup table."""
+        table = []
+        for i in range(256):
+            crc = i
+            for _ in range(8):
+                if crc & 1:
+                    crc = (crc >> 1) ^ self.POLY
+                else:
+                    crc >>= 1
+            table.append(crc)
+        return table
+
+    def update(self, data: bytes) -> None:
+        """Update the CRC with new data."""
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+
+        crc = self.crc
+        for b in data:
+            crc = (crc >> 8) ^ CRC64._TABLE[(crc ^ b) & 0xFF]  # Use class-level table
+        self.crc = crc
+
+    def digest(self) -> int:
+        """Get the current CRC value."""
+        return self.crc ^ 0xFFFFFFFFFFFFFFFF  # Final XOR value
+
+    def hexdigest(self):
+        """Get the current CRC value as a hexadecimal string."""
+        return format(self.digest(), '016x')
+
+    @classmethod
+    def checksum(cls, data: bytes) -> int:
+        """Calculate CRC64 checksum for the given data."""
+        crc = cls()
+        crc.update(data)
+        return crc.digest()
+
+    @classmethod
+    def get_hash_buff(cls, buff: bytes) -> List[bytes]:
+        """
+        Get the hash value of the given buffer, and converts it to 8 bytes in big-endian order.
+
+        Args:
+            buff (bytes): The buffer to get the hash value of.
+
+        Returns:
+            bytes: The hash value of the given buffer, and converts it to 8 bytes in big-endian order.
+        """
+        crc = cls()
+        crc.update(buff)
+        hash_val = crc.digest()
+
+        return list(struct.pack('>Q', hash_val))

scanoss/utils/simhash.py

@@ -0,0 +1,198 @@
+"""
+SPDX-License-Identifier: MIT
+
+  Copyright (c) 2025, SCANOSS
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+"""
+
+import re
+import unicodedata
+
+FNV64_OFFSET_BASIS = 14695981039346656037
+FNV64_PRIME = 1099511628211
+MASK64 = 0xFFFFFFFFFFFFFFFF
+
+
+def fnv1_64(data: bytes) -> int:
+    """Compute the 64‐bit FNV‑1 hash of data."""
+    h = FNV64_OFFSET_BASIS
+    for b in data:
+        h = (h * FNV64_PRIME) & MASK64
+        h = h ^ b
+    return h
+
+
+class SimhashFeature:
+    def __init__(self, hash_value: int, weight: int = 1):
+        self.hash_value = hash_value
+        self.weight = weight
+
+    def sum(self) -> int:
+        """Return the 64-bit hash (sum) of this feature."""
+        return self.hash_value
+
+    def get_weight(self) -> int:
+        """Return the weight of this feature."""
+        return self.weight
+
+
+def new_feature(f: bytes) -> SimhashFeature:
+    """Return a new feature for the given byte slice with weight 1."""
+    return SimhashFeature(fnv1_64(f), 1)
+
+
+def new_feature_with_weight(f: bytes, weight: int) -> SimhashFeature:
+    """Return a new feature for the given byte slice with the given weight."""
+    return SimhashFeature(fnv1_64(f), weight)
+
+
+def vectorize(features: list) -> list:
+    """
+    Given a list of features, return a 64-element vector.
+    Each feature contributes its weight to each coordinate,
+    added if that bit is set and subtracted otherwise.
+    """
+    v = [0] * 64
+    for feature in features:
+        h = feature.sum()
+        w = feature.get_weight()
+        for i in range(64):
+            if ((h >> i) & 1) == 1:
+                v[i] += w
+            else:
+                v[i] -= w
+    return v
+
+
+def vectorize_bytes(features: list) -> list:
+    """
+    Given a list of byte slices, treat each as a feature (with weight 1)
+    by computing its FNV-1 hash.
+    """
+    v = [0] * 64
+    for feat in features:
+        h = fnv1_64(feat)
+        for i in range(64):
+            if ((h >> i) & 1) == 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+    return v
+
+
+def fingerprint(v: list) -> int:
+    """
+    Given a 64-element vector, return a 64-bit fingerprint.
+    For each bit i, if v[i] >= 0, set bit i to 1; otherwise leave it 0.
+    """
+    f = 0
+    for i in range(64):
+        if v[i] >= 0:
+            f |= 1 << i
+    return f
+
+
+def compare(a: int, b: int) -> int:
+    """
+    Calculate the Hamming distance between two 64-bit integers.
+    (The number of differing bits.)
+    """
+    v = a ^ b
+    c = 0
+    while v:
+        v &= v - 1
+        c += 1
+    return c
+
+
+def simhash(fs) -> int:
+    """
+    Given a feature set (an object with a get_features() method),
+    return its 64-bit simhash.
+    """
+    return fingerprint(vectorize(fs.get_features()))
+
+
+def simhash_bytes(b: list) -> int:
+    """
+    Given a list of byte slices, return the simhash.
+    """
+    return fingerprint(vectorize_bytes(b))
+
+
+boundaries = re.compile(rb"[\w']+(?:\://[\w\./]+){0,1}")
+unicode_boundaries = re.compile(r"[\w'-]+", re.UNICODE)
+
+
+# --- Helper Functions for Feature Extraction ---
+def _get_features_bytes(b: bytes, pattern: re.Pattern) -> list:
+    """
+    Split the given byte string using the given regex pattern,
+    and return a list of features (each created with new_feature).
+    """
+    words = pattern.findall(b)
+    return [new_feature(word) for word in words]
+
+
+def _get_features_str(s: str, pattern) -> list:
+    """
+    Split the given string using the given regex pattern,
+    and return a list of features (each created by encoding to UTF-8).
+    """
+    words = pattern.findall(s)
+    return [new_feature(word.encode('utf-8')) for word in words]
+
+
+class WordFeatureSet:
+    def __init__(self, b: bytes):
+        # Normalize the input to lowercase.
+        self.b = b.lower()
+
+    def get_features(self) -> list:
+        return _get_features_bytes(self.b, boundaries)
+
+
+class UnicodeWordFeatureSet:
+    def __init__(self, b: bytes, norm_form: str = 'NFC'):
+        # Decode, normalize (using the provided form), and lowercase.
+        text = b.decode('utf-8')
+        normalized = unicodedata.normalize(norm_form, text)
+        self.text = normalized.lower()
+
+    def get_features(self) -> list:
+        return _get_features_str(self.text, unicode_boundaries)
+
+
+def shingle(w: int, b: list) -> list:
+    """
+    Return the w-shingling of the given set of byte slices.
+    For example, if b is [b"this", b"is", b"a", b"test"]
+    and w == 2, the result is [b"this is", b"is a", b"a test"].
+    """
+    if w < 1:
+        raise ValueError('simhash.shingle(): k must be a positive integer')
+    if w == 1:
+        return b
+    w = min(w, len(b))
+    count = len(b) - w + 1
+    shingles = []
+    for i in range(count):
+        shingles.append(b' '.join(b[i : i + w]))
+    return shingles