fosslight-source 2.1.19__tar.gz → 2.2.0__tar.gz

{fosslight_source-2.1.19/src/fosslight_source.egg-info → fosslight_source-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.1.19
+Version: 2.2.0
 Summary: FOSSLight Source Scanner
 Home-page: https://github.com/fosslight/fosslight_source_scanner
 Download-URL: https://github.com/fosslight/fosslight_source_scanner

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_source',
-        version='2.1.19',
+        version='2.2.0',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Source Scanner',

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/src/fosslight_source/_help.py

@@ -27,7 +27,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
             \t\t\t   ({', '.join(SUPPORT_FORMAT)})
             \t\t\t   Multiple formats can be specified separated by space.
         Options only for FOSSLight Source Scanner
-            -s <scanner>\t   Select which scanner to be run (scancode, scanoss, all)
+            -s <scanner>\t   Select which scanner to be run (scancode, scanoss, kb, all)
             -j\t\t\t   Generate raw result of scanners in json format
             -t <float>\t\t   Stop scancode scanning if scanning takes longer than a timeout in seconds.
             -c <core>\t\t   Select the number of cores to be scanned with ScanCode or threads with SCANOSS.

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/src/fosslight_source/_parsing_scancode_file_item.py

@@ -29,6 +29,14 @@ SPDX_REPLACE_WORDS = ["(", ")"]
 KEY_AND = r"(?<=\s)and(?=\s)"
 KEY_OR = r"(?<=\s)or(?=\s)"
 GPL_LICENSE_PATTERN = r'((a|l)?gpl|gfdl)'  # GPL, LGPL, AGPL, GFDL
+SOURCE_EXTENSIONS = [
+    '.java', '.cpp', '.c', '.cc', '.cxx', '.c++', '.h', '.hh', '.hpp', '.hxx', '.h++',
+    '.cs', '.py', '.pyw', '.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx',
+    '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.kts', '.scala', '.sc',
+    '.m', '.mm', '.dart', '.lua', '.pl', '.pm', '.r', '.R',
+    '.hs', '.clj', '.cljs', '.ex', '.exs', '.groovy', '.gradle',
+    '.vue', '.svelte', '.asm', '.s', '.i', '.ii'
+]
 
 
 def is_gpl_family_license(licenses: list) -> bool:
@@ -328,7 +336,11 @@ def parsing_scancode_32_later(
                 result_item.licenses = license_detected
 
                 result_item.exclude = is_exclude_file(file_path)
-                result_item.is_license_text = file.get("percentage_of_license_text", 0) > 90 or is_notice_file(file_path)
+                file_ext = os.path.splitext(file_path)[1].lower()
+                is_source_file = file_ext and file_ext in SOURCE_EXTENSIONS
+                result_item.is_license_text = is_notice_file(file_path) or (
+                    file.get("percentage_of_license_text", 0) > 90 and not is_source_file
+                )
 
                 detected_without_pom = []
                 if is_manifest_file(file_path) and len(license_detected) > 0:
@@ -371,7 +383,7 @@ def parsing_scancode_32_later(
                     license_expression = file.get("detected_license_expression", "")
                     if license_expression_spdx:
                         license_expression = license_expression_spdx
-                    if license_expression:
+                    if license_expression and "OR" in license_expression:
                         result_item.comment = license_expression
 
                 scancode_file_item.append(result_item)

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/src/fosslight_source/_scan_item.py

@@ -6,6 +6,10 @@
 import os
 import logging
 import re
+import json
+import hashlib
+import urllib.request
+import urllib.error
 import fosslight_util.constant as constant
 from fosslight_util.oss_item import FileItem, OssItem, get_checksum_sha1
 
@@ -26,6 +30,7 @@ _package_directory = ["node_modules", "venv", "Pods", "Carthage"]
 MAX_LICENSE_LENGTH = 200
 MAX_LICENSE_TOTAL_LENGTH = 600
 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)"
+KB_URL = "http://fosslight-kb.lge.com/query"
 
 
 class SourceItem(FileItem):
@@ -77,7 +82,90 @@ class SourceItem(FileItem):
         else:
             self._licenses = value
 
-    def set_oss_item(self) -> None:
+    def _get_md5_hash(self, path_to_scan: str = "") -> str:
+        try:
+            file_path = self.source_name_or_path
+            if path_to_scan and not os.path.isabs(file_path):
+                file_path = os.path.join(path_to_scan, file_path)
+            file_path = os.path.normpath(file_path)
+
+            if os.path.isfile(file_path):
+                md5_hash = hashlib.md5()
+                with open(file_path, "rb") as f:
+                    for chunk in iter(lambda: f.read(4096), b""):
+                        md5_hash.update(chunk)
+                return md5_hash.hexdigest()
+        except FileNotFoundError:
+            logger.warning(f"File not found: {self.source_name_or_path}")
+        except PermissionError:
+            logger.warning(f"Permission denied: {self.source_name_or_path}")
+        except Exception as e:
+            logger.warning(f"Failed to compute MD5 for {self.source_name_or_path}: {e}")
+        return ""
+
+    def _get_origin_url_from_md5_hash(self, md5_hash: str) -> str:
+        try:
+            request = urllib.request.Request(KB_URL, data=json.dumps({"file_hash": md5_hash}).encode('utf-8'), method='POST')
+            request.add_header('Accept', 'application/json')
+            request.add_header('Content-Type', 'application/json')
+
+            with urllib.request.urlopen(request, timeout=10) as response:
+                data = json.loads(response.read().decode())
+                if isinstance(data, dict):
+                    # Only extract output if return_code is 0 (success)
+                    return_code = data.get('return_code', -1)
+                    if return_code == 0:
+                        output = data.get('output', '')
+                        if output:
+                            return output
+        except urllib.error.URLError as e:
+            logger.warning(f"Failed to fetch origin_url from API for MD5 hash {md5_hash}: {e}")
+        except json.JSONDecodeError as e:
+            logger.warning(f"Failed to parse API response for MD5 hash {md5_hash}: {e}")
+        except Exception as e:
+            logger.warning(f"Error getting origin_url for MD5 hash {md5_hash}: {e}")
+        return ""
+
+    def _extract_oss_info_from_url(self, url: str) -> tuple:
+        """
+        Extract OSS name, version, and repository URL from GitHub URL.
+
+        Supported patterns:
+        - https://github.com/{owner}/{repo}/archive/{version}.zip
+        - https://github.com/{owner}/{repo}/archive/{tag}/{version}.zip
+        - https://github.com/{owner}/{repo}/releases/download/{version}/{filename}
+
+        :param url: GitHub URL to extract information from
+        :return: tuple of (repo_name, version, repo_url)
+        """
+        try:
+            repo_match = re.search(r'github\.com/([^/]+)/([^/]+)/', url)
+            if not repo_match:
+                return "", "", ""
+
+            owner = repo_match.group(1)
+            repo_name = repo_match.group(2)
+            repo_url = f"https://github.com/{owner}/{repo_name}"
+            version = ""
+            # Extract version from releases pattern first: /releases/download/{version}/
+            releases_match = re.search(r'/releases/download/([^/]+)/', url)
+            if releases_match:
+                version = releases_match.group(1)
+            else:
+                # Extract version from archive pattern: /archive/{version}.zip or /archive/{tag}/{version}.zip
+                # For pattern with tag, take the last segment before .zip
+                archive_match = re.search(r'/archive/(.+?)(?:\.zip|\.tar\.gz)?(?:[?#]|$)', url)
+                if archive_match:
+                    version_path = archive_match.group(1)
+                    version = version_path.split('/')[-1] if '/' in version_path else version_path
+            if re.match(r'^[0-9a-f]{7,40}$', version, re.IGNORECASE):
+                version = ""
+            return repo_name, version, repo_url
+        except Exception as e:
+            logger.debug(f"Failed to extract OSS info from URL {url}: {e}")
+            return "", "", ""
+
+    def set_oss_item(self, path_to_scan: str = "", run_kb: bool = False) -> None:
         self.oss_items = []
         if self.download_location:
             for url in self.download_location:
@@ -87,6 +175,20 @@ class SourceItem(FileItem):
                 self.oss_items.append(item)
         else:
             item = OssItem(self.oss_name, self.oss_version, self.licenses)
+            if run_kb and not self.is_license_text:
+                md5_hash = self._get_md5_hash(path_to_scan)
+                if md5_hash:
+                    origin_url = self._get_origin_url_from_md5_hash(md5_hash)
+                    if origin_url:
+                        extracted_name, extracted_version, repo_url = self._extract_oss_info_from_url(origin_url)
+                        if extracted_name:
+                            self.oss_name = extracted_name
+                        if extracted_version:
+                            self.oss_version = extracted_version
+                        download_url = repo_url if repo_url else origin_url
+                        self.download_location = [download_url]
+                        item = OssItem(self.oss_name, self.oss_version, self.licenses, download_url)
+
             item.copyright = "\n".join(self.copyright)
             item.comment = self.comment
             self.oss_items.append(item)
@@ -165,6 +267,15 @@ def is_package_dir(dir_path: str) -> bool:
     return False, ""
 
 
+def _has_parent_in_exclude_list(rel_path: str, path_to_exclude: list) -> bool:
+    path_parts = rel_path.replace('\\', '/').split('/')
+    for i in range(1, len(path_parts)):
+        parent_path = '/'.join(path_parts[:i])
+        if parent_path in path_to_exclude:
+            return True
+    return False
+
+
 def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = []) -> list:
     path_to_exclude = custom_excluded_paths.copy()
     abs_path_to_scan = os.path.abspath(path_to_scan)
@@ -173,11 +284,10 @@ def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = []) -> l
         for dir_name in dirs:
             dir_path = os.path.join(root, dir_name)
             rel_path = os.path.relpath(dir_path, abs_path_to_scan)
-            if dir_name in _package_directory:
-                if rel_path not in path_to_exclude:
+            if not _has_parent_in_exclude_list(rel_path, path_to_exclude):
+                if dir_name in _package_directory:
                     path_to_exclude.append(rel_path)
-            elif is_exclude_dir(rel_path):
-                if rel_path not in path_to_exclude:
+                elif is_exclude_dir(rel_path):
                     path_to_exclude.append(rel_path)
 
     return path_to_exclude

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/src/fosslight_source/cli.py

@@ -8,6 +8,8 @@ import os
 import platform
 import warnings
 import logging
+import urllib.request
+import urllib.error
 from datetime import datetime
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
@@ -24,7 +26,7 @@ from .run_scanoss import get_scanoss_extra_info
 import yaml
 import argparse
 from .run_spdx_extractor import get_spdx_downloads
-from ._scan_item import SourceItem
+from ._scan_item import SourceItem, KB_URL
 from fosslight_util.oss_item import ScannerItem
 from typing import Tuple
 
@@ -35,7 +37,7 @@ SCANOSS_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
 MERGED_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
                                   'OSS Version', 'License', 'Download Location',
                                   'Homepage', 'Copyright Text', 'Exclude', 'Comment', 'license_reference']}
-SCANNER_TYPE = ['scancode', 'scanoss', 'all', '']
+SCANNER_TYPE = ['kb', 'scancode', 'scanoss', 'all']
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -269,17 +271,35 @@ def create_report_file(
     return scan_item
 
 
-def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}) -> list:
+def check_kb_server_reachable() -> bool:
+    try:
+        request = urllib.request.Request(KB_URL, method='HEAD')
+        with urllib.request.urlopen(request, timeout=5) as response:
+            logger.debug(f"KB server is reachable. Response status: {response.status}")
+            return response.status != 404
+    except urllib.error.HTTPError as e:
+        return e.code != 404
+    except urllib.error.URLError:
+        return False
+    except Exception:
+        return False
+
+
+def merge_results(
+    scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
+    path_to_scan: str = "", run_kb: bool = False
+) -> list:
 
     """
     Merge scanner results and spdx parsing result.
     :param scancode_result: list of scancode results in SourceItem.
     :param scanoss_result: list of scanoss results in SourceItem.
     :param spdx_downloads: dictionary of spdx parsed results.
+    :param path_to_scan: path to the scanned directory for constructing absolute file paths.
+    :param run_kb: if True, load kb result.
     :return merged_result: list of merged result in SourceItem.
     """
 
-    # If anything that is found at SCANOSS only exist, add it to result.
     scancode_result.extend([item for item in scanoss_result if item not in scancode_result])
 
     # If download loc. in SPDX form found, overwrite the scanner result.
@@ -293,9 +313,15 @@ def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_do
                 new_result_item = SourceItem(file_name)
                 new_result_item.download_location = download_location
                 scancode_result.append(new_result_item)
+    if run_kb and not check_kb_server_reachable():
+        run_kb = False
+    if run_kb:
+        logger.info("KB server is reachable. Loading data from OSS KB.")
+    else:
+        logger.info("Skipping KB lookup.")
 
     for item in scancode_result:
-        item.set_oss_item()
+        item.set_oss_item(path_to_scan, run_kb)
 
     return scancode_result
 
@@ -347,18 +373,22 @@ def run_scanners(
 
     if success:
         excluded_path_with_default_exclusion = get_excluded_paths(path_to_scan, path_to_exclude)
-        if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '':
+        logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
+        if not selected_scanner:
+            selected_scanner = 'all'
+        if selected_scanner in ['scancode', 'all', 'kb']:
             success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
                                                                                       write_json_file, num_cores, True,
                                                                                       print_matched_text, formats, called_by_cli,
                                                                                       time_out, correct_mode, correct_filepath,
                                                                                       excluded_path_with_default_exclusion)
-        if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
+        if selected_scanner in ['scanoss', 'all']:
             scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file,
                                                               num_cores, excluded_path_with_default_exclusion)
         if selected_scanner in SCANNER_TYPE:
+            run_kb = True if selected_scanner in ['kb', 'all'] else False
             spdx_downloads = get_spdx_downloads(path_to_scan, excluded_path_with_default_exclusion)
-            merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
+            merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads, path_to_scan, run_kb)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                            correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list,

{fosslight_source-2.1.19 → fosslight_source-2.2.0}/src/fosslight_source/run_scancode.py

@@ -74,7 +74,6 @@ def run_scan(
         if os.path.isdir(path_to_scan):
             try:
                 time_out = float(time_out)
-                logger.debug(f"Skipped by Scancode: {path_to_exclude}")
                 pretty_params = {}
                 pretty_params["path_to_scan"] = path_to_scan
                 pretty_params["path_to_exclude"] = path_to_exclude

{fosslight_source-2.1.19 → fosslight_source-2.2.0/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.1.19
+Version: 2.2.0
 Summary: FOSSLight Source Scanner
 Home-page: https://github.com/fosslight/fosslight_source_scanner
 Download-URL: https://github.com/fosslight/fosslight_source_scanner