fosslight-source 2.2.17__tar.gz → 2.3.1__tar.gz

{fosslight_source-2.2.17/src/fosslight_source.egg-info → fosslight_source-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.17
+Version: 2.3.1
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
 Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
+Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.2.17"
+version = "2.3.1"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,6 +35,8 @@ dependencies = [
     "intbitset",
     "fosslight_binary>=5.1.22",
     "scancode-toolkit>=32.0.2",
+    # cryptography 49.x does not provide macOS x86_64 wheels, causing source builds to require OpenSSL/pkg-config.
+    "cryptography<49; platform_system == 'Darwin' and platform_machine == 'x86_64'",
     "fingerprints==1.2.3",
     "normality==2.6.1",
     # Python 3.13+ needs psycopg2-binary 2.9.10+ (has wheels; 2.9.9 builds fail with _PyInterpreterState_Get)

fosslight_source-2.3.1/src/fosslight_source/_kb_client.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import logging
+import time
+import urllib.error
+import urllib.request
+from typing import Dict, List, NamedTuple, Optional
+
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+_SCAN_JOB_POLL_INTERVAL_SEC = 1.0
+_SCAN_JOB_POLL_MAX_INTERVAL_SEC = 10.0
+_SCAN_JOB_REQUEST_TIMEOUT_SEC = 30
+_SCAN_JOB_MIN_WAIT_SEC = 300
+_SCAN_JOB_PER_HASH_SEC = 35
+
+
+def _kb_request(
+    kb_url: str,
+    path: str,
+    *,
+    method: str = "GET",
+    payload: dict | None = None,
+    kb_token: str = "",
+    timeout: int = _SCAN_JOB_REQUEST_TIMEOUT_SEC,
+) -> dict:
+    data = None
+    if payload is not None:
+        data = json.dumps(payload).encode("utf-8")
+    request = urllib.request.Request(f"{kb_url.rstrip('/')}/{path.lstrip('/')}", data=data, method=method)
+    request.add_header("Accept", "application/json")
+    if payload is not None:
+        request.add_header("Content-Type", "application/json")
+    if kb_token:
+        request.add_header("Authorization", f"Bearer {kb_token}")
+
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        body = response.read().decode()
+        return json.loads(body) if body else {}
+
+
+def _estimate_job_wait_timeout(file_hash_count: int) -> float:
+    return float(max(_SCAN_JOB_MIN_WAIT_SEC, file_hash_count * _SCAN_JOB_PER_HASH_SEC))
+
+
+def _coerce_count(value, default: int) -> int:
+    if value is None:
+        return default
+    try:
+        count = int(value)
+    except (TypeError, ValueError):
+        return default
+    return count if count >= 0 else default
+
+
+def _extract_response_message(response_body: dict) -> Optional[str]:
+    message = response_body.get("message")
+    if isinstance(message, str):
+        message = message.strip()
+        if message:
+            return message
+    return None
+
+
+def _scan_job_failure_message(response_body: dict) -> Optional[str]:
+    """Return server message when a scan/jobs response indicates failure."""
+    message = _extract_response_message(response_body)
+    if not message:
+        return None
+
+    status = response_body.get("status")
+    if status is None or str(status).lower() == "failed":
+        return message
+
+    if not response_body.get("job_id"):
+        return message
+
+    return None
+
+
+def _parse_http_error_body(error: urllib.error.HTTPError) -> dict:
+    try:
+        raw = error.read().decode()
+        return json.loads(raw) if raw else {}
+    except (json.JSONDecodeError, UnicodeDecodeError, OSError):
+        return {}
+
+
+class KbScanJobResult(NamedTuple):
+    origin_urls: Dict[str, str]
+    failure_message: Optional[str]
+    requested_count: int
+    returned_count: int
+
+
+def _kb_scan_job_result(
+    origin_urls: Dict[str, str],
+    failure_message: Optional[str],
+    requested_count: int,
+) -> KbScanJobResult:
+    return KbScanJobResult(
+        origin_urls=origin_urls,
+        failure_message=failure_message,
+        requested_count=requested_count,
+        returned_count=len(origin_urls),
+    )
+
+
+def fetch_origin_urls_via_scan_job(
+    file_hashes: List[str],
+    kb_url: str,
+    kb_token: str,
+) -> KbScanJobResult:
+    """
+    Create a POST /scan/jobs request, poll until completion, and return a file_hash -> origin_url map.
+    :param file_hashes: list of MD5 file hashes to look up.
+    :param kb_url: KB API base URL.
+    :param kb_token: KB API bearer token.
+    :return: origin URLs, optional failure message, and requested/returned file_hash counts.
+    """
+    unique_hashes = list(dict.fromkeys(h for h in file_hashes if h))
+    requested_count = len(unique_hashes)
+    if not unique_hashes:
+        return _kb_scan_job_result({}, None, 0)
+
+    create_payload = {"file_hashes": unique_hashes}
+    try:
+        created = _kb_request(kb_url, "scan/jobs", method="POST", payload=create_payload, kb_token=kb_token)
+    except urllib.error.HTTPError as e:
+        failure_message = _scan_job_failure_message(_parse_http_error_body(e))
+        if failure_message:
+            logger.warning(f"KB scan job create failed: {failure_message}")
+            return _kb_scan_job_result({}, failure_message, requested_count)
+        logger.warning(f"KB scan job create failed: HTTP {e.code} {e.reason}")
+        return _kb_scan_job_result({}, None, requested_count)
+    except urllib.error.URLError as e:
+        logger.warning(f"KB scan job create failed: {e}")
+        return _kb_scan_job_result({}, None, requested_count)
+    except Exception as e:
+        logger.warning(f"KB scan job create failed: {e}")
+        return _kb_scan_job_result({}, None, requested_count)
+
+    failure_message = _scan_job_failure_message(created)
+    if failure_message:
+        logger.warning(f"KB scan job create failed: {failure_message}")
+        return _kb_scan_job_result({}, failure_message, requested_count)
+
+    if str(created.get("status", "")).lower() == "failed":
+        logger.warning("KB scan job create failed")
+        return _kb_scan_job_result({}, None, requested_count)
+
+    job_id = created.get("job_id", "")
+    if not job_id:
+        logger.warning("KB scan job create response missing job_id")
+        return _kb_scan_job_result({}, None, requested_count)
+
+    fallback_count = len(unique_hashes)
+    accepted = _coerce_count(
+        created.get("accepted"),
+        _coerce_count(created.get("total"), fallback_count),
+    )
+    skipped = _coerce_count(created.get("skipped"), 0)
+    logger.info(
+        f"KB scan job created: job_id={job_id}, total={created.get('total', fallback_count)}, "
+        f"accepted={accepted}, skipped={skipped}"
+    )
+    if skipped:
+        logger.warning(f"KB scan job rate-limited: {skipped} file_hash(es) skipped by server")
+    if accepted == 0:
+        failure_message = (
+            f"rate-limited: {skipped} file_hash(es) skipped by server"
+            if skipped
+            else "scan job accepted no file_hashes"
+        )
+        return _kb_scan_job_result({}, failure_message, requested_count)
+
+    deadline = time.monotonic() + _estimate_job_wait_timeout(accepted)
+    interval = _SCAN_JOB_POLL_INTERVAL_SEC
+    origin_urls: Dict[str, str] = {}
+
+    while time.monotonic() < deadline:
+        try:
+            status = _kb_request(kb_url, f"scan/jobs/{job_id}", kb_token=kb_token)
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                logger.warning(f"KB scan job not found: {job_id}")
+                return _kb_scan_job_result(origin_urls, "scan job not found", requested_count)
+            failure_message = _scan_job_failure_message(_parse_http_error_body(e))
+            if failure_message:
+                logger.warning(f"KB scan job status failed: {failure_message}")
+                return _kb_scan_job_result(origin_urls, failure_message, requested_count)
+            logger.warning(f"KB scan job status failed: HTTP {e.code}")
+            time.sleep(interval)
+            interval = min(interval * 1.5, _SCAN_JOB_POLL_MAX_INTERVAL_SEC)
+            continue
+        except urllib.error.URLError as e:
+            logger.warning(f"KB scan job status failed: {e}")
+            time.sleep(interval)
+            interval = min(interval * 1.5, _SCAN_JOB_POLL_MAX_INTERVAL_SEC)
+            continue
+        except Exception as e:
+            logger.warning(f"KB scan job status parse failed: {e}")
+            time.sleep(interval)
+            interval = min(interval * 1.5, _SCAN_JOB_POLL_MAX_INTERVAL_SEC)
+            continue
+
+        job_status = status.get("status", "")
+        if job_status == "completed":
+            for row in status.get("results", []):
+                if not isinstance(row, dict):
+                    continue
+                file_hash = row.get("file_hash", "")
+                if row.get("success") and row.get("output") and file_hash:
+                    origin_urls[file_hash] = row["output"]
+            logger.info(
+                f"KB scan job completed: job_id={job_id}, "
+                f"matched={len(origin_urls)}, failed={status.get('failed', 0)}"
+            )
+            return _kb_scan_job_result(origin_urls, None, requested_count)
+
+        if job_status == "failed":
+            failure_message = _scan_job_failure_message(status)
+            if failure_message:
+                logger.warning(f"KB scan job failed: job_id={job_id}, message={failure_message}")
+            else:
+                logger.warning(f"KB scan job failed: job_id={job_id}")
+            return _kb_scan_job_result(origin_urls, failure_message or "scan job failed", requested_count)
+
+        time.sleep(interval)
+        interval = min(interval * 1.5, _SCAN_JOB_POLL_MAX_INTERVAL_SEC)
+
+    logger.warning(f"KB scan job timed out: job_id={job_id}")
+    return _kb_scan_job_result(origin_urls, "scan job timed out", requested_count)

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source/_parsing_scancode_file_item.py

@@ -15,7 +15,7 @@ from typing import Tuple
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 REMOVE_LICENSE = ["warranty-disclaimer"]
-regex = re.compile(r'licenseref-(\S+)', re.IGNORECASE)
+regex = re.compile(r'licenseref-([a-z0-9\.\-]+)', re.IGNORECASE)
 find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
 KEYWORD_SPDX_ID = r'SPDX-License-Identifier\s*[\S]+'
 KEYWORD_DOWNLOAD_LOC = r'DownloadLocation\s*[\S]+'

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source/_scan_item.py

@@ -6,11 +6,7 @@
 import os
 import logging
 import re
-import json
-import base64
 import hashlib
-import urllib.request
-import urllib.error
 import fosslight_util.constant as constant
 from fosslight_util.oss_item import FileItem, OssItem, get_checksum_sha1
 
@@ -63,8 +59,9 @@ class SourceItem(FileItem):
         self.oss_version = ""
 
         self.checksum = get_checksum_sha1(value)
-        self.kb_origin_url = ""  # URL from OSS KB (_get_origin_url_from_md5_hash)
+        self.kb_origin_url = ""  # URL from OSS KB
         self.kb_evidence = ""   # Evidence from KB API (exact_match or code snippet)
+        self._cached_kb_md5 = ""  # MD5 precomputed for KB lookup (set by _collect_kb_file_hashes)
 
     def __del__(self) -> None:
         pass
@@ -124,37 +121,18 @@ class SourceItem(FileItem):
             logger.debug(f"Failed to compute MD5 for {self.source_name_or_path}: {e}")
         return md5_hex, wfp
 
-    def _get_origin_url_from_md5_hash(
-        self, md5_hash: str, wfp: str = "", kb_url: str = DEFAULT_KB_URL, kb_token: str = ""
-    ) -> str:
-        """Return origin_url from KB API."""
-        try:
-            payload = {"file_hash": md5_hash}
-            if wfp and wfp.strip():
-                payload["wfp_base64"] = base64.b64encode(wfp.strip().encode("utf-8")).decode("ascii")
-            request = urllib.request.Request(
-                f"{kb_url}query", data=json.dumps(payload).encode('utf-8'), method='POST'
-            )
-            request.add_header('Accept', 'application/json')
-            request.add_header('Content-Type', 'application/json')
-            if kb_token:
-                request.add_header('Authorization', f'Bearer {kb_token}')
-
-            with urllib.request.urlopen(request, timeout=10) as response:
-                data = json.loads(response.read().decode())
-                if isinstance(data, dict):
-                    return_code = data.get('return_code', -1)
-                    if return_code == 0:
-                        output = data.get('output', '')
-                        if output:
-                            return output
-        except urllib.error.URLError as e:
-            logger.debug(f"Failed to fetch origin_url from API for MD5 hash {md5_hash}: {e}")
-        except json.JSONDecodeError as e:
-            logger.debug(f"Failed to parse API response for MD5 hash {md5_hash}: {e}")
-        except Exception as e:
-            logger.debug(f"Error getting origin_url for MD5 hash {md5_hash}: {e}")
-        return ""
+    def _apply_kb_origin_url(self, origin_url: str) -> tuple[str, str, str]:
+        """Apply KB origin URL and return (oss_name, oss_version, download_url)."""
+        self.kb_origin_url = origin_url
+        self.kb_evidence = "exact_match"
+        extracted_name, extracted_version, repo_url = self._extract_oss_info_from_url(origin_url)
+        if extracted_name:
+            self.oss_name = extracted_name
+        if extracted_version:
+            self.oss_version = extracted_version
+        download_url = repo_url if repo_url else origin_url
+        self.download_location = [download_url]
+        return self.oss_name, self.oss_version, download_url
 
     def _extract_oss_info_from_url(self, url: str) -> tuple:
         """
@@ -196,7 +174,9 @@ class SourceItem(FileItem):
             return "", "", ""
 
     def set_oss_item(
-        self, path_to_scan: str = "", run_kb: bool = False, kb_url: str = DEFAULT_KB_URL, kb_token: str = ""
+        self,
+        path_to_scan: str = "",
+        kb_origin_urls: dict[str, str] | None = None,
     ) -> None:
         self.oss_items = []
         if self.download_location:
@@ -207,21 +187,15 @@ class SourceItem(FileItem):
                 self.oss_items.append(item)
         else:
             item = OssItem(self.oss_name, self.oss_version, self.licenses)
-            if run_kb and not self.is_license_text:
-                md5_hash, wfp = self._get_hash(path_to_scan)
+            if kb_origin_urls and not self.is_license_text:
+                md5_hash = self._cached_kb_md5
+                if not md5_hash:
+                    md5_hash, _wfp = self._get_hash(path_to_scan)
                 if md5_hash:
-                    origin_url = self._get_origin_url_from_md5_hash(md5_hash, wfp, kb_url, kb_token)
+                    origin_url = kb_origin_urls.get(md5_hash, "")
                     if origin_url:
-                        self.kb_origin_url = origin_url
-                        self.kb_evidence = "exact_match"
-                        extracted_name, extracted_version, repo_url = self._extract_oss_info_from_url(origin_url)
-                        if extracted_name:
-                            self.oss_name = extracted_name
-                        if extracted_version:
-                            self.oss_version = extracted_version
-                        download_url = repo_url if repo_url else origin_url
-                        self.download_location = [download_url]
-                        item = OssItem(self.oss_name, self.oss_version, self.licenses, download_url)
+                        oss_name, oss_version, download_url = self._apply_kb_origin_url(origin_url)
+                        item = OssItem(oss_name, oss_version, self.licenses, download_url)
 
             item.copyright = "\n".join(self.copyright)
             item.comment = self.comment

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source/cli.py

@@ -25,12 +25,14 @@ from fosslight_util.exclude import get_excluded_paths
 from .run_scanoss import run_scanoss_py
 from .run_scanoss import get_scanoss_extra_info
 import yaml
+import tqdm
 import argparse
 from .run_spdx_extractor import get_spdx_downloads
 from .run_manifest_extractor import get_manifest_licenses
-from ._scan_item import SourceItem, resolve_kb_config
+from ._scan_item import SourceItem, resolve_kb_config, is_notice_file
+from ._kb_client import fetch_origin_urls_via_scan_job
 from fosslight_util.oss_item import ScannerItem
-from typing import Tuple
+from typing import Optional, Tuple
 from ._scan_item import is_manifest_file
 import shutil
 
@@ -330,11 +332,57 @@ def mark_oss_info_correction_files_as_excluded(scan_results: list) -> None:
             item.comment = OSS_INFO_CORRECTION_COMMENT
 
 
+def _collect_kb_file_hashes(
+    scancode_result: list,
+    path_to_scan: str,
+    excluded_files: set,
+    hide_progress: bool,
+) -> tuple[list[str], list[tuple[SourceItem, str]]]:
+    """Collect MD5 hashes from scancode results and walk targets, plus (extra_item, md5) candidates.
+
+    Skips license/notice files and scancode_result items that already have download_location.
+    ScanOSS/SPDX results are merged into scancode_result before this runs.
+    """
+    file_hashes: list[str] = []
+    extra_candidates: list[tuple[SourceItem, str]] = []
+
+    for item in scancode_result:
+        if item.is_license_text or is_notice_file(item.source_name_or_path):
+            continue
+        if item.download_location:
+            continue
+        md5_hash, _wfp = item._get_hash(path_to_scan)
+        if md5_hash:
+            item._cached_kb_md5 = md5_hash
+            file_hashes.append(md5_hash)
+
+    abs_path_to_scan = os.path.abspath(path_to_scan)
+    scancode_paths = {item.source_name_or_path for item in scancode_result}
+
+    files_to_scan = []
+    for root, _dirs, files in os.walk(path_to_scan):
+        for file in files:
+            files_to_scan.append(os.path.join(root, file))
+
+    for file_path in tqdm.tqdm(files_to_scan, desc="KB Hashing", disable=hide_progress):
+        rel_path = os.path.relpath(file_path, abs_path_to_scan).replace("\\", "/")
+        if rel_path in scancode_paths or rel_path in excluded_files or is_notice_file(file_path):
+            continue
+        extra_item = SourceItem(rel_path)
+        md5_hash, _wfp = extra_item._get_hash(path_to_scan)
+        if md5_hash:
+            extra_item._cached_kb_md5 = md5_hash
+            file_hashes.append(md5_hash)
+            extra_candidates.append((extra_item, md5_hash))
+
+    return file_hashes, extra_candidates
+
+
 def merge_results(
     scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
     path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {},
     excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
-) -> list:
+) -> tuple[list, Optional[str], int, int]:
 
     """
     Merge scanner results and spdx parsing result.
@@ -346,7 +394,7 @@ def merge_results(
     :param excluded_files: set of relative paths to exclude from KB-only file discovery.
     :param kb_url: KB API base URL.
     :param kb_token: KB API bearer token.
-    :return merged_result: list of merged result in SourceItem.
+    :return: (merged_result, kb failure message, requested file_hash count, returned match count).
     """
     if excluded_files is None:
         excluded_files = set()
@@ -381,32 +429,60 @@ def merge_results(
                 new_result_item.is_manifest_file = True
                 scancode_result.append(new_result_item)
 
+    kb_origin_urls: dict[str, str] = {}
+    kb_status_message: Optional[str] = None
+    kb_requested_count = 0
+    kb_returned_count = 0
+    extra_candidates: list[tuple[SourceItem, str]] = []
+    if run_kb:
+        file_hashes, extra_candidates = _collect_kb_file_hashes(
+            scancode_result, path_to_scan, excluded_files, hide_progress
+        )
+        if file_hashes:
+            kb_result = fetch_origin_urls_via_scan_job(file_hashes, kb_url, kb_token)
+            kb_origin_urls = kb_result.origin_urls
+            kb_status_message = kb_result.failure_message
+            kb_requested_count = kb_result.requested_count
+            kb_returned_count = kb_result.returned_count
+
     for item in scancode_result:
-        item.set_oss_item(path_to_scan, run_kb, kb_url, kb_token)
+        item.set_oss_item(path_to_scan, kb_origin_urls=kb_origin_urls)
 
     # Add OSSItem for files in path_to_scan that are not in scancode_result
     # when KB returns an origin URL for their MD5 hash (skip excluded_files)
     if run_kb:
-        import tqdm
-        abs_path_to_scan = os.path.abspath(path_to_scan)
-        scancode_paths = {item.source_name_or_path for item in scancode_result}
-
-        files_to_scan = []
-        for root, _dirs, files in os.walk(path_to_scan):
-            for file in files:
-                files_to_scan.append(os.path.join(root, file))
-
-        for file_path in tqdm.tqdm(files_to_scan, desc="KB Scanning", disable=hide_progress):
-            rel_path = os.path.relpath(file_path, abs_path_to_scan).replace("\\", "/")
-            if rel_path in scancode_paths or rel_path in excluded_files:
-                continue
-            extra_item = SourceItem(rel_path)
-            extra_item.set_oss_item(path_to_scan, run_kb, kb_url, kb_token)
+        for extra_item, _md5_hash in extra_candidates:
+            extra_item.set_oss_item(path_to_scan, kb_origin_urls=kb_origin_urls)
             if extra_item.download_location:
                 scancode_result.append(extra_item)
-                scancode_paths.add(rel_path)
 
-    return scancode_result
+    return scancode_result, kb_status_message, kb_requested_count, kb_returned_count
+
+
+def _finalize_temp_output(
+    temp_output_path: str,
+    final_output_path: str,
+    publish: bool,
+    log: Optional[logging.Logger] = None,
+) -> bool:
+    """Copy scan artifacts from temp dir, then always remove the temp directory."""
+    if not temp_output_path or not os.path.isdir(temp_output_path):
+        return True
+    publish_ok = True
+    try:
+        if publish:
+            shutil.copytree(temp_output_path, final_output_path, dirs_exist_ok=True)
+    except Exception as ex:
+        publish_ok = False
+        if log:
+            log.error(f"Failed to publish scan artifacts: {ex}")
+    finally:
+        try:
+            shutil.rmtree(temp_output_path)
+        except Exception as ex:
+            if log:
+                log.debug(f"Failed to cleanup temp output directory: {ex}")
+    return publish_ok
 
 
 def run_scanners(
@@ -454,77 +530,94 @@ def run_scanners(
         output_path = os.getcwd()
     final_output_path = output_path
     output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{start_time}')
+    publish_temp_output = False
+    logger = None
+    publish_ok = True
 
-    logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
-                                  True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
+    try:
+        logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
+                                      True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
 
-    logger.info(f"Tool Info : {result_log['Tool Info']}")
+        logger.info(f"Tool Info : {result_log['Tool Info']}")
 
-    if '.xlsx' not in output_extensions and print_matched_text:
-        logger.warning("-m option is only available for excel.")
-        print_matched_text = False
+        if '.xlsx' not in output_extensions and print_matched_text:
+            logger.warning("-m option is only available for excel.")
+            print_matched_text = False
 
-    if success:
-        if all_exclude_mode and len(all_exclude_mode) == 4:
-            (excluded_path_with_default_exclusion,
-             excluded_path_without_dot,
-             excluded_files,
-             cnt_file_except_skipped) = all_exclude_mode
-        else:
-            path_to_exclude_with_filename = path_to_exclude
-            (excluded_path_with_default_exclusion,
-             excluded_path_without_dot,
-             excluded_files,
-             cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename)
-            logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
-
-        if not selected_scanner:
-            selected_scanner = ALL_MODE
-        if selected_scanner in ['scancode', ALL_MODE]:
-            success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
-                                                                                      write_json_file, num_cores, True,
-                                                                                      print_matched_text, formats, called_by_cli,
-                                                                                      time_out, correct_mode, correct_filepath,
-                                                                                      excluded_path_with_default_exclusion,
-                                                                                      excluded_files, hide_progress)
-        excluded_files = set(excluded_files) if excluded_files else set()
-        if selected_scanner in ['scanoss', ALL_MODE]:
-            scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
-                                                              excluded_path_with_default_exclusion, excluded_files,
-                                                              write_json_file, hide_progress)
-
-        run_kb_msg = ""
-        if selected_scanner in SCANNER_TYPE:
-            run_kb = True if selected_scanner in ['kb', ALL_MODE] else False
-            if run_kb:
-                if not check_kb_server_reachable(kb_url, kb_token):
-                    run_kb = False
-                    run_kb_msg = f"KB({kb_url}) Unreachable"
-                else:
-                    run_kb_msg = f"KB({kb_url}) Enabled"
-
-            spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
-            merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads,
-                                          path_to_scan, run_kb, manifest_licenses, excluded_files,
-                                          hide_progress, kb_url, kb_token)
-            mark_oss_info_correction_files_as_excluded(merged_result)
-            scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
-                                           print_matched_text, output_path, output_files, output_extensions, correct_mode,
-                                           correct_filepath, path_to_scan, excluded_path_without_dot, formats,
-                                           api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
+        if success:
+            if all_exclude_mode and len(all_exclude_mode) == 4:
+                (excluded_path_with_default_exclusion,
+                 excluded_path_without_dot,
+                 excluded_files,
+                 cnt_file_except_skipped) = all_exclude_mode
+            else:
+                path_to_exclude_with_filename = path_to_exclude
+                (excluded_path_with_default_exclusion,
+                 excluded_path_without_dot,
+                 excluded_files,
+                 cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename)
+                logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
+
+            if not selected_scanner:
+                selected_scanner = ALL_MODE
+            if selected_scanner in ['scancode', ALL_MODE]:
+                success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(
+                    path_to_scan, output_file_name, write_json_file, num_cores, True,
+                    print_matched_text, formats, called_by_cli, time_out, correct_mode,
+                    correct_filepath, excluded_path_with_default_exclusion,
+                    excluded_files, hide_progress,
+                )
+            excluded_files = set(excluded_files) if excluded_files else set()
+            if selected_scanner in ['scanoss', ALL_MODE]:
+                scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
+                                                                  excluded_path_with_default_exclusion, excluded_files,
+                                                                  write_json_file, hide_progress)
+
+            run_kb_msg = ""
+            if selected_scanner in SCANNER_TYPE:
+                run_kb = True if selected_scanner in ['kb', ALL_MODE] else False
+                if run_kb:
+                    if not check_kb_server_reachable(kb_url, kb_token):
+                        run_kb = False
+                        run_kb_msg = f"KB({kb_url}) Unreachable"
+
+                spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
+                merged_result, kb_status_message, kb_requested_count, kb_returned_count = merge_results(
+                    scancode_result, scanoss_result, spdx_downloads,
+                    path_to_scan, run_kb, manifest_licenses, excluded_files,
+                    hide_progress, kb_url, kb_token,
+                )
+                if kb_status_message:
+                    run_kb_msg = f"KB({kb_url}) {kb_status_message}"
+                elif run_kb and kb_requested_count > 0:
+                    run_kb_msg = (
+                        f"KB({kb_url}) response : {kb_returned_count}/"
+                        f" requested: {kb_requested_count}"
+                    )
+                mark_oss_info_correction_files_as_excluded(merged_result)
+                scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
+                                               print_matched_text, output_path, output_files, output_extensions, correct_mode,
+                                               correct_filepath, path_to_scan, excluded_path_without_dot, formats,
+                                               api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
+            else:
+                print_help_msg_source_scanner()
+                result_log[RESULT_KEY] = "Unsupported scanner"
+                success = False
         else:
-            print_help_msg_source_scanner()
-            result_log[RESULT_KEY] = "Unsupported scanner"
+            result_log[RESULT_KEY] = f"Format error. {msg}"
             success = False
-    else:
-        result_log[RESULT_KEY] = f"Format error. {msg}"
-        success = False
 
-    try:
-        shutil.copytree(output_path, final_output_path, dirs_exist_ok=True)
-        shutil.rmtree(output_path)
-    except Exception as ex:
-        logger.debug(f"Failed to move temp files: {ex}")
+        publish_temp_output = True
+    finally:
+        publish_ok = _finalize_temp_output(output_path, final_output_path, publish_temp_output, logger)
+
+    if publish_temp_output and not publish_ok:
+        success = False
+        prev_msg = result_log.get(RESULT_KEY, "")
+        result_log[RESULT_KEY] = (
+            f"{prev_msg}, Failed to publish scan artifacts" if prev_msg
+            else "Failed to publish scan artifacts"
+        )
 
     return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
 

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source/run_scancode.py

@@ -63,14 +63,23 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None:
         logger.debug("scancode UNSET workaround skipped: %s", ex)
 
 
+def _directory_ignore_pattern(dir_name: str) -> str:
+    """Path-based glob for a directory name (avoids matching the scan root itself)."""
+    normalized = dir_name.strip().strip("/").replace("\\", "/")
+    if not normalized:
+        return dir_name
+    return f"**/{normalized}/**"
+
+
 def _default_scancode_coarse_ignore_patterns() -> frozenset:
     """
     Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules.
-    Uses segment-style globs so scancode does not need one pattern per file.
+    Directory names use path-based globs (e.g. **/tests/**) so they do not match
+    the scan root directory name itself.
     """
     patterns = {".*"}
     for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY:
-        patterns.add(name)
+        patterns.add(_directory_ignore_pattern(name))
     for ext in EXCLUDE_FILE_EXTENSION:
         patterns.add(f"*.{ext}")
     for name in EXCLUDE_FILENAME:

{fosslight_source-2.2.17 → fosslight_source-2.3.1/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.17
+Version: 2.3.1
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
 Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
+Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ README.md
 pyproject.toml
 src/fosslight_source/__init__.py
 src/fosslight_source/_help.py
+src/fosslight_source/_kb_client.py
 src/fosslight_source/_license_matched.py
 src/fosslight_source/_parsing_scancode_file_item.py
 src/fosslight_source/_parsing_scanoss_file.py

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/src/fosslight_source.egg-info/requires.txt

@@ -12,6 +12,9 @@ fingerprints==1.2.3
 normality==2.6.1
 tqdm
 
+[:platform_system == "Darwin" and platform_machine == "x86_64"]
+cryptography<49
+
 [:python_version < "3.11"]
 tomli
 

{fosslight_source-2.2.17 → fosslight_source-2.3.1}/tests/test_tox.py

@@ -3,10 +3,13 @@
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
+import shlex
 import subprocess
 import pytest
 import shutil
 import sys
+import csv
+import glob
 
 # Add project root to sys.path for importing FL Source modules
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
@@ -18,6 +21,26 @@ from fosslight_source._parsing_scancode_file_item import (
 )
 
 remove_directories = ["test_scan", "test_scan2", "test_scan3"]
+TEST_FILES_SCAN_DIR = "test_scan"
+
+
+def _parse_license_tokens(license_value: str) -> set[str]:
+    return {token.strip().lower() for token in (license_value or "").split(",") if token.strip()}
+
+
+def _read_src_csv_rows(csv_path: str) -> list[dict]:
+    with open(csv_path, "r", encoding="utf-8") as file:
+        return list(csv.DictReader(file, delimiter="\t"))
+
+
+def _rows_for_source(rows: list[dict], source_name: str) -> list[dict]:
+    return [row for row in rows if row.get("Source Path") == source_name]
+
+
+def _find_scan_csv(output_dir: str) -> str:
+    csv_files = sorted(glob.glob(os.path.join(output_dir, "*.csv")))
+    assert csv_files, f"No CSV report found under {output_dir}"
+    return csv_files[-1]
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -31,8 +54,22 @@ def setup_test_result_dir():
 
 
 def run_command(command):
-    process = subprocess.run(command, shell=True, capture_output=True, text=True)
-    success = (process.returncode == 0)
+    command = command.strip()
+    if command.startswith("fosslight_source"):
+        args = shlex.split(command, posix=(os.name != "nt"))[1:]
+        if os.environ.get("FOSSLIGHT_USE_LOCAL_SRC"):
+            src_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src"))
+            env = os.environ.copy()
+            existing = env.get("PYTHONPATH", "")
+            env["PYTHONPATH"] = src_path if not existing else f"{src_path}{os.pathsep}{existing}"
+            cmd = [sys.executable, "-m", "fosslight_source.cli", *args]
+            process = subprocess.run(cmd, capture_output=True, text=True, env=env)
+        else:
+            cmd = ["fosslight_source", *args]
+            process = subprocess.run(cmd, capture_output=True, text=True)
+    else:
+        process = subprocess.run(command, shell=True, capture_output=True, text=True)
+    success = process.returncode == 0
     return success, process.stdout if success else process.stderr
 
 
@@ -112,6 +149,32 @@ def test_run():
     assert len(scan2_files) > 0, "Test Run: No scan files created in test_scan2 directory"
 
 
+def test_test_files_scan_results():
+    os.makedirs(TEST_FILES_SCAN_DIR, exist_ok=True)
+
+    success, msg = run_command(
+        f"fosslight_source -p tests/test_files -s scancode -f csv -o {TEST_FILES_SCAN_DIR}/"
+    )
+    assert success is True, f"Test Run: test_files scan failed: {msg}"
+
+    csv_path = _find_scan_csv(TEST_FILES_SCAN_DIR)
+    rows = _read_src_csv_rows(csv_path)
+
+    sample_rows = _rows_for_source(rows, "sample.cpp")
+    assert sample_rows, "Test Run: sample.cpp not found in scan result"
+    for row in sample_rows:
+        licenses = _parse_license_tokens(row.get("License", ""))
+        assert "apache-2.0" in licenses, f"sample.cpp missing Apache-2.0 license: {row.get('License')}"
+        assert "mit" in licenses, f"sample.cpp missing MIT license: {row.get('License')}"
+
+    temp_rows = _rows_for_source(rows, "temp.cpp")
+    assert temp_rows, "Test Run: temp.cpp not found in scan result"
+    temp_row = temp_rows[0]
+    temp_licenses = _parse_license_tokens(temp_row.get("License", ""))
+    assert "apache-2.0" in temp_licenses, f"temp.cpp missing Apache-2.0 license: {temp_row.get('License')}"
+    assert (temp_row.get("Copyright Text") or "").strip(), "Test Run: temp.cpp copyright not extracted"
+
+
 def test_help_command():
     success, msg = run_command("fosslight_source -h")
     assert success is True, f"Test Release: Help command failed :{msg}"