fosslight-source 2.3.0__tar.gz → 2.3.1__tar.gz

{fosslight_source-2.3.0/src/fosslight_source.egg-info → fosslight_source-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.0
+Version: 2.3.1
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0

{fosslight_source-2.3.0 → fosslight_source-2.3.1}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.3.0"
+version = "2.3.1"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"

{fosslight_source-2.3.0 → fosslight_source-2.3.1}/src/fosslight_source/cli.py

@@ -338,13 +338,19 @@ def _collect_kb_file_hashes(
     excluded_files: set,
     hide_progress: bool,
 ) -> tuple[list[str], list[tuple[SourceItem, str]]]:
-    """Collect MD5 hashes from scancode results and walk targets, plus (extra_item, md5) candidates."""
+    """Collect MD5 hashes from scancode results and walk targets, plus (extra_item, md5) candidates.
+
+    Skips license/notice files and scancode_result items that already have download_location.
+    ScanOSS/SPDX results are merged into scancode_result before this runs.
+    """
     file_hashes: list[str] = []
     extra_candidates: list[tuple[SourceItem, str]] = []
 
     for item in scancode_result:
         if item.is_license_text or is_notice_file(item.source_name_or_path):
             continue
+        if item.download_location:
+            continue
         md5_hash, _wfp = item._get_hash(path_to_scan)
         if md5_hash:
             item._cached_kb_md5 = md5_hash
@@ -453,6 +459,32 @@ def merge_results(
     return scancode_result, kb_status_message, kb_requested_count, kb_returned_count
 
 
+def _finalize_temp_output(
+    temp_output_path: str,
+    final_output_path: str,
+    publish: bool,
+    log: Optional[logging.Logger] = None,
+) -> bool:
+    """Copy scan artifacts from temp dir, then always remove the temp directory."""
+    if not temp_output_path or not os.path.isdir(temp_output_path):
+        return True
+    publish_ok = True
+    try:
+        if publish:
+            shutil.copytree(temp_output_path, final_output_path, dirs_exist_ok=True)
+    except Exception as ex:
+        publish_ok = False
+        if log:
+            log.error(f"Failed to publish scan artifacts: {ex}")
+    finally:
+        try:
+            shutil.rmtree(temp_output_path)
+        except Exception as ex:
+            if log:
+                log.debug(f"Failed to cleanup temp output directory: {ex}")
+    return publish_ok
+
+
 def run_scanners(
     path_to_scan: str, output_file_name: str = "",
     write_json_file: bool = False, num_cores: int = -1,
@@ -498,84 +530,94 @@ def run_scanners(
         output_path = os.getcwd()
     final_output_path = output_path
     output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{start_time}')
+    publish_temp_output = False
+    logger = None
+    publish_ok = True
 
-    logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
-                                  True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
+    try:
+        logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
+                                      True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
 
-    logger.info(f"Tool Info : {result_log['Tool Info']}")
+        logger.info(f"Tool Info : {result_log['Tool Info']}")
 
-    if '.xlsx' not in output_extensions and print_matched_text:
-        logger.warning("-m option is only available for excel.")
-        print_matched_text = False
+        if '.xlsx' not in output_extensions and print_matched_text:
+            logger.warning("-m option is only available for excel.")
+            print_matched_text = False
 
-    if success:
-        if all_exclude_mode and len(all_exclude_mode) == 4:
-            (excluded_path_with_default_exclusion,
-             excluded_path_without_dot,
-             excluded_files,
-             cnt_file_except_skipped) = all_exclude_mode
-        else:
-            path_to_exclude_with_filename = path_to_exclude
-            (excluded_path_with_default_exclusion,
-             excluded_path_without_dot,
-             excluded_files,
-             cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename)
-            logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
-
-        if not selected_scanner:
-            selected_scanner = ALL_MODE
-        if selected_scanner in ['scancode', ALL_MODE]:
-            success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
-                                                                                      write_json_file, num_cores, True,
-                                                                                      print_matched_text, formats, called_by_cli,
-                                                                                      time_out, correct_mode, correct_filepath,
-                                                                                      excluded_path_with_default_exclusion,
-                                                                                      excluded_files, hide_progress)
-        excluded_files = set(excluded_files) if excluded_files else set()
-        if selected_scanner in ['scanoss', ALL_MODE]:
-            scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
-                                                              excluded_path_with_default_exclusion, excluded_files,
-                                                              write_json_file, hide_progress)
-
-        run_kb_msg = ""
-        if selected_scanner in SCANNER_TYPE:
-            run_kb = True if selected_scanner in ['kb', ALL_MODE] else False
-            if run_kb:
-                if not check_kb_server_reachable(kb_url, kb_token):
-                    run_kb = False
-                    run_kb_msg = f"KB({kb_url}) Unreachable"
-
-            spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
-            merged_result, kb_status_message, kb_requested_count, kb_returned_count = merge_results(
-                scancode_result, scanoss_result, spdx_downloads,
-                path_to_scan, run_kb, manifest_licenses, excluded_files,
-                hide_progress, kb_url, kb_token,
-            )
-            if kb_status_message:
-                run_kb_msg = f"KB({kb_url}) {kb_status_message}"
-            elif run_kb and kb_requested_count > 0:
-                run_kb_msg = (
-                    f"KB({kb_url}) response : {kb_returned_count}/"
-                    f" requested: {kb_requested_count}"
+        if success:
+            if all_exclude_mode and len(all_exclude_mode) == 4:
+                (excluded_path_with_default_exclusion,
+                 excluded_path_without_dot,
+                 excluded_files,
+                 cnt_file_except_skipped) = all_exclude_mode
+            else:
+                path_to_exclude_with_filename = path_to_exclude
+                (excluded_path_with_default_exclusion,
+                 excluded_path_without_dot,
+                 excluded_files,
+                 cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename)
+                logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
+
+            if not selected_scanner:
+                selected_scanner = ALL_MODE
+            if selected_scanner in ['scancode', ALL_MODE]:
+                success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(
+                    path_to_scan, output_file_name, write_json_file, num_cores, True,
+                    print_matched_text, formats, called_by_cli, time_out, correct_mode,
+                    correct_filepath, excluded_path_with_default_exclusion,
+                    excluded_files, hide_progress,
+                )
+            excluded_files = set(excluded_files) if excluded_files else set()
+            if selected_scanner in ['scanoss', ALL_MODE]:
+                scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
+                                                                  excluded_path_with_default_exclusion, excluded_files,
+                                                                  write_json_file, hide_progress)
+
+            run_kb_msg = ""
+            if selected_scanner in SCANNER_TYPE:
+                run_kb = True if selected_scanner in ['kb', ALL_MODE] else False
+                if run_kb:
+                    if not check_kb_server_reachable(kb_url, kb_token):
+                        run_kb = False
+                        run_kb_msg = f"KB({kb_url}) Unreachable"
+
+                spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
+                merged_result, kb_status_message, kb_requested_count, kb_returned_count = merge_results(
+                    scancode_result, scanoss_result, spdx_downloads,
+                    path_to_scan, run_kb, manifest_licenses, excluded_files,
+                    hide_progress, kb_url, kb_token,
                 )
-            mark_oss_info_correction_files_as_excluded(merged_result)
-            scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
-                                           print_matched_text, output_path, output_files, output_extensions, correct_mode,
-                                           correct_filepath, path_to_scan, excluded_path_without_dot, formats,
-                                           api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
+                if kb_status_message:
+                    run_kb_msg = f"KB({kb_url}) {kb_status_message}"
+                elif run_kb and kb_requested_count > 0:
+                    run_kb_msg = (
+                        f"KB({kb_url}) response : {kb_returned_count}/"
+                        f" requested: {kb_requested_count}"
+                    )
+                mark_oss_info_correction_files_as_excluded(merged_result)
+                scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
+                                               print_matched_text, output_path, output_files, output_extensions, correct_mode,
+                                               correct_filepath, path_to_scan, excluded_path_without_dot, formats,
+                                               api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
+            else:
+                print_help_msg_source_scanner()
+                result_log[RESULT_KEY] = "Unsupported scanner"
+                success = False
         else:
-            print_help_msg_source_scanner()
-            result_log[RESULT_KEY] = "Unsupported scanner"
+            result_log[RESULT_KEY] = f"Format error. {msg}"
             success = False
-    else:
-        result_log[RESULT_KEY] = f"Format error. {msg}"
-        success = False
 
-    try:
-        shutil.copytree(output_path, final_output_path, dirs_exist_ok=True)
-        shutil.rmtree(output_path)
-    except Exception as ex:
-        logger.debug(f"Failed to move temp files: {ex}")
+        publish_temp_output = True
+    finally:
+        publish_ok = _finalize_temp_output(output_path, final_output_path, publish_temp_output, logger)
+
+    if publish_temp_output and not publish_ok:
+        success = False
+        prev_msg = result_log.get(RESULT_KEY, "")
+        result_log[RESULT_KEY] = (
+            f"{prev_msg}, Failed to publish scan artifacts" if prev_msg
+            else "Failed to publish scan artifacts"
+        )
 
     return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
 

{fosslight_source-2.3.0 → fosslight_source-2.3.1}/src/fosslight_source/run_scancode.py

@@ -63,14 +63,23 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None:
         logger.debug("scancode UNSET workaround skipped: %s", ex)
 
 
+def _directory_ignore_pattern(dir_name: str) -> str:
+    """Path-based glob for a directory name (avoids matching the scan root itself)."""
+    normalized = dir_name.strip().strip("/").replace("\\", "/")
+    if not normalized:
+        return dir_name
+    return f"**/{normalized}/**"
+
+
 def _default_scancode_coarse_ignore_patterns() -> frozenset:
     """
     Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules.
-    Uses segment-style globs so scancode does not need one pattern per file.
+    Directory names use path-based globs (e.g. **/tests/**) so they do not match
+    the scan root directory name itself.
     """
     patterns = {".*"}
     for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY:
-        patterns.add(name)
+        patterns.add(_directory_ignore_pattern(name))
     for ext in EXCLUDE_FILE_EXTENSION:
         patterns.add(f"*.{ext}")
     for name in EXCLUDE_FILENAME:

{fosslight_source-2.3.0 → fosslight_source-2.3.1/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.0
+Version: 2.3.1
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0

{fosslight_source-2.3.0 → fosslight_source-2.3.1}/tests/test_tox.py

@@ -3,10 +3,13 @@
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
+import shlex
 import subprocess
 import pytest
 import shutil
 import sys
+import csv
+import glob
 
 # Add project root to sys.path for importing FL Source modules
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
@@ -18,6 +21,26 @@ from fosslight_source._parsing_scancode_file_item import (
 )
 
 remove_directories = ["test_scan", "test_scan2", "test_scan3"]
+TEST_FILES_SCAN_DIR = "test_scan"
+
+
+def _parse_license_tokens(license_value: str) -> set[str]:
+    return {token.strip().lower() for token in (license_value or "").split(",") if token.strip()}
+
+
+def _read_src_csv_rows(csv_path: str) -> list[dict]:
+    with open(csv_path, "r", encoding="utf-8") as file:
+        return list(csv.DictReader(file, delimiter="\t"))
+
+
+def _rows_for_source(rows: list[dict], source_name: str) -> list[dict]:
+    return [row for row in rows if row.get("Source Path") == source_name]
+
+
+def _find_scan_csv(output_dir: str) -> str:
+    csv_files = sorted(glob.glob(os.path.join(output_dir, "*.csv")))
+    assert csv_files, f"No CSV report found under {output_dir}"
+    return csv_files[-1]
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -31,8 +54,22 @@ def setup_test_result_dir():
 
 
 def run_command(command):
-    process = subprocess.run(command, shell=True, capture_output=True, text=True)
-    success = (process.returncode == 0)
+    command = command.strip()
+    if command.startswith("fosslight_source"):
+        args = shlex.split(command, posix=(os.name != "nt"))[1:]
+        if os.environ.get("FOSSLIGHT_USE_LOCAL_SRC"):
+            src_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src"))
+            env = os.environ.copy()
+            existing = env.get("PYTHONPATH", "")
+            env["PYTHONPATH"] = src_path if not existing else f"{src_path}{os.pathsep}{existing}"
+            cmd = [sys.executable, "-m", "fosslight_source.cli", *args]
+            process = subprocess.run(cmd, capture_output=True, text=True, env=env)
+        else:
+            cmd = ["fosslight_source", *args]
+            process = subprocess.run(cmd, capture_output=True, text=True)
+    else:
+        process = subprocess.run(command, shell=True, capture_output=True, text=True)
+    success = process.returncode == 0
     return success, process.stdout if success else process.stderr
 
 
@@ -112,6 +149,32 @@ def test_run():
     assert len(scan2_files) > 0, "Test Run: No scan files created in test_scan2 directory"
 
 
+def test_test_files_scan_results():
+    os.makedirs(TEST_FILES_SCAN_DIR, exist_ok=True)
+
+    success, msg = run_command(
+        f"fosslight_source -p tests/test_files -s scancode -f csv -o {TEST_FILES_SCAN_DIR}/"
+    )
+    assert success is True, f"Test Run: test_files scan failed: {msg}"
+
+    csv_path = _find_scan_csv(TEST_FILES_SCAN_DIR)
+    rows = _read_src_csv_rows(csv_path)
+
+    sample_rows = _rows_for_source(rows, "sample.cpp")
+    assert sample_rows, "Test Run: sample.cpp not found in scan result"
+    for row in sample_rows:
+        licenses = _parse_license_tokens(row.get("License", ""))
+        assert "apache-2.0" in licenses, f"sample.cpp missing Apache-2.0 license: {row.get('License')}"
+        assert "mit" in licenses, f"sample.cpp missing MIT license: {row.get('License')}"
+
+    temp_rows = _rows_for_source(rows, "temp.cpp")
+    assert temp_rows, "Test Run: temp.cpp not found in scan result"
+    temp_row = temp_rows[0]
+    temp_licenses = _parse_license_tokens(temp_row.get("License", ""))
+    assert "apache-2.0" in temp_licenses, f"temp.cpp missing Apache-2.0 license: {temp_row.get('License')}"
+    assert (temp_row.get("Copyright Text") or "").strip(), "Test Run: temp.cpp copyright not extracted"
+
+
 def test_help_command():
     success, msg = run_command("fosslight_source -h")
     assert success is True, f"Test Release: Help command failed :{msg}"