fosslight-source 2.1.17__py3-none-any.whl → 2.1.18__py3-none-any.whl

fosslight_source/_help.py

@@ -19,7 +19,9 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
             -h\t\t\t   Print help message
             -v\t\t\t   Print FOSSLight Source Scanner version
             -m\t\t\t   Print additional information for scan result on separate sheets
-            -e <path>\t\t   Path to exclude from analysis (file and directory)
+            -e <path>\t\t   Path to exclude from analysis (files and directories)
+            \t\t\t   * IMPORTANT: Always wrap patterns in double quotes ("") to avoid shell expansion.
+            \t\t\t     Example) fosslight_source -e "dev/" "tests/"
             -o <output_path>\t   Output path (Path or file name)
             -f <format>\t\t   Output file formats
             \t\t\t   ({', '.join(SUPPORT_FORMAT)})

fosslight_source/_parsing_scancode_file_item.py

@@ -19,10 +19,6 @@ from ._scan_item import is_package_dir
 from typing import Tuple
 
 logger = logging.getLogger(constant.LOGGER_NAME)
-_exclude_directory = ["test", "tests", "doc", "docs"]
-_exclude_directory = [os.path.sep + dir_name +
-                      os.path.sep for dir_name in _exclude_directory]
-_exclude_directory.append("/.")
 REMOVE_LICENSE = ["warranty-disclaimer"]
 regex = re.compile(r'licenseref-(\S+)', re.IGNORECASE)
 find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)

fosslight_source/_scan_item.py

@@ -108,7 +108,7 @@ class SourceItem(FileItem):
 
 
 def is_exclude_dir(dir_path: str) -> bool:
-    if dir_path != "":
+    if dir_path:
         dir_path = dir_path.lower()
         dir_path = dir_path if dir_path.endswith(
             os.path.sep) else dir_path + os.path.sep
@@ -163,3 +163,21 @@ def is_package_dir(dir_path: str) -> bool:
             pkg_path = '/'.join(path_parts[:pkg_index + 1])
             return True, pkg_path
     return False, ""
+
+
+def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = []) -> list:
+    path_to_exclude = custom_excluded_paths.copy()
+    abs_path_to_scan = os.path.abspath(path_to_scan)
+
+    for root, dirs, files in os.walk(path_to_scan):
+        for dir_name in dirs:
+            dir_path = os.path.join(root, dir_name)
+            rel_path = os.path.relpath(dir_path, abs_path_to_scan)
+            if dir_name in _package_directory:
+                if rel_path not in path_to_exclude:
+                    path_to_exclude.append(rel_path)
+            elif is_exclude_dir(rel_path):
+                if rel_path not in path_to_exclude:
+                    path_to_exclude.append(rel_path)
+
+    return path_to_exclude

fosslight_source/cli.py

@@ -18,6 +18,7 @@ from ._license_matched import get_license_list_to_print
 from fosslight_util.output_format import check_output_formats_v2, write_output_file
 from fosslight_util.correct import correct_with_yaml
 from .run_scancode import run_scan
+from ._scan_item import get_excluded_paths
 from .run_scanoss import run_scanoss_py
 from .run_scanoss import get_scanoss_extra_info
 import yaml
@@ -206,10 +207,8 @@ def create_report_file(
 
     scan_item = ScannerItem(PKG_NAME, _start_time)
     scan_item.set_cover_pathinfo(path_to_scan, path_to_exclude)
-    files_count, removed_files_count = count_files(path_to_scan, path_to_exclude)
-
-    scan_item.set_cover_comment(f"Total number of files : {files_count}")
-    scan_item.set_cover_comment(f"Removed files : {removed_files_count}")
+    files_count, _ = count_files(path_to_scan, path_to_exclude)
+    scan_item.set_cover_comment(f"Scanned files: {files_count}")
 
     if api_limit_exceed:
         scan_item.set_cover_comment("(Some of) SCANOSS scan was skipped. (API limits being exceeded)")
@@ -347,16 +346,18 @@ def run_scanners(
         print_matched_text = False
 
     if success:
+        excluded_path_with_default_exclusion = get_excluded_paths(path_to_scan, path_to_exclude)
         if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '':
             success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
                                                                                       write_json_file, num_cores, True,
                                                                                       print_matched_text, formats, called_by_cli,
-                                                                                      time_out, correct_mode, correct_filepath)
+                                                                                      time_out, correct_mode, correct_filepath,
+                                                                                      excluded_path_with_default_exclusion)
         if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
             scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file,
-                                                              num_cores, path_to_exclude)
+                                                              num_cores, excluded_path_with_default_exclusion)
         if selected_scanner in SCANNER_TYPE:
-            spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude)
+            spdx_downloads = get_spdx_downloads(path_to_scan, excluded_path_with_default_exclusion)
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,

fosslight_source/run_scancode.py

@@ -45,7 +45,6 @@ def run_scan(
 
     if not correct_filepath:
         correct_filepath = path_to_scan
-
     success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats)
     if success:
         if output_path == "":  # if json output with _write_json_file not used, output_path won't be needed.
@@ -75,36 +74,66 @@ def run_scan(
         if os.path.isdir(path_to_scan):
             try:
                 time_out = float(time_out)
+                logger.debug(f"Skipped by Scancode: {path_to_exclude}")
                 pretty_params = {}
                 pretty_params["path_to_scan"] = path_to_scan
                 pretty_params["path_to_exclude"] = path_to_exclude
                 pretty_params["output_file"] = output_file_name
                 total_files_to_excluded = []
-
                 if path_to_exclude:
-                    target_path = os.path.basename(path_to_scan) if os.path.isabs(path_to_scan) else path_to_scan
-
+                    abs_path_to_scan = os.path.abspath(path_to_scan)
                     for path in path_to_exclude:
-                        exclude_path = path
-                        isabs_exclude = os.path.isabs(path)
-                        if isabs_exclude:
-                            exclude_path = os.path.relpath(path, os.path.abspath(path_to_scan))
-
-                        exclude_path = os.path.join(target_path, exclude_path)
-                        if os.path.isdir(exclude_path):
-                            for root, _, files in os.walk(exclude_path):
-                                total_files_to_excluded.extend([os.path.normpath(os.path.join(root, file)).replace("\\", "/")
-                                                                for file in files])
-                        elif os.path.isfile(exclude_path):
-                            total_files_to_excluded.append(os.path.normpath(exclude_path).replace("\\", "/"))
-
+                        if os.path.isabs(path):
+                            exclude_path = os.path.relpath(path, abs_path_to_scan)
+                        else:
+                            exclude_path = path
+
+                        exclude_path_normalized = os.path.normpath(exclude_path).replace("\\", "/")
+
+                        if exclude_path_normalized.endswith("/**"):
+                            exclude_path_normalized = exclude_path_normalized[:-3]
+                        elif exclude_path_normalized.endswith("**"):
+                            exclude_path_normalized = exclude_path_normalized.rstrip("*")
+
+                        if exclude_path_normalized.startswith("**/"):
+                            exclude_path_normalized = exclude_path_normalized[3:]
+
+                        full_exclude_path = os.path.join(abs_path_to_scan, exclude_path)
+                        is_dir = os.path.isdir(full_exclude_path)
+                        is_file = os.path.isfile(full_exclude_path)
+                        if is_dir:
+                            dir_name = os.path.basename(exclude_path_normalized.rstrip("/"))
+                            base_path = exclude_path_normalized.rstrip("/")
+
+                            if dir_name:
+                                total_files_to_excluded.append(dir_name)
+                                max_depth = 0
+                                for root, dirs, files in os.walk(full_exclude_path):
+                                    depth = root[len(full_exclude_path):].count(os.sep)
+                                    max_depth = max(max_depth, depth)
+                                for depth in range(1, max_depth + 2):
+                                    pattern = base_path + "/*" * depth
+                                    total_files_to_excluded.append(pattern)
+                            else:
+                                total_files_to_excluded.append(exclude_path_normalized)
+                        elif is_file:
+                            total_files_to_excluded.append(exclude_path_normalized)
+                        else:
+                            if "/" in exclude_path_normalized:
+                                dir_name = os.path.basename(exclude_path_normalized.rstrip("/"))
+                                if dir_name:
+                                    total_files_to_excluded.append(dir_name)
+                            total_files_to_excluded.append(exclude_path_normalized)
+
+                total_files_to_excluded = sorted(list(set(total_files_to_excluded)))
+                ignore_tuple = tuple(total_files_to_excluded)
                 rc, results = cli.run_scan(path_to_scan, max_depth=100,
                                            strip_root=True, license=True,
                                            copyright=True, return_results=True,
                                            processes=num_cores, pretty_params=pretty_params,
                                            output_json_pp=output_json_file, only_findings=True,
                                            license_text=True, url=True, timeout=time_out,
-                                           include=(), ignore=tuple(total_files_to_excluded))
+                                           include=(), ignore=ignore_tuple)
                 if not rc:
                     msg = "Source code analysis failed."
                     success = False

{fosslight_source-2.1.17.dist-info → fosslight_source-2.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.1.17
+Version: 2.1.18
 Summary: FOSSLight Source Scanner
 Home-page: https://github.com/fosslight/fosslight_source_scanner
 Download-URL: https://github.com/fosslight/fosslight_source_scanner

fosslight_source-2.1.18.dist-info/RECORD

@@ -0,0 +1,16 @@
+fosslight_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fosslight_source/_help.py,sha256=QuoQvxBPV00IfuD2ft88uRQXMSdrL2rJB7CQr05w3Ng,2312
+fosslight_source/_license_matched.py,sha256=-3H881XQjFDafRttBsuboS3VbCPYEvPH1pwWXptknE4,2164
+fosslight_source/_parsing_scancode_file_item.py,sha256=VZf_-5f7DZi8Zkj9Bx9LETTxcW-9f0KyNQD_DVOUNes,20024
+fosslight_source/_parsing_scanoss_file.py,sha256=0f5JzjnFU-kcPZRX7OKnextyvANjKwwNZeyCJVC7eME,4624
+fosslight_source/_scan_item.py,sha256=5HWJ8j58snEjTqzYtKRB8RVfywVrzivkJQ6WMh7nBwA,7299
+fosslight_source/cli.py,sha256=ApdTDaLEuH1LskLtcMrLyeRDIgIJUZlOp3RrWaju2Pc,16854
+fosslight_source/run_scancode.py,sha256=h8HWoZr5R17kXCYjiR56ZTdpDwpFKPAurpUpjTvT424,9006
+fosslight_source/run_scanoss.py,sha256=8wu3sa-YBqjfb5x2dbDJuAdw3rrExueOW23WdzqDCaU,5721
+fosslight_source/run_spdx_extractor.py,sha256=Hr9sTv06cJaVITy8amwexIW2FV8_rUcFw6hKmR9ZYws,1990
+fosslight_source-2.1.18.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_source-2.1.18.dist-info/METADATA,sha256=Iks42aiZ_NvP3ip3923gmbO5k_tZr9r0Tt0W8FZBwV0,3558
+fosslight_source-2.1.18.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fosslight_source-2.1.18.dist-info/entry_points.txt,sha256=G4bBRWqSrJ68g-2M-JtNDrSZsdym_M7_KohQ2qR1vG8,113
+fosslight_source-2.1.18.dist-info/top_level.txt,sha256=C2vw-0OIent84Vq-UEk1gt_kK1EL8dIItzBzp3WNyA4,17
+fosslight_source-2.1.18.dist-info/RECORD,,

fosslight_source-2.1.17.dist-info/RECORD

@@ -1,16 +0,0 @@
-fosslight_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_source/_help.py,sha256=UEfOI6Jx38cDLN4CZ8TTk3u8TL2r0aG2V9IZKa-l3aI,2138
-fosslight_source/_license_matched.py,sha256=-3H881XQjFDafRttBsuboS3VbCPYEvPH1pwWXptknE4,2164
-fosslight_source/_parsing_scancode_file_item.py,sha256=DA2tEbjCHXFLfavCh0TjRIF1-dE4Ep7X2ivDF8IJqS8,20227
-fosslight_source/_parsing_scanoss_file.py,sha256=0f5JzjnFU-kcPZRX7OKnextyvANjKwwNZeyCJVC7eME,4624
-fosslight_source/_scan_item.py,sha256=9bm1kOeBudIb2M8wmmRKAzOFdkdBTUtoyO2LQKaJeDQ,6584
-fosslight_source/cli.py,sha256=2TuHZvDKUp8R12DM9gesTF23RT7OBw968AdzNLtVanU,16650
-fosslight_source/run_scancode.py,sha256=YSzLoS4p-Kge91uQpI4483ZfiapF-3umgJHggxKtiuU,7220
-fosslight_source/run_scanoss.py,sha256=8wu3sa-YBqjfb5x2dbDJuAdw3rrExueOW23WdzqDCaU,5721
-fosslight_source/run_spdx_extractor.py,sha256=Hr9sTv06cJaVITy8amwexIW2FV8_rUcFw6hKmR9ZYws,1990
-fosslight_source-2.1.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_source-2.1.17.dist-info/METADATA,sha256=8vCF2Vq_rTqWgf9DOizhRBPyhNhVsS9S6GLT9RtbtPw,3558
-fosslight_source-2.1.17.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-fosslight_source-2.1.17.dist-info/entry_points.txt,sha256=G4bBRWqSrJ68g-2M-JtNDrSZsdym_M7_KohQ2qR1vG8,113
-fosslight_source-2.1.17.dist-info/top_level.txt,sha256=C2vw-0OIent84Vq-UEk1gt_kK1EL8dIItzBzp3WNyA4,17
-fosslight_source-2.1.17.dist-info/RECORD,,