fosslight-source 2.3.11__tar.gz → 2.3.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_source-2.3.11/src/fosslight_source.egg-info → fosslight_source-2.3.13}/PKG-INFO +3 -3
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/pyproject.toml +3 -3
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_parsing_scancode_file_item.py +330 -88
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/cli.py +20 -11
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/run_manifest_extractor.py +20 -3
- {fosslight_source-2.3.11 → fosslight_source-2.3.13/src/fosslight_source.egg-info}/PKG-INFO +3 -3
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source.egg-info/SOURCES.txt +1 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source.egg-info/requires.txt +2 -2
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_manifest_android_bp.py +5 -5
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_manifest_recommended_scenarios.py +24 -7
- fosslight_source-2.3.13/tests/test_manifest_setup_cfg.py +41 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_parsing_unknown_spdx.py +353 -5
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/LICENSE +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/MANIFEST.in +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/README.md +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/setup.cfg +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/__init__.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_exclude.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_help.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_kb_client.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_license_matched.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_merge.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_parsing_scanoss_file.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_scan_item.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/_scancode_ignore_binaries.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/run_scancode.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/run_scanoss.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source/run_spdx_extractor.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source.egg-info/dependency_links.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source.egg-info/entry_points.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/src/fosslight_source.egg-info/top_level.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_kb_ssl.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_manifest_composer.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_manifest_pyproject.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_multi_value_order.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.13}/tests/test_tox.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_source
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.13
|
|
4
4
|
Summary: FOSSLight Source Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -20,11 +20,11 @@ Requires-Dist: setuptools<=80.10.2
|
|
|
20
20
|
Requires-Dist: pyparsing
|
|
21
21
|
Requires-Dist: scanoss>=1.45.0
|
|
22
22
|
Requires-Dist: XlsxWriter
|
|
23
|
-
Requires-Dist: fosslight_util>=2.2.
|
|
23
|
+
Requires-Dist: fosslight_util>=2.2.14
|
|
24
24
|
Requires-Dist: PyYAML
|
|
25
25
|
Requires-Dist: wheel>=0.38.1
|
|
26
26
|
Requires-Dist: intbitset
|
|
27
|
-
Requires-Dist: scancode-toolkit>=32.0
|
|
27
|
+
Requires-Dist: scancode-toolkit>=32.5.0
|
|
28
28
|
Requires-Dist: lxml>=6.0.1
|
|
29
29
|
Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"
|
|
30
30
|
Requires-Dist: fingerprints==1.2.3
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "fosslight_source"
|
|
10
|
-
version = "2.3.
|
|
10
|
+
version = "2.3.13"
|
|
11
11
|
description = "FOSSLight Source Scanner"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = "Apache-2.0"
|
|
@@ -29,11 +29,11 @@ dependencies = [
|
|
|
29
29
|
"pyparsing",
|
|
30
30
|
"scanoss>=1.45.0",
|
|
31
31
|
"XlsxWriter",
|
|
32
|
-
"fosslight_util>=2.2.
|
|
32
|
+
"fosslight_util>=2.2.14",
|
|
33
33
|
"PyYAML",
|
|
34
34
|
"wheel>=0.38.1",
|
|
35
35
|
"intbitset",
|
|
36
|
-
"scancode-toolkit>=32.0
|
|
36
|
+
"scancode-toolkit>=32.5.0",
|
|
37
37
|
"lxml>=6.0.1",
|
|
38
38
|
# cryptography 49.x does not provide macOS x86_64 wheels, causing source builds to require OpenSSL/pkg-config.
|
|
39
39
|
"cryptography<49; platform_system == 'Darwin' and platform_machine == 'x86_64'",
|
|
@@ -19,7 +19,10 @@ logger = logging.getLogger(constant.LOGGER_NAME)
|
|
|
19
19
|
REMOVE_LICENSE = ["warranty-disclaimer"]
|
|
20
20
|
find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
|
|
21
21
|
SPDX_LICENSE_IDENTIFIER_PATTERN = re.compile(
|
|
22
|
-
|
|
22
|
+
# Require : , or - after Identifier. Bare whitespace is rejected so that
|
|
23
|
+
# messages like 'Misplaced SPDX-License-Identifier tag - use line ...'
|
|
24
|
+
# are not treated as license declarations.
|
|
25
|
+
r'SPDX[-\s]+License[-\s]+Identifier\s*[:,-]\s*([^\r\n]+)',
|
|
23
26
|
re.IGNORECASE,
|
|
24
27
|
)
|
|
25
28
|
# Android Soong license_kinds string, e.g. "SPDX-license-identifier-BSD"
|
|
@@ -28,6 +31,7 @@ SOONG_SPDX_LICENSE_KIND_PATTERN = re.compile(
|
|
|
28
31
|
re.IGNORECASE,
|
|
29
32
|
)
|
|
30
33
|
LICENSE_REF_PREFIX_PATTERN = re.compile(r'^LicenseRef-', re.IGNORECASE)
|
|
34
|
+
LICENSE_REF_SCANCODE_PREFIX_PATTERN = re.compile(r'LicenseRef-scancode-', re.IGNORECASE)
|
|
31
35
|
# Trailing closers from comments and quoted lists, e.g. "MIT */", "MIT -->", or '"MIT",'.
|
|
32
36
|
SPDX_DECLARATION_TRAILER_PATTERN = re.compile(r'\s*(?:\*/|-->|["\']\s*,?)\s*$')
|
|
33
37
|
SPDX_DECLARATION_LINE_COMMENT_PATTERN = re.compile(r'\s//.*$')
|
|
@@ -39,6 +43,7 @@ HTTP_URL_PATTERN = re.compile(r'https?://', re.IGNORECASE)
|
|
|
39
43
|
SPDX_REPLACE_WORDS = ["(", ")"]
|
|
40
44
|
KEY_AND_OR = re.compile(r"(?<=\s)(?:and|or)(?=\s)", re.IGNORECASE)
|
|
41
45
|
KEY_AND_OR_CAPTURE = re.compile(r"(?<=\s)(and|or)(?=\s)", re.IGNORECASE)
|
|
46
|
+
DETECTED_COMMENT_PREFIX = "Detected: "
|
|
42
47
|
# GPL, LGPL, AGPL, GFDL
|
|
43
48
|
GPL_LICENSE_PATTERN = re.compile(r'((a|l)?gpl|gfdl)', re.IGNORECASE)
|
|
44
49
|
FSF_IN_COPYRIGHT = "free software foundation"
|
|
@@ -86,6 +91,59 @@ def _file_has_other_license(matches: list) -> bool:
|
|
|
86
91
|
return False
|
|
87
92
|
|
|
88
93
|
|
|
94
|
+
def _matched_text_has_spdx_license_identifier(matched_txt: str) -> bool:
|
|
95
|
+
"""True for classic SPDX-License-Identifier with a colon. Soong dash form does not count."""
|
|
96
|
+
return bool(
|
|
97
|
+
re.search(
|
|
98
|
+
r'SPDX[-\s]+License[-\s]+Identifier\s*:\s*\S',
|
|
99
|
+
matched_txt or "",
|
|
100
|
+
flags=re.IGNORECASE,
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _is_spdx_declaration_line(matched_txt: str) -> bool:
|
|
106
|
+
"""
|
|
107
|
+
True when matched_text is (mostly) an SPDX declaration line, not code that
|
|
108
|
+
merely mentions SPDX-License-Identifier (e.g. sed replacements).
|
|
109
|
+
"""
|
|
110
|
+
if not _extract_spdx_declared_expression(matched_txt):
|
|
111
|
+
return False
|
|
112
|
+
stripped = re.sub(r"""^[\s"'#/\*-]+""", "", (matched_txt or "").strip())
|
|
113
|
+
return bool(
|
|
114
|
+
re.match(
|
|
115
|
+
r"SPDX[-\s]+License[-\s]+Identifier\s*[:,-]",
|
|
116
|
+
stripped,
|
|
117
|
+
flags=re.IGNORECASE,
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _file_has_spdx_license_identifier(matches: list) -> bool:
|
|
123
|
+
"""True if any match is a classic SPDX-License-Identifier declaration line (colon form)."""
|
|
124
|
+
return any(
|
|
125
|
+
_is_spdx_declaration_line(m.get("matched_text") or "")
|
|
126
|
+
and _matched_text_has_spdx_license_identifier(m.get("matched_text") or "")
|
|
127
|
+
for m in (matches or [])
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _file_has_spdx_declared_license(matches: list) -> bool:
|
|
132
|
+
"""True if any match has a recoverable SPDX declaration (incl. Soong kinds)."""
|
|
133
|
+
for matched_lic in matches or []:
|
|
134
|
+
if _extract_spdx_declared_expression(matched_lic.get("matched_text") or ""):
|
|
135
|
+
return True
|
|
136
|
+
return False
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _filter_spdx_declaration_matches(matches: list) -> list:
|
|
140
|
+
"""Keep only matches that are SPDX declaration lines (not code mentions)."""
|
|
141
|
+
return [
|
|
142
|
+
matched_lic for matched_lic in (matches or [])
|
|
143
|
+
if _is_spdx_declaration_line(matched_lic.get("matched_text") or "")
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
|
|
89
147
|
def _matched_text_has_http_url(matched_text: str) -> bool:
|
|
90
148
|
"""Return whether matched text contains an HTTP or HTTPS URL."""
|
|
91
149
|
return bool(HTTP_URL_PATTERN.search(matched_text or ""))
|
|
@@ -341,6 +399,97 @@ def _strip_license_ref_prefix(token: str) -> str:
|
|
|
341
399
|
return LICENSE_REF_PREFIX_PATTERN.sub('', (token or "").strip())
|
|
342
400
|
|
|
343
401
|
|
|
402
|
+
def _display_expression_for_detected_comment(match: dict) -> str:
|
|
403
|
+
"""
|
|
404
|
+
Prefer license_expression_spdx; strip LicenseRef-scancode- for display.
|
|
405
|
+
Falls back to license_expression when SPDX form is absent.
|
|
406
|
+
"""
|
|
407
|
+
spdx = (match.get("license_expression_spdx") or "").strip()
|
|
408
|
+
expr = (match.get("license_expression") or "").strip()
|
|
409
|
+
value = spdx or expr
|
|
410
|
+
if not value:
|
|
411
|
+
return ""
|
|
412
|
+
return LICENSE_REF_SCANCODE_PREFIX_PATTERN.sub("", value).strip()
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _dedupe_detected_comment_expressions(expressions: list[str]) -> list[str]:
|
|
416
|
+
"""
|
|
417
|
+
Drop exact duplicates, then drop a value if it appears as an AND/OR token
|
|
418
|
+
of another remaining value (e.g. BSD-2-Clause covered by
|
|
419
|
+
``GPL-2.0-only OR BSD-2-Clause``).
|
|
420
|
+
"""
|
|
421
|
+
unique: list[str] = []
|
|
422
|
+
for expr in expressions:
|
|
423
|
+
expr = (expr or "").strip()
|
|
424
|
+
if not expr:
|
|
425
|
+
continue
|
|
426
|
+
if any(expr.lower() == kept.lower() for kept in unique):
|
|
427
|
+
continue
|
|
428
|
+
unique.append(expr)
|
|
429
|
+
|
|
430
|
+
kept: list[str] = []
|
|
431
|
+
for idx, candidate in enumerate(unique):
|
|
432
|
+
others = [unique[j] for j in range(len(unique)) if j != idx]
|
|
433
|
+
candidate_lower = candidate.lower()
|
|
434
|
+
covered = False
|
|
435
|
+
for other in others:
|
|
436
|
+
other_tokens = [
|
|
437
|
+
token.lower()
|
|
438
|
+
for token in split_spdx_expression(other)
|
|
439
|
+
if token and token.strip()
|
|
440
|
+
]
|
|
441
|
+
if candidate_lower in other_tokens:
|
|
442
|
+
covered = True
|
|
443
|
+
break
|
|
444
|
+
if not covered:
|
|
445
|
+
kept.append(candidate)
|
|
446
|
+
return kept
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _expression_covered_by_reported_licenses(
|
|
450
|
+
expression: str, reported_licenses: list
|
|
451
|
+
) -> bool:
|
|
452
|
+
"""
|
|
453
|
+
True when expression is already represented in License column values.
|
|
454
|
+
|
|
455
|
+
Matches the whole expression, or every AND/OR token, case-insensitively.
|
|
456
|
+
"""
|
|
457
|
+
reported = {
|
|
458
|
+
(lic or "").strip().lower()
|
|
459
|
+
for lic in (reported_licenses or [])
|
|
460
|
+
if (lic or "").strip()
|
|
461
|
+
}
|
|
462
|
+
if not expression or not reported:
|
|
463
|
+
return False
|
|
464
|
+
expr = expression.strip()
|
|
465
|
+
if expr.lower() in reported:
|
|
466
|
+
return True
|
|
467
|
+
tokens = [
|
|
468
|
+
token.strip().lower()
|
|
469
|
+
for token in split_spdx_expression(expr)
|
|
470
|
+
if token and token.strip()
|
|
471
|
+
]
|
|
472
|
+
return bool(tokens) and all(token in reported for token in tokens)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def build_detected_comment_from_dropped_matches(
|
|
476
|
+
dropped_matches: list, reported_licenses: list | None = None
|
|
477
|
+
) -> str:
|
|
478
|
+
"""Build ``Detected: a, b OR c`` from matches dropped by SPDX priority."""
|
|
479
|
+
displays = []
|
|
480
|
+
for match in (dropped_matches or []):
|
|
481
|
+
display = _display_expression_for_detected_comment(match)
|
|
482
|
+
if not display:
|
|
483
|
+
continue
|
|
484
|
+
if _expression_covered_by_reported_licenses(display, reported_licenses or []):
|
|
485
|
+
continue
|
|
486
|
+
displays.append(display)
|
|
487
|
+
deduped = _dedupe_detected_comment_expressions(displays)
|
|
488
|
+
if not deduped:
|
|
489
|
+
return ""
|
|
490
|
+
return DETECTED_COMMENT_PREFIX + ", ".join(deduped)
|
|
491
|
+
|
|
492
|
+
|
|
344
493
|
def _extract_spdx_declared_expression(matched_txt: str) -> str:
|
|
345
494
|
"""
|
|
346
495
|
Extract SPDX-License-Identifier value from matched_text.
|
|
@@ -385,59 +534,112 @@ def _declared_licenses_from_matched_text(matched_txt: str) -> tuple[list[str], l
|
|
|
385
534
|
return split_spdx_expression_with_ops(declared)
|
|
386
535
|
|
|
387
536
|
|
|
388
|
-
def
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
if KEYWORD_SCANCODE_UNKNOWN not in expr:
|
|
394
|
-
continue
|
|
395
|
-
match_tokens, _ = split_spdx_expression_with_ops(expr)
|
|
396
|
-
declared_tokens, declared_ops = _declared_licenses_from_matched_text(
|
|
397
|
-
matched_lic.get("matched_text") or ""
|
|
398
|
-
)
|
|
399
|
-
if not declared_tokens:
|
|
400
|
-
continue
|
|
537
|
+
def _unknown_spdx_replacements_for_match(
|
|
538
|
+
expression: str, matched_txt: str
|
|
539
|
+
) -> list[str]:
|
|
540
|
+
"""
|
|
541
|
+
Ordered replacements for each unknown-spdx token in one match expression.
|
|
401
542
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
543
|
+
Known tokens are never replaced. When token counts match, each unknown maps
|
|
544
|
+
to the declared token at the same index (e.g. gpl-2.0 OR unknown-spdx with
|
|
545
|
+
``GPL-2.0 or MIT-like`` → MIT-like only).
|
|
546
|
+
"""
|
|
547
|
+
expr = (expression or "").lower()
|
|
548
|
+
if KEYWORD_SCANCODE_UNKNOWN not in expr:
|
|
549
|
+
return []
|
|
550
|
+
match_tokens, _ = split_spdx_expression_with_ops(expr)
|
|
551
|
+
declared_tokens, declared_ops = _declared_licenses_from_matched_text(matched_txt)
|
|
552
|
+
if not declared_tokens:
|
|
553
|
+
return []
|
|
554
|
+
|
|
555
|
+
unknown_indexes = [
|
|
556
|
+
idx for idx, token in enumerate(match_tokens)
|
|
557
|
+
if KEYWORD_SCANCODE_UNKNOWN in token
|
|
558
|
+
]
|
|
559
|
+
if not unknown_indexes:
|
|
560
|
+
return []
|
|
561
|
+
|
|
562
|
+
queue: list[str] = []
|
|
563
|
+
if len(match_tokens) == len(declared_tokens):
|
|
564
|
+
for idx in unknown_indexes:
|
|
565
|
+
queue.append(
|
|
566
|
+
_normalize_license_token(declared_tokens[idx]) or declared_tokens[idx]
|
|
567
|
+
)
|
|
568
|
+
return queue
|
|
569
|
+
|
|
570
|
+
if all(KEYWORD_SCANCODE_UNKNOWN in token for token in match_tokens):
|
|
571
|
+
if len(match_tokens) == 1:
|
|
572
|
+
norms = [_normalize_license_token(token) or token for token in declared_tokens]
|
|
573
|
+
queue.append(join_licenses_with_ops(norms, declared_ops))
|
|
574
|
+
else:
|
|
575
|
+
for i, _ in enumerate(unknown_indexes):
|
|
576
|
+
declared = (
|
|
577
|
+
declared_tokens[i] if i < len(declared_tokens) else declared_tokens[-1]
|
|
578
|
+
)
|
|
579
|
+
queue.append(_normalize_license_token(declared) or declared)
|
|
580
|
+
return queue
|
|
581
|
+
|
|
582
|
+
# Mixed known + unknown with different declared length:
|
|
583
|
+
# drop declared tokens that correspond to known match tokens, then assign the rest.
|
|
584
|
+
unused = list(declared_tokens)
|
|
585
|
+
for token in match_tokens:
|
|
586
|
+
if KEYWORD_SCANCODE_UNKNOWN in token:
|
|
407
587
|
continue
|
|
588
|
+
token_norm = (_normalize_license_token(token) or token).lower()
|
|
589
|
+
for idx, declared in enumerate(unused):
|
|
590
|
+
declared_norm = (_normalize_license_token(declared) or declared).lower()
|
|
591
|
+
if declared_norm == token_norm or declared.lower() == token.lower():
|
|
592
|
+
unused.pop(idx)
|
|
593
|
+
break
|
|
594
|
+
for _ in unknown_indexes:
|
|
595
|
+
if not unused:
|
|
596
|
+
break
|
|
597
|
+
declared = unused.pop(0)
|
|
598
|
+
queue.append(_normalize_license_token(declared) or declared)
|
|
599
|
+
return queue
|
|
408
600
|
|
|
409
|
-
if len(match_tokens) == len(declared_tokens):
|
|
410
|
-
for idx in unknown_indexes:
|
|
411
|
-
queue.append(_normalize_license_token(declared_tokens[idx]) or declared_tokens[idx])
|
|
412
|
-
continue
|
|
413
601
|
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
queue.append(join_licenses_with_ops(norms, declared_ops))
|
|
418
|
-
else:
|
|
419
|
-
for i, _ in enumerate(unknown_indexes):
|
|
420
|
-
declared = declared_tokens[i] if i < len(declared_tokens) else declared_tokens[-1]
|
|
421
|
-
queue.append(_normalize_license_token(declared) or declared)
|
|
422
|
-
continue
|
|
602
|
+
def _resolve_unknown_spdx_in_expression(expression: str, matched_txt: str) -> str:
|
|
603
|
+
"""
|
|
604
|
+
Restore unknown-spdx in an expression from matched_text.
|
|
423
605
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
606
|
+
- All tokens unknown → use the full declared expression
|
|
607
|
+
- Mixed known + unknown → replace only unknown tokens; keep known tokens
|
|
608
|
+
- Returns empty string when unknown cannot be restored from matched_text
|
|
609
|
+
"""
|
|
610
|
+
tokens, ops = split_spdx_expression_with_ops(expression)
|
|
611
|
+
if not tokens:
|
|
612
|
+
return ""
|
|
613
|
+
unknown_indexes = [
|
|
614
|
+
idx for idx, token in enumerate(tokens)
|
|
615
|
+
if KEYWORD_SCANCODE_UNKNOWN in token.lower()
|
|
616
|
+
]
|
|
617
|
+
if not unknown_indexes:
|
|
618
|
+
return expression
|
|
619
|
+
|
|
620
|
+
if all(KEYWORD_SCANCODE_UNKNOWN in token.lower() for token in tokens):
|
|
621
|
+
return _extract_spdx_declared_expression(matched_txt)
|
|
622
|
+
|
|
623
|
+
queue = _unknown_spdx_replacements_for_match(expression, matched_txt)
|
|
624
|
+
new_tokens = list(tokens)
|
|
625
|
+
for i, idx in enumerate(unknown_indexes):
|
|
626
|
+
new_tokens[idx] = queue[i] if i < len(queue) else ""
|
|
627
|
+
kept_tokens, kept_ops = _kept_tokens_with_merged_ops(new_tokens, ops)
|
|
628
|
+
if not kept_tokens:
|
|
629
|
+
return ""
|
|
630
|
+
return join_licenses_with_ops(kept_tokens, kept_ops)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _build_unknown_spdx_replacement_queue(matches: list) -> list[str]:
|
|
634
|
+
"""Ordered replacements for each unknown-spdx token in match order."""
|
|
635
|
+
queue = []
|
|
636
|
+
for matched_lic in matches or []:
|
|
637
|
+
queue.extend(
|
|
638
|
+
_unknown_spdx_replacements_for_match(
|
|
639
|
+
matched_lic.get("license_expression") or "",
|
|
640
|
+
matched_lic.get("matched_text") or "",
|
|
641
|
+
)
|
|
642
|
+
)
|
|
441
643
|
return queue
|
|
442
644
|
|
|
443
645
|
|
|
@@ -574,53 +776,78 @@ def parsing_scancode(
|
|
|
574
776
|
all_matches = []
|
|
575
777
|
for lic in licenses or []:
|
|
576
778
|
all_matches.extend(lic.get("matches") or [])
|
|
779
|
+
# SPDX priority: if the file has SPDX-License-Identifier (:/,/-),
|
|
780
|
+
# keep only matches that are SPDX declarations (drop body rule hits).
|
|
781
|
+
prefer_spdx_declarations = _file_has_spdx_license_identifier(all_matches)
|
|
782
|
+
matches_to_process = (
|
|
783
|
+
_filter_spdx_declaration_matches(all_matches)
|
|
784
|
+
if prefer_spdx_declarations
|
|
785
|
+
else list(all_matches or [])
|
|
786
|
+
)
|
|
577
787
|
has_other_license_in_file = _file_has_other_license(all_matches)
|
|
788
|
+
has_spdx_declared_license = _file_has_spdx_declared_license(all_matches)
|
|
578
789
|
suppress_unknown_license_reference = (
|
|
579
790
|
_should_suppress_unknown_license_reference(
|
|
580
791
|
all_matches, has_other_license_in_file
|
|
581
792
|
)
|
|
582
793
|
)
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
794
|
+
# Prefer-SPDX filters matches; matched_text extraction is only for unknown-spdx.
|
|
795
|
+
# Mixed expressions keep known tokens and replace unknown tokens only.
|
|
796
|
+
for matched_lic in matches_to_process:
|
|
797
|
+
found_lic_list = matched_lic.get("license_expression", "")
|
|
798
|
+
matched_txt = matched_lic.get("matched_text", "")
|
|
799
|
+
if not found_lic_list:
|
|
800
|
+
continue
|
|
801
|
+
found_lic_list = found_lic_list.lower()
|
|
802
|
+
if KEYWORD_SCANCODE_UNKNOWN in found_lic_list:
|
|
803
|
+
resolved = _resolve_unknown_spdx_in_expression(
|
|
804
|
+
found_lic_list, matched_txt
|
|
805
|
+
)
|
|
806
|
+
if resolved:
|
|
807
|
+
found_lic_list = resolved
|
|
808
|
+
resolved_unknown_spdx = True
|
|
809
|
+
elif has_spdx_declared_license:
|
|
810
|
+
# File already has SPDX declarations; drop unrestorable
|
|
811
|
+
# unknown-spdx noise (e.g. misplaced-tag messages).
|
|
812
|
+
tokens = [
|
|
813
|
+
t for t in split_spdx_expression(found_lic_list)
|
|
814
|
+
if KEYWORD_SCANCODE_UNKNOWN not in t.lower()
|
|
815
|
+
]
|
|
816
|
+
if not tokens:
|
|
817
|
+
continue
|
|
818
|
+
found_lic_list = " AND ".join(tokens)
|
|
819
|
+
|
|
820
|
+
for found_lic in split_spdx_expression(found_lic_list):
|
|
821
|
+
if found_lic:
|
|
822
|
+
found_lic = found_lic.strip()
|
|
823
|
+
if found_lic in REMOVE_LICENSE:
|
|
824
|
+
continue
|
|
825
|
+
if (
|
|
826
|
+
KEYWORD_UNKNOWN_LICENSE_REFERENCE in found_lic.lower()
|
|
827
|
+
and not _should_keep_unknown_license_reference(
|
|
828
|
+
matched_txt, has_other_license_in_file
|
|
829
|
+
)
|
|
830
|
+
):
|
|
831
|
+
continue
|
|
832
|
+
if KEYWORD_SCANCODE_UNKNOWN in found_lic.lower():
|
|
591
833
|
declared = _extract_spdx_declared_expression(matched_txt)
|
|
592
834
|
if declared:
|
|
593
|
-
|
|
835
|
+
found_lic = declared
|
|
594
836
|
resolved_unknown_spdx = True
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
if declared:
|
|
610
|
-
found_lic = declared
|
|
611
|
-
resolved_unknown_spdx = True
|
|
612
|
-
found_lic = _strip_license_ref_prefix(found_lic)
|
|
613
|
-
found_lic = _normalize_license_token(found_lic) or found_lic
|
|
614
|
-
if not found_lic:
|
|
615
|
-
continue
|
|
616
|
-
if matched_txt:
|
|
617
|
-
lic_matched_key = found_lic + matched_txt
|
|
618
|
-
if lic_matched_key in license_list:
|
|
619
|
-
license_list[lic_matched_key].set_files(file_path)
|
|
620
|
-
else:
|
|
621
|
-
lic_info = MatchedLicense(found_lic, "", matched_txt, file_path)
|
|
622
|
-
license_list[lic_matched_key] = lic_info
|
|
623
|
-
license_detected.append(found_lic)
|
|
837
|
+
elif has_spdx_declared_license:
|
|
838
|
+
continue
|
|
839
|
+
found_lic = _strip_license_ref_prefix(found_lic)
|
|
840
|
+
found_lic = _normalize_license_token(found_lic) or found_lic
|
|
841
|
+
if not found_lic:
|
|
842
|
+
continue
|
|
843
|
+
if matched_txt:
|
|
844
|
+
lic_matched_key = found_lic + matched_txt
|
|
845
|
+
if lic_matched_key in license_list:
|
|
846
|
+
license_list[lic_matched_key].set_files(file_path)
|
|
847
|
+
else:
|
|
848
|
+
lic_info = MatchedLicense(found_lic, "", matched_txt, file_path)
|
|
849
|
+
license_list[lic_matched_key] = lic_info
|
|
850
|
+
license_detected.append(found_lic)
|
|
624
851
|
result_item.licenses = license_detected
|
|
625
852
|
file_ext = os.path.splitext(file_path)[1].lower()
|
|
626
853
|
is_source_file = file_ext and file_ext in SOURCE_EXTENSIONS
|
|
@@ -634,7 +861,22 @@ def parsing_scancode(
|
|
|
634
861
|
)
|
|
635
862
|
)
|
|
636
863
|
|
|
637
|
-
|
|
864
|
+
# Comment:
|
|
865
|
+
# - If SPDX priority dropped body matches, always record them as
|
|
866
|
+
# ``Detected: ...`` (OR unrelated).
|
|
867
|
+
# - Else, for multi-license OR expressions, rebuild from detected_*.
|
|
868
|
+
dropped_matches = []
|
|
869
|
+
if prefer_spdx_declarations:
|
|
870
|
+
process_ids = {id(match) for match in matches_to_process}
|
|
871
|
+
dropped_matches = [
|
|
872
|
+
match for match in all_matches if id(match) not in process_ids
|
|
873
|
+
]
|
|
874
|
+
detected_comment = build_detected_comment_from_dropped_matches(
|
|
875
|
+
dropped_matches, result_item.licenses
|
|
876
|
+
)
|
|
877
|
+
if detected_comment:
|
|
878
|
+
result_item.comment = detected_comment
|
|
879
|
+
if not result_item.comment and len(license_detected) > 1:
|
|
638
880
|
detected_expression = file.get("detected_license_expression", "") or ""
|
|
639
881
|
detected_expression_spdx = file.get("detected_license_expression_spdx", "") or ""
|
|
640
882
|
if (
|
|
@@ -176,15 +176,14 @@ def create_report_file(
|
|
|
176
176
|
for i, output_extension in enumerate(output_extensions):
|
|
177
177
|
if output_files[i] is None or output_files[i] == "":
|
|
178
178
|
if formats:
|
|
179
|
-
if formats[i].startswith('spdx')
|
|
179
|
+
if formats[i].startswith('spdx'):
|
|
180
180
|
if platform.system() == 'Windows':
|
|
181
181
|
logger.warning(f'{formats[i]} is not supported on Windows.Please remove {formats[i]} from format.')
|
|
182
182
|
to_remove.append(i)
|
|
183
183
|
else:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
output_files[i] = f'fosslight_cyclonedx_src_{name_time}'
|
|
184
|
+
output_files[i] = f"fosslight_spdx_src_{name_time}"
|
|
185
|
+
elif formats[i].startswith('cyclonedx'):
|
|
186
|
+
output_files[i] = f'fosslight_cyclonedx_src_{name_time}'
|
|
188
187
|
else:
|
|
189
188
|
if output_extension == _json_ext:
|
|
190
189
|
output_files[i] = f"fosslight_opossum_src_{name_time}"
|
|
@@ -276,7 +275,8 @@ def create_report_file(
|
|
|
276
275
|
# if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list:
|
|
277
276
|
# del sheet_list["scanoss_reference"]
|
|
278
277
|
result = write_output_file(
|
|
279
|
-
combined_path_and_file, output_extension, scan_item, hide_header="", format=output_format
|
|
278
|
+
combined_path_and_file, output_extension, scan_item, hide_header="", format=output_format,
|
|
279
|
+
scanner_covers=[scan_item.cover]
|
|
280
280
|
)
|
|
281
281
|
results.append(result)
|
|
282
282
|
for success, msg, result_file in results:
|
|
@@ -454,16 +454,22 @@ def merge_results(
|
|
|
454
454
|
scancode_result.append(new_result_item)
|
|
455
455
|
if manifest_licenses:
|
|
456
456
|
for file_name, licenses in manifest_licenses.items():
|
|
457
|
-
|
|
457
|
+
# None: marker-only (keep ScanCode). list (incl. []): overwrite with extracted result.
|
|
458
|
+
keep_scancode_licenses = licenses is None
|
|
459
|
+
license_list = [] if keep_scancode_licenses else licenses
|
|
460
|
+
valid_licenses = [
|
|
461
|
+
lic.strip() for lic in license_list if isinstance(lic, str) and lic.strip()
|
|
462
|
+
]
|
|
458
463
|
item = _get_or_append_source_item(
|
|
459
464
|
scancode_result, file_name, append=bool(valid_licenses) or ui_mode
|
|
460
465
|
)
|
|
461
466
|
if item is None:
|
|
462
467
|
continue
|
|
463
468
|
item.is_manifest_file = True
|
|
469
|
+
if keep_scancode_licenses:
|
|
470
|
+
continue
|
|
471
|
+
item.licenses = []
|
|
464
472
|
if valid_licenses:
|
|
465
|
-
# overwrite existing detected licenses with manifest-provided licenses
|
|
466
|
-
item.licenses = [] # clear existing licenses (setter clears when value falsy)
|
|
467
473
|
item.licenses = valid_licenses
|
|
468
474
|
|
|
469
475
|
kb_origin_urls: dict[str, str] = {}
|
|
@@ -719,7 +725,9 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di
|
|
|
719
725
|
|
|
720
726
|
- Traverse files with exclusions applied
|
|
721
727
|
- spdx_downloads: {rel_path: [download_urls]}
|
|
722
|
-
- manifest_licenses: {rel_path: [
|
|
728
|
+
- manifest_licenses: {rel_path: list[str] | None}
|
|
729
|
+
- list (incl. []): overwrite ScanCode licenses with this result
|
|
730
|
+
- None: marker-only; keep ScanCode licenses
|
|
723
731
|
|
|
724
732
|
:return: (spdx_downloads, manifest_licenses)
|
|
725
733
|
"""
|
|
@@ -739,7 +747,8 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di
|
|
|
739
747
|
spdx_downloads[rel_path_file] = downloads
|
|
740
748
|
|
|
741
749
|
if is_manifest_file(file_path):
|
|
742
|
-
|
|
750
|
+
# Preserve None (marker-only); do not coerce with `or []`.
|
|
751
|
+
manifest_licenses[rel_path_file] = get_manifest_licenses(file_path)
|
|
743
752
|
|
|
744
753
|
return spdx_downloads, manifest_licenses
|
|
745
754
|
|