fosslight-source 2.3.11__tar.gz → 2.3.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_source-2.3.11/src/fosslight_source.egg-info → fosslight_source-2.3.12}/PKG-INFO +1 -1
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/pyproject.toml +1 -1
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_parsing_scancode_file_item.py +330 -88
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/cli.py +14 -5
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_manifest_extractor.py +20 -3
- {fosslight_source-2.3.11 → fosslight_source-2.3.12/src/fosslight_source.egg-info}/PKG-INFO +1 -1
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/SOURCES.txt +1 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_manifest_android_bp.py +5 -5
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_manifest_recommended_scenarios.py +24 -7
- fosslight_source-2.3.12/tests/test_manifest_setup_cfg.py +41 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_parsing_unknown_spdx.py +353 -5
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/LICENSE +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/MANIFEST.in +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/README.md +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/setup.cfg +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/__init__.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_exclude.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_help.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_kb_client.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_license_matched.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_merge.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_parsing_scanoss_file.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_scan_item.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_scancode_ignore_binaries.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_scancode.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_scanoss.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_spdx_extractor.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/dependency_links.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/entry_points.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/requires.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/top_level.txt +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_kb_ssl.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_manifest_composer.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_manifest_pyproject.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_multi_value_order.py +0 -0
- {fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_tox.py +0 -0
|
@@ -19,7 +19,10 @@ logger = logging.getLogger(constant.LOGGER_NAME)
|
|
|
19
19
|
REMOVE_LICENSE = ["warranty-disclaimer"]
|
|
20
20
|
find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
|
|
21
21
|
SPDX_LICENSE_IDENTIFIER_PATTERN = re.compile(
|
|
22
|
-
|
|
22
|
+
# Require : , or - after Identifier. Bare whitespace is rejected so that
|
|
23
|
+
# messages like 'Misplaced SPDX-License-Identifier tag - use line ...'
|
|
24
|
+
# are not treated as license declarations.
|
|
25
|
+
r'SPDX[-\s]+License[-\s]+Identifier\s*[:,-]\s*([^\r\n]+)',
|
|
23
26
|
re.IGNORECASE,
|
|
24
27
|
)
|
|
25
28
|
# Android Soong license_kinds string, e.g. "SPDX-license-identifier-BSD"
|
|
@@ -28,6 +31,7 @@ SOONG_SPDX_LICENSE_KIND_PATTERN = re.compile(
|
|
|
28
31
|
re.IGNORECASE,
|
|
29
32
|
)
|
|
30
33
|
LICENSE_REF_PREFIX_PATTERN = re.compile(r'^LicenseRef-', re.IGNORECASE)
|
|
34
|
+
LICENSE_REF_SCANCODE_PREFIX_PATTERN = re.compile(r'LicenseRef-scancode-', re.IGNORECASE)
|
|
31
35
|
# Trailing closers from comments and quoted lists, e.g. "MIT */", "MIT -->", or '"MIT",'.
|
|
32
36
|
SPDX_DECLARATION_TRAILER_PATTERN = re.compile(r'\s*(?:\*/|-->|["\']\s*,?)\s*$')
|
|
33
37
|
SPDX_DECLARATION_LINE_COMMENT_PATTERN = re.compile(r'\s//.*$')
|
|
@@ -39,6 +43,7 @@ HTTP_URL_PATTERN = re.compile(r'https?://', re.IGNORECASE)
|
|
|
39
43
|
SPDX_REPLACE_WORDS = ["(", ")"]
|
|
40
44
|
KEY_AND_OR = re.compile(r"(?<=\s)(?:and|or)(?=\s)", re.IGNORECASE)
|
|
41
45
|
KEY_AND_OR_CAPTURE = re.compile(r"(?<=\s)(and|or)(?=\s)", re.IGNORECASE)
|
|
46
|
+
DETECTED_COMMENT_PREFIX = "Detected: "
|
|
42
47
|
# GPL, LGPL, AGPL, GFDL
|
|
43
48
|
GPL_LICENSE_PATTERN = re.compile(r'((a|l)?gpl|gfdl)', re.IGNORECASE)
|
|
44
49
|
FSF_IN_COPYRIGHT = "free software foundation"
|
|
@@ -86,6 +91,59 @@ def _file_has_other_license(matches: list) -> bool:
|
|
|
86
91
|
return False
|
|
87
92
|
|
|
88
93
|
|
|
94
|
+
def _matched_text_has_spdx_license_identifier(matched_txt: str) -> bool:
|
|
95
|
+
"""True for classic SPDX-License-Identifier with a colon. Soong dash form does not count."""
|
|
96
|
+
return bool(
|
|
97
|
+
re.search(
|
|
98
|
+
r'SPDX[-\s]+License[-\s]+Identifier\s*:\s*\S',
|
|
99
|
+
matched_txt or "",
|
|
100
|
+
flags=re.IGNORECASE,
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _is_spdx_declaration_line(matched_txt: str) -> bool:
|
|
106
|
+
"""
|
|
107
|
+
True when matched_text is (mostly) an SPDX declaration line, not code that
|
|
108
|
+
merely mentions SPDX-License-Identifier (e.g. sed replacements).
|
|
109
|
+
"""
|
|
110
|
+
if not _extract_spdx_declared_expression(matched_txt):
|
|
111
|
+
return False
|
|
112
|
+
stripped = re.sub(r"""^[\s"'#/\*-]+""", "", (matched_txt or "").strip())
|
|
113
|
+
return bool(
|
|
114
|
+
re.match(
|
|
115
|
+
r"SPDX[-\s]+License[-\s]+Identifier\s*[:,-]",
|
|
116
|
+
stripped,
|
|
117
|
+
flags=re.IGNORECASE,
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _file_has_spdx_license_identifier(matches: list) -> bool:
|
|
123
|
+
"""True if any match is a classic SPDX-License-Identifier declaration line (colon form)."""
|
|
124
|
+
return any(
|
|
125
|
+
_is_spdx_declaration_line(m.get("matched_text") or "")
|
|
126
|
+
and _matched_text_has_spdx_license_identifier(m.get("matched_text") or "")
|
|
127
|
+
for m in (matches or [])
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _file_has_spdx_declared_license(matches: list) -> bool:
|
|
132
|
+
"""True if any match has a recoverable SPDX declaration (incl. Soong kinds)."""
|
|
133
|
+
for matched_lic in matches or []:
|
|
134
|
+
if _extract_spdx_declared_expression(matched_lic.get("matched_text") or ""):
|
|
135
|
+
return True
|
|
136
|
+
return False
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _filter_spdx_declaration_matches(matches: list) -> list:
|
|
140
|
+
"""Keep only matches that are SPDX declaration lines (not code mentions)."""
|
|
141
|
+
return [
|
|
142
|
+
matched_lic for matched_lic in (matches or [])
|
|
143
|
+
if _is_spdx_declaration_line(matched_lic.get("matched_text") or "")
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
|
|
89
147
|
def _matched_text_has_http_url(matched_text: str) -> bool:
|
|
90
148
|
"""Return whether matched text contains an HTTP or HTTPS URL."""
|
|
91
149
|
return bool(HTTP_URL_PATTERN.search(matched_text or ""))
|
|
@@ -341,6 +399,97 @@ def _strip_license_ref_prefix(token: str) -> str:
|
|
|
341
399
|
return LICENSE_REF_PREFIX_PATTERN.sub('', (token or "").strip())
|
|
342
400
|
|
|
343
401
|
|
|
402
|
+
def _display_expression_for_detected_comment(match: dict) -> str:
|
|
403
|
+
"""
|
|
404
|
+
Prefer license_expression_spdx; strip LicenseRef-scancode- for display.
|
|
405
|
+
Falls back to license_expression when SPDX form is absent.
|
|
406
|
+
"""
|
|
407
|
+
spdx = (match.get("license_expression_spdx") or "").strip()
|
|
408
|
+
expr = (match.get("license_expression") or "").strip()
|
|
409
|
+
value = spdx or expr
|
|
410
|
+
if not value:
|
|
411
|
+
return ""
|
|
412
|
+
return LICENSE_REF_SCANCODE_PREFIX_PATTERN.sub("", value).strip()
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _dedupe_detected_comment_expressions(expressions: list[str]) -> list[str]:
|
|
416
|
+
"""
|
|
417
|
+
Drop exact duplicates, then drop a value if it appears as an AND/OR token
|
|
418
|
+
of another remaining value (e.g. BSD-2-Clause covered by
|
|
419
|
+
``GPL-2.0-only OR BSD-2-Clause``).
|
|
420
|
+
"""
|
|
421
|
+
unique: list[str] = []
|
|
422
|
+
for expr in expressions:
|
|
423
|
+
expr = (expr or "").strip()
|
|
424
|
+
if not expr:
|
|
425
|
+
continue
|
|
426
|
+
if any(expr.lower() == kept.lower() for kept in unique):
|
|
427
|
+
continue
|
|
428
|
+
unique.append(expr)
|
|
429
|
+
|
|
430
|
+
kept: list[str] = []
|
|
431
|
+
for idx, candidate in enumerate(unique):
|
|
432
|
+
others = [unique[j] for j in range(len(unique)) if j != idx]
|
|
433
|
+
candidate_lower = candidate.lower()
|
|
434
|
+
covered = False
|
|
435
|
+
for other in others:
|
|
436
|
+
other_tokens = [
|
|
437
|
+
token.lower()
|
|
438
|
+
for token in split_spdx_expression(other)
|
|
439
|
+
if token and token.strip()
|
|
440
|
+
]
|
|
441
|
+
if candidate_lower in other_tokens:
|
|
442
|
+
covered = True
|
|
443
|
+
break
|
|
444
|
+
if not covered:
|
|
445
|
+
kept.append(candidate)
|
|
446
|
+
return kept
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _expression_covered_by_reported_licenses(
|
|
450
|
+
expression: str, reported_licenses: list
|
|
451
|
+
) -> bool:
|
|
452
|
+
"""
|
|
453
|
+
True when expression is already represented in License column values.
|
|
454
|
+
|
|
455
|
+
Matches the whole expression, or every AND/OR token, case-insensitively.
|
|
456
|
+
"""
|
|
457
|
+
reported = {
|
|
458
|
+
(lic or "").strip().lower()
|
|
459
|
+
for lic in (reported_licenses or [])
|
|
460
|
+
if (lic or "").strip()
|
|
461
|
+
}
|
|
462
|
+
if not expression or not reported:
|
|
463
|
+
return False
|
|
464
|
+
expr = expression.strip()
|
|
465
|
+
if expr.lower() in reported:
|
|
466
|
+
return True
|
|
467
|
+
tokens = [
|
|
468
|
+
token.strip().lower()
|
|
469
|
+
for token in split_spdx_expression(expr)
|
|
470
|
+
if token and token.strip()
|
|
471
|
+
]
|
|
472
|
+
return bool(tokens) and all(token in reported for token in tokens)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def build_detected_comment_from_dropped_matches(
|
|
476
|
+
dropped_matches: list, reported_licenses: list | None = None
|
|
477
|
+
) -> str:
|
|
478
|
+
"""Build ``Detected: a, b OR c`` from matches dropped by SPDX priority."""
|
|
479
|
+
displays = []
|
|
480
|
+
for match in (dropped_matches or []):
|
|
481
|
+
display = _display_expression_for_detected_comment(match)
|
|
482
|
+
if not display:
|
|
483
|
+
continue
|
|
484
|
+
if _expression_covered_by_reported_licenses(display, reported_licenses or []):
|
|
485
|
+
continue
|
|
486
|
+
displays.append(display)
|
|
487
|
+
deduped = _dedupe_detected_comment_expressions(displays)
|
|
488
|
+
if not deduped:
|
|
489
|
+
return ""
|
|
490
|
+
return DETECTED_COMMENT_PREFIX + ", ".join(deduped)
|
|
491
|
+
|
|
492
|
+
|
|
344
493
|
def _extract_spdx_declared_expression(matched_txt: str) -> str:
|
|
345
494
|
"""
|
|
346
495
|
Extract SPDX-License-Identifier value from matched_text.
|
|
@@ -385,59 +534,112 @@ def _declared_licenses_from_matched_text(matched_txt: str) -> tuple[list[str], l
|
|
|
385
534
|
return split_spdx_expression_with_ops(declared)
|
|
386
535
|
|
|
387
536
|
|
|
388
|
-
def
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
if KEYWORD_SCANCODE_UNKNOWN not in expr:
|
|
394
|
-
continue
|
|
395
|
-
match_tokens, _ = split_spdx_expression_with_ops(expr)
|
|
396
|
-
declared_tokens, declared_ops = _declared_licenses_from_matched_text(
|
|
397
|
-
matched_lic.get("matched_text") or ""
|
|
398
|
-
)
|
|
399
|
-
if not declared_tokens:
|
|
400
|
-
continue
|
|
537
|
+
def _unknown_spdx_replacements_for_match(
|
|
538
|
+
expression: str, matched_txt: str
|
|
539
|
+
) -> list[str]:
|
|
540
|
+
"""
|
|
541
|
+
Ordered replacements for each unknown-spdx token in one match expression.
|
|
401
542
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
543
|
+
Known tokens are never replaced. When token counts match, each unknown maps
|
|
544
|
+
to the declared token at the same index (e.g. gpl-2.0 OR unknown-spdx with
|
|
545
|
+
``GPL-2.0 or MIT-like`` → MIT-like only).
|
|
546
|
+
"""
|
|
547
|
+
expr = (expression or "").lower()
|
|
548
|
+
if KEYWORD_SCANCODE_UNKNOWN not in expr:
|
|
549
|
+
return []
|
|
550
|
+
match_tokens, _ = split_spdx_expression_with_ops(expr)
|
|
551
|
+
declared_tokens, declared_ops = _declared_licenses_from_matched_text(matched_txt)
|
|
552
|
+
if not declared_tokens:
|
|
553
|
+
return []
|
|
554
|
+
|
|
555
|
+
unknown_indexes = [
|
|
556
|
+
idx for idx, token in enumerate(match_tokens)
|
|
557
|
+
if KEYWORD_SCANCODE_UNKNOWN in token
|
|
558
|
+
]
|
|
559
|
+
if not unknown_indexes:
|
|
560
|
+
return []
|
|
561
|
+
|
|
562
|
+
queue: list[str] = []
|
|
563
|
+
if len(match_tokens) == len(declared_tokens):
|
|
564
|
+
for idx in unknown_indexes:
|
|
565
|
+
queue.append(
|
|
566
|
+
_normalize_license_token(declared_tokens[idx]) or declared_tokens[idx]
|
|
567
|
+
)
|
|
568
|
+
return queue
|
|
569
|
+
|
|
570
|
+
if all(KEYWORD_SCANCODE_UNKNOWN in token for token in match_tokens):
|
|
571
|
+
if len(match_tokens) == 1:
|
|
572
|
+
norms = [_normalize_license_token(token) or token for token in declared_tokens]
|
|
573
|
+
queue.append(join_licenses_with_ops(norms, declared_ops))
|
|
574
|
+
else:
|
|
575
|
+
for i, _ in enumerate(unknown_indexes):
|
|
576
|
+
declared = (
|
|
577
|
+
declared_tokens[i] if i < len(declared_tokens) else declared_tokens[-1]
|
|
578
|
+
)
|
|
579
|
+
queue.append(_normalize_license_token(declared) or declared)
|
|
580
|
+
return queue
|
|
581
|
+
|
|
582
|
+
# Mixed known + unknown with different declared length:
|
|
583
|
+
# drop declared tokens that correspond to known match tokens, then assign the rest.
|
|
584
|
+
unused = list(declared_tokens)
|
|
585
|
+
for token in match_tokens:
|
|
586
|
+
if KEYWORD_SCANCODE_UNKNOWN in token:
|
|
407
587
|
continue
|
|
588
|
+
token_norm = (_normalize_license_token(token) or token).lower()
|
|
589
|
+
for idx, declared in enumerate(unused):
|
|
590
|
+
declared_norm = (_normalize_license_token(declared) or declared).lower()
|
|
591
|
+
if declared_norm == token_norm or declared.lower() == token.lower():
|
|
592
|
+
unused.pop(idx)
|
|
593
|
+
break
|
|
594
|
+
for _ in unknown_indexes:
|
|
595
|
+
if not unused:
|
|
596
|
+
break
|
|
597
|
+
declared = unused.pop(0)
|
|
598
|
+
queue.append(_normalize_license_token(declared) or declared)
|
|
599
|
+
return queue
|
|
408
600
|
|
|
409
|
-
if len(match_tokens) == len(declared_tokens):
|
|
410
|
-
for idx in unknown_indexes:
|
|
411
|
-
queue.append(_normalize_license_token(declared_tokens[idx]) or declared_tokens[idx])
|
|
412
|
-
continue
|
|
413
601
|
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
queue.append(join_licenses_with_ops(norms, declared_ops))
|
|
418
|
-
else:
|
|
419
|
-
for i, _ in enumerate(unknown_indexes):
|
|
420
|
-
declared = declared_tokens[i] if i < len(declared_tokens) else declared_tokens[-1]
|
|
421
|
-
queue.append(_normalize_license_token(declared) or declared)
|
|
422
|
-
continue
|
|
602
|
+
def _resolve_unknown_spdx_in_expression(expression: str, matched_txt: str) -> str:
|
|
603
|
+
"""
|
|
604
|
+
Restore unknown-spdx in an expression from matched_text.
|
|
423
605
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
606
|
+
- All tokens unknown → use the full declared expression
|
|
607
|
+
- Mixed known + unknown → replace only unknown tokens; keep known tokens
|
|
608
|
+
- Returns empty string when unknown cannot be restored from matched_text
|
|
609
|
+
"""
|
|
610
|
+
tokens, ops = split_spdx_expression_with_ops(expression)
|
|
611
|
+
if not tokens:
|
|
612
|
+
return ""
|
|
613
|
+
unknown_indexes = [
|
|
614
|
+
idx for idx, token in enumerate(tokens)
|
|
615
|
+
if KEYWORD_SCANCODE_UNKNOWN in token.lower()
|
|
616
|
+
]
|
|
617
|
+
if not unknown_indexes:
|
|
618
|
+
return expression
|
|
619
|
+
|
|
620
|
+
if all(KEYWORD_SCANCODE_UNKNOWN in token.lower() for token in tokens):
|
|
621
|
+
return _extract_spdx_declared_expression(matched_txt)
|
|
622
|
+
|
|
623
|
+
queue = _unknown_spdx_replacements_for_match(expression, matched_txt)
|
|
624
|
+
new_tokens = list(tokens)
|
|
625
|
+
for i, idx in enumerate(unknown_indexes):
|
|
626
|
+
new_tokens[idx] = queue[i] if i < len(queue) else ""
|
|
627
|
+
kept_tokens, kept_ops = _kept_tokens_with_merged_ops(new_tokens, ops)
|
|
628
|
+
if not kept_tokens:
|
|
629
|
+
return ""
|
|
630
|
+
return join_licenses_with_ops(kept_tokens, kept_ops)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _build_unknown_spdx_replacement_queue(matches: list) -> list[str]:
|
|
634
|
+
"""Ordered replacements for each unknown-spdx token in match order."""
|
|
635
|
+
queue = []
|
|
636
|
+
for matched_lic in matches or []:
|
|
637
|
+
queue.extend(
|
|
638
|
+
_unknown_spdx_replacements_for_match(
|
|
639
|
+
matched_lic.get("license_expression") or "",
|
|
640
|
+
matched_lic.get("matched_text") or "",
|
|
641
|
+
)
|
|
642
|
+
)
|
|
441
643
|
return queue
|
|
442
644
|
|
|
443
645
|
|
|
@@ -574,53 +776,78 @@ def parsing_scancode(
|
|
|
574
776
|
all_matches = []
|
|
575
777
|
for lic in licenses or []:
|
|
576
778
|
all_matches.extend(lic.get("matches") or [])
|
|
779
|
+
# SPDX priority: if the file has SPDX-License-Identifier (:/,/-),
|
|
780
|
+
# keep only matches that are SPDX declarations (drop body rule hits).
|
|
781
|
+
prefer_spdx_declarations = _file_has_spdx_license_identifier(all_matches)
|
|
782
|
+
matches_to_process = (
|
|
783
|
+
_filter_spdx_declaration_matches(all_matches)
|
|
784
|
+
if prefer_spdx_declarations
|
|
785
|
+
else list(all_matches or [])
|
|
786
|
+
)
|
|
577
787
|
has_other_license_in_file = _file_has_other_license(all_matches)
|
|
788
|
+
has_spdx_declared_license = _file_has_spdx_declared_license(all_matches)
|
|
578
789
|
suppress_unknown_license_reference = (
|
|
579
790
|
_should_suppress_unknown_license_reference(
|
|
580
791
|
all_matches, has_other_license_in_file
|
|
581
792
|
)
|
|
582
793
|
)
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
794
|
+
# Prefer-SPDX filters matches; matched_text extraction is only for unknown-spdx.
|
|
795
|
+
# Mixed expressions keep known tokens and replace unknown tokens only.
|
|
796
|
+
for matched_lic in matches_to_process:
|
|
797
|
+
found_lic_list = matched_lic.get("license_expression", "")
|
|
798
|
+
matched_txt = matched_lic.get("matched_text", "")
|
|
799
|
+
if not found_lic_list:
|
|
800
|
+
continue
|
|
801
|
+
found_lic_list = found_lic_list.lower()
|
|
802
|
+
if KEYWORD_SCANCODE_UNKNOWN in found_lic_list:
|
|
803
|
+
resolved = _resolve_unknown_spdx_in_expression(
|
|
804
|
+
found_lic_list, matched_txt
|
|
805
|
+
)
|
|
806
|
+
if resolved:
|
|
807
|
+
found_lic_list = resolved
|
|
808
|
+
resolved_unknown_spdx = True
|
|
809
|
+
elif has_spdx_declared_license:
|
|
810
|
+
# File already has SPDX declarations; drop unrestorable
|
|
811
|
+
# unknown-spdx noise (e.g. misplaced-tag messages).
|
|
812
|
+
tokens = [
|
|
813
|
+
t for t in split_spdx_expression(found_lic_list)
|
|
814
|
+
if KEYWORD_SCANCODE_UNKNOWN not in t.lower()
|
|
815
|
+
]
|
|
816
|
+
if not tokens:
|
|
817
|
+
continue
|
|
818
|
+
found_lic_list = " AND ".join(tokens)
|
|
819
|
+
|
|
820
|
+
for found_lic in split_spdx_expression(found_lic_list):
|
|
821
|
+
if found_lic:
|
|
822
|
+
found_lic = found_lic.strip()
|
|
823
|
+
if found_lic in REMOVE_LICENSE:
|
|
824
|
+
continue
|
|
825
|
+
if (
|
|
826
|
+
KEYWORD_UNKNOWN_LICENSE_REFERENCE in found_lic.lower()
|
|
827
|
+
and not _should_keep_unknown_license_reference(
|
|
828
|
+
matched_txt, has_other_license_in_file
|
|
829
|
+
)
|
|
830
|
+
):
|
|
831
|
+
continue
|
|
832
|
+
if KEYWORD_SCANCODE_UNKNOWN in found_lic.lower():
|
|
591
833
|
declared = _extract_spdx_declared_expression(matched_txt)
|
|
592
834
|
if declared:
|
|
593
|
-
|
|
835
|
+
found_lic = declared
|
|
594
836
|
resolved_unknown_spdx = True
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
if declared:
|
|
610
|
-
found_lic = declared
|
|
611
|
-
resolved_unknown_spdx = True
|
|
612
|
-
found_lic = _strip_license_ref_prefix(found_lic)
|
|
613
|
-
found_lic = _normalize_license_token(found_lic) or found_lic
|
|
614
|
-
if not found_lic:
|
|
615
|
-
continue
|
|
616
|
-
if matched_txt:
|
|
617
|
-
lic_matched_key = found_lic + matched_txt
|
|
618
|
-
if lic_matched_key in license_list:
|
|
619
|
-
license_list[lic_matched_key].set_files(file_path)
|
|
620
|
-
else:
|
|
621
|
-
lic_info = MatchedLicense(found_lic, "", matched_txt, file_path)
|
|
622
|
-
license_list[lic_matched_key] = lic_info
|
|
623
|
-
license_detected.append(found_lic)
|
|
837
|
+
elif has_spdx_declared_license:
|
|
838
|
+
continue
|
|
839
|
+
found_lic = _strip_license_ref_prefix(found_lic)
|
|
840
|
+
found_lic = _normalize_license_token(found_lic) or found_lic
|
|
841
|
+
if not found_lic:
|
|
842
|
+
continue
|
|
843
|
+
if matched_txt:
|
|
844
|
+
lic_matched_key = found_lic + matched_txt
|
|
845
|
+
if lic_matched_key in license_list:
|
|
846
|
+
license_list[lic_matched_key].set_files(file_path)
|
|
847
|
+
else:
|
|
848
|
+
lic_info = MatchedLicense(found_lic, "", matched_txt, file_path)
|
|
849
|
+
license_list[lic_matched_key] = lic_info
|
|
850
|
+
license_detected.append(found_lic)
|
|
624
851
|
result_item.licenses = license_detected
|
|
625
852
|
file_ext = os.path.splitext(file_path)[1].lower()
|
|
626
853
|
is_source_file = file_ext and file_ext in SOURCE_EXTENSIONS
|
|
@@ -634,7 +861,22 @@ def parsing_scancode(
|
|
|
634
861
|
)
|
|
635
862
|
)
|
|
636
863
|
|
|
637
|
-
|
|
864
|
+
# Comment:
|
|
865
|
+
# - If SPDX priority dropped body matches, always record them as
|
|
866
|
+
# ``Detected: ...`` (OR unrelated).
|
|
867
|
+
# - Else, for multi-license OR expressions, rebuild from detected_*.
|
|
868
|
+
dropped_matches = []
|
|
869
|
+
if prefer_spdx_declarations:
|
|
870
|
+
process_ids = {id(match) for match in matches_to_process}
|
|
871
|
+
dropped_matches = [
|
|
872
|
+
match for match in all_matches if id(match) not in process_ids
|
|
873
|
+
]
|
|
874
|
+
detected_comment = build_detected_comment_from_dropped_matches(
|
|
875
|
+
dropped_matches, result_item.licenses
|
|
876
|
+
)
|
|
877
|
+
if detected_comment:
|
|
878
|
+
result_item.comment = detected_comment
|
|
879
|
+
if not result_item.comment and len(license_detected) > 1:
|
|
638
880
|
detected_expression = file.get("detected_license_expression", "") or ""
|
|
639
881
|
detected_expression_spdx = file.get("detected_license_expression_spdx", "") or ""
|
|
640
882
|
if (
|
|
@@ -454,16 +454,22 @@ def merge_results(
|
|
|
454
454
|
scancode_result.append(new_result_item)
|
|
455
455
|
if manifest_licenses:
|
|
456
456
|
for file_name, licenses in manifest_licenses.items():
|
|
457
|
-
|
|
457
|
+
# None: marker-only (keep ScanCode). list (incl. []): overwrite with extracted result.
|
|
458
|
+
keep_scancode_licenses = licenses is None
|
|
459
|
+
license_list = [] if keep_scancode_licenses else licenses
|
|
460
|
+
valid_licenses = [
|
|
461
|
+
lic.strip() for lic in license_list if isinstance(lic, str) and lic.strip()
|
|
462
|
+
]
|
|
458
463
|
item = _get_or_append_source_item(
|
|
459
464
|
scancode_result, file_name, append=bool(valid_licenses) or ui_mode
|
|
460
465
|
)
|
|
461
466
|
if item is None:
|
|
462
467
|
continue
|
|
463
468
|
item.is_manifest_file = True
|
|
469
|
+
if keep_scancode_licenses:
|
|
470
|
+
continue
|
|
471
|
+
item.licenses = []
|
|
464
472
|
if valid_licenses:
|
|
465
|
-
# overwrite existing detected licenses with manifest-provided licenses
|
|
466
|
-
item.licenses = [] # clear existing licenses (setter clears when value falsy)
|
|
467
473
|
item.licenses = valid_licenses
|
|
468
474
|
|
|
469
475
|
kb_origin_urls: dict[str, str] = {}
|
|
@@ -719,7 +725,9 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di
|
|
|
719
725
|
|
|
720
726
|
- Traverse files with exclusions applied
|
|
721
727
|
- spdx_downloads: {rel_path: [download_urls]}
|
|
722
|
-
- manifest_licenses: {rel_path: [
|
|
728
|
+
- manifest_licenses: {rel_path: list[str] | None}
|
|
729
|
+
- list (incl. []): overwrite ScanCode licenses with this result
|
|
730
|
+
- None: marker-only; keep ScanCode licenses
|
|
723
731
|
|
|
724
732
|
:return: (spdx_downloads, manifest_licenses)
|
|
725
733
|
"""
|
|
@@ -739,7 +747,8 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di
|
|
|
739
747
|
spdx_downloads[rel_path_file] = downloads
|
|
740
748
|
|
|
741
749
|
if is_manifest_file(file_path):
|
|
742
|
-
|
|
750
|
+
# Preserve None (marker-only); do not coerce with `or []`.
|
|
751
|
+
manifest_licenses[rel_path_file] = get_manifest_licenses(file_path)
|
|
743
752
|
|
|
744
753
|
return spdx_downloads, manifest_licenses
|
|
745
754
|
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_manifest_extractor.py
RENAMED
|
@@ -6,6 +6,7 @@ import os
|
|
|
6
6
|
import json
|
|
7
7
|
import re
|
|
8
8
|
import logging
|
|
9
|
+
from typing import Optional
|
|
9
10
|
from fosslight_util.get_pom_license import get_license_from_pom
|
|
10
11
|
import fosslight_util.constant as constant
|
|
11
12
|
|
|
@@ -111,6 +112,11 @@ def get_licenses_from_composer_json(file_path: str) -> list[str]:
|
|
|
111
112
|
return unique
|
|
112
113
|
|
|
113
114
|
|
|
115
|
+
def _is_setup_cfg_license_file_ref(value: str) -> bool:
|
|
116
|
+
"""True when setup.cfg license points at a LICENSE file name, not an SPDX id."""
|
|
117
|
+
return value.strip().upper() == 'LICENSE'
|
|
118
|
+
|
|
119
|
+
|
|
114
120
|
def get_licenses_from_setup_cfg(file_path: str) -> list[str]:
|
|
115
121
|
try:
|
|
116
122
|
import configparser
|
|
@@ -119,6 +125,9 @@ def get_licenses_from_setup_cfg(file_path: str) -> list[str]:
|
|
|
119
125
|
if parser.has_section('metadata'):
|
|
120
126
|
license_value = parser.get('metadata', 'license', fallback='').strip()
|
|
121
127
|
if license_value:
|
|
128
|
+
# license = LICENSE is a file reference; treat as empty (clears ScanCode on merge).
|
|
129
|
+
if _is_setup_cfg_license_file_ref(license_value):
|
|
130
|
+
return []
|
|
122
131
|
return _split_spdx_expression(license_value)
|
|
123
132
|
except Exception as ex:
|
|
124
133
|
logger.info(f"Failed to parse setup.cfg with configparser for {file_path}: {ex}")
|
|
@@ -138,6 +147,8 @@ def get_licenses_from_setup_cfg(file_path: str) -> list[str]:
|
|
|
138
147
|
val = val[1:-1].strip()
|
|
139
148
|
if not val:
|
|
140
149
|
return []
|
|
150
|
+
if _is_setup_cfg_license_file_ref(val):
|
|
151
|
+
return []
|
|
141
152
|
return _split_spdx_expression(val)
|
|
142
153
|
except Exception as ex:
|
|
143
154
|
logger.info(f"Failed to parse setup.cfg {file_path} via regex fallback: {ex}")
|
|
@@ -372,10 +383,15 @@ def get_licenses_from_huggingface_metadata(file_path: str) -> list[str]:
|
|
|
372
383
|
return licenses
|
|
373
384
|
|
|
374
385
|
|
|
375
|
-
def get_manifest_licenses(file_path: str) -> list[str]:
|
|
376
|
-
|
|
386
|
+
def get_manifest_licenses(file_path: str) -> Optional[list[str]]:
|
|
387
|
+
"""Extract licenses from a manifest file.
|
|
388
|
+
|
|
389
|
+
Returns:
|
|
390
|
+
list[str]: licenses to apply in merge (empty list clears ScanCode licenses).
|
|
391
|
+
None: marker-only manifest; merge sets is_manifest_file and keeps ScanCode licenses.
|
|
392
|
+
"""
|
|
377
393
|
if os.path.basename(file_path).lower() == 'android.bp':
|
|
378
|
-
return
|
|
394
|
+
return None
|
|
379
395
|
if file_path.endswith('.pom'):
|
|
380
396
|
try:
|
|
381
397
|
pom_licenses = get_license_from_pom(group_id='', artifact_id='', version='', pom_path=file_path, check_parent=True)
|
|
@@ -433,3 +449,4 @@ def get_manifest_licenses(file_path: str) -> list[str]:
|
|
|
433
449
|
except Exception as ex:
|
|
434
450
|
logger.info(f"Failed to extract license from huggingface_hub_metadata.json {file_path}: {ex}")
|
|
435
451
|
return []
|
|
452
|
+
return []
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/SOURCES.txt
RENAMED
|
@@ -28,6 +28,7 @@ tests/test_manifest_android_bp.py
|
|
|
28
28
|
tests/test_manifest_composer.py
|
|
29
29
|
tests/test_manifest_pyproject.py
|
|
30
30
|
tests/test_manifest_recommended_scenarios.py
|
|
31
|
+
tests/test_manifest_setup_cfg.py
|
|
31
32
|
tests/test_multi_value_order.py
|
|
32
33
|
tests/test_parsing_unknown_spdx.py
|
|
33
34
|
tests/test_tox.py
|
|
@@ -18,10 +18,10 @@ def test_is_manifest_file_recognizes_android_bp():
|
|
|
18
18
|
assert is_manifest_file("/tmp/module/package.json") is True
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def
|
|
21
|
+
def test_get_manifest_licenses_returns_none_for_android_bp(tmp_path):
|
|
22
22
|
android_bp = tmp_path / "Android.bp"
|
|
23
23
|
android_bp.write_text('license { name: "test_license" }', encoding="utf-8")
|
|
24
|
-
assert get_manifest_licenses(str(android_bp))
|
|
24
|
+
assert get_manifest_licenses(str(android_bp)) is None
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def test_metadata_collector_adds_android_bp_without_license_extraction(tmp_path):
|
|
@@ -34,7 +34,7 @@ def test_metadata_collector_adds_android_bp_without_license_extraction(tmp_path)
|
|
|
34
34
|
spdx_downloads, manifest_licenses = metadata_collector(str(tmp_path), set())
|
|
35
35
|
|
|
36
36
|
assert spdx_downloads == {}
|
|
37
|
-
assert manifest_licenses == {"Android.bp":
|
|
37
|
+
assert manifest_licenses == {"Android.bp": None, "package.json": ["MIT"]}
|
|
38
38
|
assert mock_get.call_count == 2
|
|
39
39
|
|
|
40
40
|
|
|
@@ -44,7 +44,7 @@ def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(
|
|
|
44
44
|
|
|
45
45
|
merged, _, _, _ = merge_results(
|
|
46
46
|
scancode_result=[scancode_item],
|
|
47
|
-
manifest_licenses={"carrois-gothic-sc/Android.bp":
|
|
47
|
+
manifest_licenses={"carrois-gothic-sc/Android.bp": None},
|
|
48
48
|
)
|
|
49
49
|
|
|
50
50
|
assert len(merged) == 1
|
|
@@ -55,7 +55,7 @@ def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(
|
|
|
55
55
|
def test_merge_results_skips_android_bp_not_in_scancode_result():
|
|
56
56
|
merged, _, _, _ = merge_results(
|
|
57
57
|
scancode_result=[],
|
|
58
|
-
manifest_licenses={"module/Android.bp":
|
|
58
|
+
manifest_licenses={"module/Android.bp": None},
|
|
59
59
|
)
|
|
60
60
|
|
|
61
61
|
assert merged == []
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/tests/test_manifest_recommended_scenarios.py
RENAMED
|
@@ -13,7 +13,7 @@ def test_scenario1_android_bp_keeps_scancode_licenses():
|
|
|
13
13
|
|
|
14
14
|
merged, _, _, _ = merge_results(
|
|
15
15
|
scancode_result=[scancode_item],
|
|
16
|
-
manifest_licenses={"Android.bp":
|
|
16
|
+
manifest_licenses={"Android.bp": None},
|
|
17
17
|
)
|
|
18
18
|
|
|
19
19
|
assert len(merged) == 1
|
|
@@ -21,10 +21,11 @@ def test_scenario1_android_bp_keeps_scancode_licenses():
|
|
|
21
21
|
assert merged[0].licenses == ["Apache-2.0", "BSD", "MIT", "OFL", "unknown-license-reference"]
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def
|
|
25
|
-
"""package.json manifest
|
|
24
|
+
def test_scenario2_package_json_empty_clears_scancode_licenses():
|
|
25
|
+
"""package.json empty manifest licenses overwrite/clear ScanCode licenses."""
|
|
26
26
|
scancode_item = SourceItem("app/package.json")
|
|
27
27
|
scancode_item.licenses = ["Apache-2.0"]
|
|
28
|
+
scancode_item.download_location = ["https://example.com/app"]
|
|
28
29
|
|
|
29
30
|
merged, _, _, _ = merge_results(
|
|
30
31
|
scancode_result=[scancode_item],
|
|
@@ -33,7 +34,23 @@ def test_scenario2_package_json_manifest_fail_keeps_scancode_licenses():
|
|
|
33
34
|
|
|
34
35
|
assert len(merged) == 1
|
|
35
36
|
assert merged[0].is_manifest_file is True
|
|
36
|
-
assert merged[0].licenses == [
|
|
37
|
+
assert merged[0].licenses == []
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_scenario2_pyproject_license_file_clears_scancode_licenses():
|
|
41
|
+
"""pyproject.toml license={file=...} yields []; ScanCode licenses are cleared."""
|
|
42
|
+
scancode_item = SourceItem("tornado/pyproject.toml")
|
|
43
|
+
scancode_item.licenses = ["Apache-2.0"]
|
|
44
|
+
scancode_item.download_location = ["https://example.com/tornado"]
|
|
45
|
+
|
|
46
|
+
merged, _, _, _ = merge_results(
|
|
47
|
+
scancode_result=[scancode_item],
|
|
48
|
+
manifest_licenses={"tornado/pyproject.toml": []},
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
assert len(merged) == 1
|
|
52
|
+
assert merged[0].is_manifest_file is True
|
|
53
|
+
assert merged[0].licenses == []
|
|
37
54
|
|
|
38
55
|
|
|
39
56
|
def test_scenario3_android_bp_from_spdx_marks_manifest_without_new_row():
|
|
@@ -43,7 +60,7 @@ def test_scenario3_android_bp_from_spdx_marks_manifest_without_new_row():
|
|
|
43
60
|
|
|
44
61
|
merged, _, _, _ = merge_results(
|
|
45
62
|
scancode_result=[spdx_item],
|
|
46
|
-
manifest_licenses={"module/Android.bp":
|
|
63
|
+
manifest_licenses={"module/Android.bp": None},
|
|
47
64
|
)
|
|
48
65
|
|
|
49
66
|
assert len(merged) == 1
|
|
@@ -56,7 +73,7 @@ def test_scenario3_android_bp_not_in_result_no_row_non_ui():
|
|
|
56
73
|
"""Android.bp absent from merge result: no row in non-UI mode."""
|
|
57
74
|
merged_non_ui, _, _, _ = merge_results(
|
|
58
75
|
scancode_result=[],
|
|
59
|
-
manifest_licenses={"module/Android.bp":
|
|
76
|
+
manifest_licenses={"module/Android.bp": None},
|
|
60
77
|
ui_mode=False,
|
|
61
78
|
)
|
|
62
79
|
assert merged_non_ui == []
|
|
@@ -66,7 +83,7 @@ def test_scenario3_android_bp_not_in_result_ui_keeps_empty_row():
|
|
|
66
83
|
"""Android.bp absent from merge result: UI mode still creates manifest row."""
|
|
67
84
|
merged_ui, _, _, _ = merge_results(
|
|
68
85
|
scancode_result=[],
|
|
69
|
-
manifest_licenses={"module/Android.bp":
|
|
86
|
+
manifest_licenses={"module/Android.bp": None},
|
|
70
87
|
ui_mode=True,
|
|
71
88
|
)
|
|
72
89
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Copyright (c) 2026 LG Electronics Inc.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Tests for setup.cfg manifest license extraction."""
|
|
4
|
+
|
|
5
|
+
from fosslight_source._scan_item import SourceItem
|
|
6
|
+
from fosslight_source.cli import merge_results
|
|
7
|
+
from fosslight_source.run_manifest_extractor import get_licenses_from_setup_cfg
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_setup_cfg_spdx_license(tmp_path):
|
|
11
|
+
path = tmp_path / "setup.cfg"
|
|
12
|
+
path.write_text("[metadata]\nname = demo\nlicense = MIT\n", encoding="utf-8")
|
|
13
|
+
assert get_licenses_from_setup_cfg(str(path)) == ["MIT"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_setup_cfg_license_filename_returns_empty(tmp_path):
|
|
17
|
+
path = tmp_path / "setup.cfg"
|
|
18
|
+
path.write_text("[metadata]\nname = demo\nlicense = LICENSE\n", encoding="utf-8")
|
|
19
|
+
assert get_licenses_from_setup_cfg(str(path)) == []
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_setup_cfg_license_filename_case_insensitive(tmp_path):
|
|
23
|
+
path = tmp_path / "setup.cfg"
|
|
24
|
+
path.write_text("[metadata]\nname = demo\nlicense = License\n", encoding="utf-8")
|
|
25
|
+
assert get_licenses_from_setup_cfg(str(path)) == []
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_setup_cfg_license_filename_clears_scancode_licenses(tmp_path):
|
|
29
|
+
path = tmp_path / "setup.cfg"
|
|
30
|
+
path.write_text("[metadata]\nname = demo\nlicense = LICENSE\n", encoding="utf-8")
|
|
31
|
+
assert get_licenses_from_setup_cfg(str(path)) == []
|
|
32
|
+
|
|
33
|
+
scancode_item = SourceItem("pkg/setup.cfg")
|
|
34
|
+
scancode_item.licenses = ["Apache-2.0"]
|
|
35
|
+
scancode_item.download_location = ["https://example.com/pkg"]
|
|
36
|
+
merged, _, _, _ = merge_results(
|
|
37
|
+
scancode_result=[scancode_item],
|
|
38
|
+
manifest_licenses={"pkg/setup.cfg": []},
|
|
39
|
+
)
|
|
40
|
+
assert merged[0].is_manifest_file is True
|
|
41
|
+
assert merged[0].licenses == []
|
|
@@ -21,7 +21,6 @@ from fosslight_source._parsing_scancode_file_item import (
|
|
|
21
21
|
("// SPDX-License-Identifier: MIT", "MIT"),
|
|
22
22
|
("// SPDX-License-Identifier-MIT", "MIT"),
|
|
23
23
|
("// SPDX-License-Identifier, MIT", "MIT"),
|
|
24
|
-
("// SPDX-License-Identifier MIT", "MIT"),
|
|
25
24
|
(' "SPDX-license-identifier-BSD",', "BSD"),
|
|
26
25
|
(' "SPDX-license-identifier-OFL", // by exception only', "OFL"),
|
|
27
26
|
("/* SPDX-License-Identifier: MIT */", "MIT"),
|
|
@@ -41,6 +40,20 @@ def test_extract_spdx_declared_expression(matched_text, expected):
|
|
|
41
40
|
assert _extract_spdx_declared_expression(matched_text) == expected
|
|
42
41
|
|
|
43
42
|
|
|
43
|
+
@pytest.mark.parametrize(
|
|
44
|
+
"matched_text",
|
|
45
|
+
[
|
|
46
|
+
"// SPDX-License-Identifier MIT",
|
|
47
|
+
(
|
|
48
|
+
'\t\t\t "Misplaced SPDX-License-Identifier tag - use line '
|
|
49
|
+
'$checklicenseline instead " . $herecurr);'
|
|
50
|
+
),
|
|
51
|
+
],
|
|
52
|
+
)
|
|
53
|
+
def test_extract_spdx_rejects_whitespace_only_separator(matched_text):
|
|
54
|
+
assert _extract_spdx_declared_expression(matched_text) == ""
|
|
55
|
+
|
|
56
|
+
|
|
44
57
|
def test_declared_licenses_strips_licenseref_per_token():
|
|
45
58
|
tokens, ops = _declared_licenses_from_matched_text(
|
|
46
59
|
"# SPDX-License-Identifier: LicenseRef-Foo AND LicenseRef-Bar"
|
|
@@ -128,9 +141,11 @@ def test_unknown_spdx_comment_preserves_and_or_from_detected_expression():
|
|
|
128
141
|
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
129
142
|
|
|
130
143
|
assert success is True
|
|
131
|
-
|
|
144
|
+
# Classic SPDX-License-Identifier with colon present → SPDX priority drops
|
|
145
|
+
# non-declaration matches from licenses; dropped findings go to Detected comment.
|
|
146
|
+
assert results[0].licenses == ["DApache-2.0", "NEW"]
|
|
132
147
|
assert "unknown-license-reference" not in [lic.lower() for lic in results[0].licenses]
|
|
133
|
-
assert results[0].comment == "
|
|
148
|
+
assert results[0].comment == "Detected: unknown-license-reference, gpl-2.0"
|
|
134
149
|
|
|
135
150
|
|
|
136
151
|
def test_unknown_license_reference_suppressed_when_same_matched_text_has_other_license():
|
|
@@ -238,7 +253,7 @@ def test_unknown_license_reference_suppressed_by_other_license_in_same_file():
|
|
|
238
253
|
success, results, _messages, license_list = parsing_scancode(scancode_file_list)
|
|
239
254
|
|
|
240
255
|
assert success is True
|
|
241
|
-
assert results[0].licenses == ["
|
|
256
|
+
assert results[0].licenses == ["Apache-2.0", "MIT"]
|
|
242
257
|
assert results[0].comment == "MIT OR Apache-2.0"
|
|
243
258
|
assert all(
|
|
244
259
|
item.license != "unknown-license-reference"
|
|
@@ -499,12 +514,345 @@ def test_android_bp_soong_license_kinds_without_line_comment_in_license():
|
|
|
499
514
|
|
|
500
515
|
assert success is True
|
|
501
516
|
licenses = results[0].licenses
|
|
517
|
+
# unknown-license-reference has no URL and other licenses exist → suppressed
|
|
502
518
|
assert licenses == [
|
|
503
519
|
"Apache-2.0",
|
|
504
520
|
"BSD",
|
|
505
521
|
"MIT",
|
|
506
522
|
"OFL",
|
|
507
|
-
"unknown-license-reference",
|
|
508
523
|
]
|
|
509
524
|
assert all("//" not in lic for lic in results[0].licenses)
|
|
510
525
|
assert all('"' not in lic for lic in results[0].licenses)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def test_ignore_unknown_spdx_when_file_has_spdx_declaration():
|
|
529
|
+
"""SPDX-License-Identifier present → ignore unrestorable unknown-spdx match."""
|
|
530
|
+
scancode_file_list = [{
|
|
531
|
+
"path": "sample.c",
|
|
532
|
+
"type": "file",
|
|
533
|
+
"detected_license_expression": "gpl-2.0 AND unknown-spdx",
|
|
534
|
+
"license_detections": [
|
|
535
|
+
{
|
|
536
|
+
"matches": [{
|
|
537
|
+
"license_expression": "gpl-2.0",
|
|
538
|
+
"matched_text": "# SPDX-License-Identifier: GPL-2.0",
|
|
539
|
+
}],
|
|
540
|
+
},
|
|
541
|
+
{
|
|
542
|
+
"matches": [{
|
|
543
|
+
"license_expression": "unknown-spdx",
|
|
544
|
+
"matched_text": (
|
|
545
|
+
'\t\t\t "Misplaced SPDX-License-Identifier tag - use line '
|
|
546
|
+
'$checklicenseline instead " . $herecurr);'
|
|
547
|
+
),
|
|
548
|
+
}],
|
|
549
|
+
},
|
|
550
|
+
],
|
|
551
|
+
"copyrights": [],
|
|
552
|
+
}]
|
|
553
|
+
|
|
554
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
555
|
+
|
|
556
|
+
assert success is True
|
|
557
|
+
assert results[0].licenses == ["GPL-2.0"]
|
|
558
|
+
assert all("unknown" not in lic.lower() for lic in results[0].licenses)
|
|
559
|
+
assert all("tag" not in lic.lower() for lic in results[0].licenses)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def test_spdx_priority_drops_body_rule_matches():
|
|
563
|
+
"""When classic SPDX-License-Identifier with colon exists, keep declaration only."""
|
|
564
|
+
scancode_file_list = [{
|
|
565
|
+
"path": "sample.c",
|
|
566
|
+
"type": "file",
|
|
567
|
+
"detected_license_expression": (
|
|
568
|
+
"gpl-2.0 AND (gpl-2.0 AND bsd-simplified AND (gpl-2.0 OR bsd-simplified))"
|
|
569
|
+
),
|
|
570
|
+
"license_detections": [
|
|
571
|
+
{
|
|
572
|
+
"matches": [{
|
|
573
|
+
"license_expression": "gpl-2.0",
|
|
574
|
+
"matched_text": "# SPDX-License-Identifier: GPL-2.0",
|
|
575
|
+
}],
|
|
576
|
+
},
|
|
577
|
+
{
|
|
578
|
+
"matches": [
|
|
579
|
+
{
|
|
580
|
+
"license_expression": "gpl-2.0",
|
|
581
|
+
"matched_text": (
|
|
582
|
+
"\t\t\t\t\t not $spdx_license =~ /GPL-2\\.0.*BSD-2-Clause/) {"
|
|
583
|
+
),
|
|
584
|
+
},
|
|
585
|
+
{
|
|
586
|
+
"license_expression": "bsd-simplified",
|
|
587
|
+
"matched_text": (
|
|
588
|
+
"\t\t\t\t\t not $spdx_license =~ /GPL-2\\.0.*BSD-2-Clause/) {"
|
|
589
|
+
),
|
|
590
|
+
},
|
|
591
|
+
{
|
|
592
|
+
"license_expression": "gpl-2.0 OR bsd-simplified",
|
|
593
|
+
"matched_text": (
|
|
594
|
+
"DT binding documents should be licensed "
|
|
595
|
+
"(GPL-2.0-only OR BSD-2-Clause)"
|
|
596
|
+
),
|
|
597
|
+
},
|
|
598
|
+
],
|
|
599
|
+
},
|
|
600
|
+
],
|
|
601
|
+
"copyrights": [],
|
|
602
|
+
}]
|
|
603
|
+
|
|
604
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
605
|
+
|
|
606
|
+
assert success is True
|
|
607
|
+
assert results[0].licenses == ["GPL-2.0"]
|
|
608
|
+
# Body singles covered by OR expression are dropped; Detected keeps the OR form.
|
|
609
|
+
assert results[0].comment == "Detected: gpl-2.0 OR bsd-simplified"
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def test_prefer_spdx_uses_license_expression_not_matched_text_trailer():
|
|
613
|
+
"""Known license keeps ScanCode expression; HTML trailer in matched_text is ignored."""
|
|
614
|
+
scancode_file_list = [{
|
|
615
|
+
"path": "index.html",
|
|
616
|
+
"type": "file",
|
|
617
|
+
"detected_license_expression": "apache-2.0",
|
|
618
|
+
"detected_license_expression_spdx": "Apache-2.0",
|
|
619
|
+
"license_detections": [{
|
|
620
|
+
"matches": [{
|
|
621
|
+
"license_expression": "apache-2.0",
|
|
622
|
+
"license_expression_spdx": "Apache-2.0",
|
|
623
|
+
"matched_text": " SPDX-License-Identifier: Apache-2.0</p>",
|
|
624
|
+
}],
|
|
625
|
+
}],
|
|
626
|
+
"copyrights": [],
|
|
627
|
+
}]
|
|
628
|
+
|
|
629
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
630
|
+
|
|
631
|
+
assert success is True
|
|
632
|
+
assert results[0].licenses == ["Apache-2.0"]
|
|
633
|
+
assert all("</p>" not in lic for lic in results[0].licenses)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def test_mixed_unknown_spdx_preserves_known_tokens():
|
|
637
|
+
"""unknown-spdx AND mit with a shorter declared id must keep MIT."""
|
|
638
|
+
scancode_file_list = [{
|
|
639
|
+
"path": "mixed.c",
|
|
640
|
+
"type": "file",
|
|
641
|
+
"detected_license_expression": "unknown-spdx AND mit",
|
|
642
|
+
"license_detections": [{
|
|
643
|
+
"matches": [{
|
|
644
|
+
"license_expression": "unknown-spdx AND mit",
|
|
645
|
+
"matched_text": "# SPDX-License-Identifier: Apache-2.0",
|
|
646
|
+
}],
|
|
647
|
+
}],
|
|
648
|
+
"copyrights": [],
|
|
649
|
+
}]
|
|
650
|
+
|
|
651
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
652
|
+
|
|
653
|
+
assert success is True
|
|
654
|
+
assert results[0].licenses == ["Apache-2.0", "MIT"]
|
|
655
|
+
assert results[0].comment == ""
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
@pytest.mark.parametrize(
|
|
659
|
+
"dropped_exprs, expected_comment",
|
|
660
|
+
[
|
|
661
|
+
(
|
|
662
|
+
[
|
|
663
|
+
("bsd-simplified", "BSD-2-Clause"),
|
|
664
|
+
("gpl-2.0 OR bsd-simplified", "GPL-2.0-only OR BSD-2-Clause"),
|
|
665
|
+
],
|
|
666
|
+
"Detected: GPL-2.0-only OR BSD-2-Clause",
|
|
667
|
+
),
|
|
668
|
+
(
|
|
669
|
+
[
|
|
670
|
+
("gpl-2.0", "GPL-2.0"),
|
|
671
|
+
("gpl-2.0 OR bsd-simplified", "GPL-2.0-only OR BSD-2-Clause"),
|
|
672
|
+
],
|
|
673
|
+
"Detected: GPL-2.0, GPL-2.0-only OR BSD-2-Clause",
|
|
674
|
+
),
|
|
675
|
+
(
|
|
676
|
+
[
|
|
677
|
+
("gpl-3.0", "GPL-3.0"),
|
|
678
|
+
("apache-2.0", "Apache-2.0"),
|
|
679
|
+
("gpl-2.0 OR bsd-simplified", "GPL-2.0-only OR BSD-2-Clause"),
|
|
680
|
+
],
|
|
681
|
+
"Detected: GPL-3.0, Apache-2.0, GPL-2.0-only OR BSD-2-Clause",
|
|
682
|
+
),
|
|
683
|
+
(
|
|
684
|
+
[
|
|
685
|
+
("unknown-spdx", "LicenseRef-scancode-unknown-spdx"),
|
|
686
|
+
],
|
|
687
|
+
"Detected: unknown-spdx",
|
|
688
|
+
),
|
|
689
|
+
],
|
|
690
|
+
)
|
|
691
|
+
def test_dedupe_detected_comment_examples(dropped_exprs, expected_comment):
|
|
692
|
+
from fosslight_source._parsing_scancode_file_item import (
|
|
693
|
+
build_detected_comment_from_dropped_matches,
|
|
694
|
+
)
|
|
695
|
+
dropped = [
|
|
696
|
+
{
|
|
697
|
+
"license_expression": expr,
|
|
698
|
+
"license_expression_spdx": spdx,
|
|
699
|
+
"matched_text": "body",
|
|
700
|
+
}
|
|
701
|
+
for expr, spdx in dropped_exprs
|
|
702
|
+
]
|
|
703
|
+
assert build_detected_comment_from_dropped_matches(dropped) == expected_comment
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def test_detected_comment_excludes_licenses_already_in_license_column():
|
|
707
|
+
from fosslight_source._parsing_scancode_file_item import (
|
|
708
|
+
build_detected_comment_from_dropped_matches,
|
|
709
|
+
)
|
|
710
|
+
dropped = [
|
|
711
|
+
{
|
|
712
|
+
"license_expression": "apache-2.0",
|
|
713
|
+
"license_expression_spdx": "Apache-2.0",
|
|
714
|
+
"matched_text": "Licensed under the Apache License, Version 2.0",
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
"license_expression": "mit",
|
|
718
|
+
"license_expression_spdx": "MIT",
|
|
719
|
+
"matched_text": "Permission is hereby granted",
|
|
720
|
+
},
|
|
721
|
+
]
|
|
722
|
+
assert (
|
|
723
|
+
build_detected_comment_from_dropped_matches(dropped, ["Apache-2.0"])
|
|
724
|
+
== "Detected: MIT"
|
|
725
|
+
)
|
|
726
|
+
assert build_detected_comment_from_dropped_matches(dropped, ["Apache-2.0", "MIT"]) == ""
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def test_detected_comment_uses_final_licenses_after_column_limit():
|
|
730
|
+
"""License dropped by column length limit must still be allowed in Detected."""
|
|
731
|
+
fillers = [f"AaaLic{i:02d}-{'y' * 80}" for i in range(20)]
|
|
732
|
+
spdx_matches = [
|
|
733
|
+
{
|
|
734
|
+
"license_expression": name.lower(),
|
|
735
|
+
"license_expression_spdx": name,
|
|
736
|
+
"matched_text": f"SPDX-License-Identifier: {name}",
|
|
737
|
+
}
|
|
738
|
+
for name in fillers + ["ZebraExtra"]
|
|
739
|
+
]
|
|
740
|
+
scancode_file_list = [{
|
|
741
|
+
"path": "crowded.c",
|
|
742
|
+
"type": "file",
|
|
743
|
+
"detected_license_expression": " AND ".join(
|
|
744
|
+
name.lower() for name in fillers + ["ZebraExtra"]
|
|
745
|
+
),
|
|
746
|
+
"license_detections": [
|
|
747
|
+
{"matches": spdx_matches},
|
|
748
|
+
{
|
|
749
|
+
"matches": [{
|
|
750
|
+
"license_expression": "zebraextra",
|
|
751
|
+
"license_expression_spdx": "ZebraExtra",
|
|
752
|
+
"matched_text": "This file is under ZebraExtra terms.",
|
|
753
|
+
}],
|
|
754
|
+
},
|
|
755
|
+
],
|
|
756
|
+
"copyrights": [],
|
|
757
|
+
}]
|
|
758
|
+
|
|
759
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
760
|
+
|
|
761
|
+
assert success is True
|
|
762
|
+
assert "ZebraExtra" not in results[0].licenses
|
|
763
|
+
assert "Detected: ZebraExtra" in results[0].comment
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def test_readme_spdx_apache_does_not_repeat_in_detected_comment():
|
|
767
|
+
"""Body apache-2.0 match dropped by SPDX priority must not echo License column."""
|
|
768
|
+
scancode_file_list = [{
|
|
769
|
+
"path": "README.md",
|
|
770
|
+
"type": "file",
|
|
771
|
+
"detected_license_expression": "apache-2.0",
|
|
772
|
+
"detected_license_expression_spdx": "Apache-2.0",
|
|
773
|
+
"license_detections": [
|
|
774
|
+
{
|
|
775
|
+
"matches": [{
|
|
776
|
+
"license_expression": "apache-2.0",
|
|
777
|
+
"license_expression_spdx": "Apache-2.0",
|
|
778
|
+
"matched_text": "SPDX-License-Identifier: Apache-2.0",
|
|
779
|
+
}],
|
|
780
|
+
},
|
|
781
|
+
{
|
|
782
|
+
"matches": [
|
|
783
|
+
{
|
|
784
|
+
"license_expression": "apache-2.0",
|
|
785
|
+
"license_expression_spdx": "Apache-2.0",
|
|
786
|
+
"matched_text": (
|
|
787
|
+
'Licensed under the Apache License, Version 2.0 '
|
|
788
|
+
'(the "License"); you may not use this file except '
|
|
789
|
+
"in compliance with the License."
|
|
790
|
+
),
|
|
791
|
+
},
|
|
792
|
+
{
|
|
793
|
+
"license_expression": "apache-2.0",
|
|
794
|
+
"license_expression_spdx": "Apache-2.0",
|
|
795
|
+
"matched_text": "SPDX-License-Identifier: Apache-2.0",
|
|
796
|
+
},
|
|
797
|
+
],
|
|
798
|
+
},
|
|
799
|
+
],
|
|
800
|
+
"copyrights": [],
|
|
801
|
+
}]
|
|
802
|
+
|
|
803
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
804
|
+
|
|
805
|
+
assert success is True
|
|
806
|
+
assert results[0].licenses == ["Apache-2.0"]
|
|
807
|
+
assert results[0].comment == ""
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def test_spdx_priority_detected_comment_uses_spdx_ids():
|
|
811
|
+
"""Dropped body matches use license_expression_spdx in Detected comment."""
|
|
812
|
+
scancode_file_list = [{
|
|
813
|
+
"path": "tag.pl",
|
|
814
|
+
"type": "file",
|
|
815
|
+
"detected_license_expression": (
|
|
816
|
+
"gpl-2.0 AND bsd-simplified AND (gpl-2.0 OR bsd-simplified) AND unknown-spdx"
|
|
817
|
+
),
|
|
818
|
+
"license_detections": [
|
|
819
|
+
{
|
|
820
|
+
"matches": [{
|
|
821
|
+
"license_expression": "gpl-2.0",
|
|
822
|
+
"license_expression_spdx": "GPL-2.0-only",
|
|
823
|
+
"matched_text": "# SPDX-License-Identifier: GPL-2.0",
|
|
824
|
+
}],
|
|
825
|
+
},
|
|
826
|
+
{
|
|
827
|
+
"matches": [
|
|
828
|
+
{
|
|
829
|
+
"license_expression": "bsd-simplified",
|
|
830
|
+
"license_expression_spdx": "BSD-2-Clause",
|
|
831
|
+
"matched_text": "not $spdx_license =~ /GPL-2\\.0.*BSD-2-Clause/",
|
|
832
|
+
},
|
|
833
|
+
{
|
|
834
|
+
"license_expression": "gpl-2.0 OR bsd-simplified",
|
|
835
|
+
"license_expression_spdx": "GPL-2.0-only OR BSD-2-Clause",
|
|
836
|
+
"matched_text": "licensed (GPL-2.0-only OR BSD-2-Clause)",
|
|
837
|
+
},
|
|
838
|
+
{
|
|
839
|
+
"license_expression": "unknown-spdx",
|
|
840
|
+
"license_expression_spdx": "LicenseRef-scancode-unknown-spdx",
|
|
841
|
+
"matched_text": (
|
|
842
|
+
'"Misplaced SPDX-License-Identifier tag - use line '
|
|
843
|
+
'$checklicenseline instead "'
|
|
844
|
+
),
|
|
845
|
+
},
|
|
846
|
+
],
|
|
847
|
+
},
|
|
848
|
+
],
|
|
849
|
+
"copyrights": [],
|
|
850
|
+
}]
|
|
851
|
+
|
|
852
|
+
success, results, _messages, _ = parsing_scancode(scancode_file_list)
|
|
853
|
+
|
|
854
|
+
assert success is True
|
|
855
|
+
assert results[0].licenses == ["GPL-2.0"]
|
|
856
|
+
assert results[0].comment == (
|
|
857
|
+
"Detected: GPL-2.0-only OR BSD-2-Clause, unknown-spdx"
|
|
858
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_license_matched.py
RENAMED
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/_parsing_scanoss_file.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source/run_spdx_extractor.py
RENAMED
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/requires.txt
RENAMED
|
File without changes
|
{fosslight_source-2.3.11 → fosslight_source-2.3.12}/src/fosslight_source.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|