fosslight-dependency 4.1.25__tar.gz → 4.1.27__tar.gz

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_dependency
-Version: 4.1.25
+Version: 4.1.27
 Summary: FOSSLight Dependency Scanner
 Home-page: https://github.com/fosslight/fosslight_dependency_scanner
 Download-URL: https://github.com/fosslight/fosslight_dependency_scanner

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/setup.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
 
     setup(
         name=_PACKAEG_NAME,
-        version='4.1.25',
+        version='4.1.27',
         package_dir={"": "src"},
         packages=find_namespace_packages(where='src'),
         description='FOSSLight Dependency Scanner',

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/_analyze_dependency.py

@@ -6,6 +6,7 @@
 import os
 import logging
 import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import deduplicate_dep_items
 from fosslight_dependency.package_manager.Pypi import Pypi
 from fosslight_dependency.package_manager.Npm import Npm
 from fosslight_dependency.package_manager.Yarn import Yarn
@@ -104,16 +105,19 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
         for f_name in package_manager.input_package_list_file:
             logger.info(f"Parse oss information with file: {f_name}")
 
-            if os.path.isfile(f_name):
+            file_path = os.path.join(input_dir, f_name) if not os.path.isabs(f_name) else f_name
+            if os.path.isfile(file_path):
                 package_manager.parse_oss_information(f_name)
                 package_dep_item_list.extend(package_manager.dep_items)
             else:
-                logger.error(f"Failed to open input file: {f_name}")
+                logger.error(f"Failed to open input file: {file_path}")
                 ret = False
         if package_manager_name == const.PNPM:
             logger.info("Parse oss information for pnpm")
             package_manager.parse_oss_information_for_pnpm()
             package_dep_item_list.extend(package_manager.dep_items)
+        if package_dep_item_list:
+            package_dep_item_list = deduplicate_dep_items(package_dep_item_list)
     if ret:
         logger.warning(f"### Complete to analyze: {package_manager_name}({input_dir}: {','.join(manifest_file_name)})")
         if package_manager.cover_comment:

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/_help.py

@@ -37,7 +37,9 @@ _HELP_MESSAGE_DEPENDENCY = f"""
                                         \t(npm, maven, gradle, pypi, pub, cocoapods, android, swift, carthage,
                                         \t go, nuget, helm, unity, cargo, pnpm, yarn)
             -p <input_path>\t\t    Enter the path where the script will be run.
-            -e <exclude_path>\t\t    Enter the path where the analysis will not be performed.
+            -e <exclude_path>\t\t    Enter the path where the analysis will not be performed (files and directories).
+            \t\t\t\t    * IMPORTANT: Always wrap patterns in double quotes ("") to avoid shell expansion.
+            \t\t\t\t      Example) fosslight_dependency -e "test/abc.py" "*.jar"
             -o <output_path>\t\t    Output path
             \t\t\t\t\t(If you want to generate the specific file name, add the output path with file name.)
             -f <format> [<format> ...]\t    Output formats

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/_package_manager.py

@@ -369,3 +369,29 @@ def change_file_mode(filepath, mode=''):
         os.chmod(filepath, new_mode)
         logger.debug(f"File mode of {filepath} has been changed to {oct(new_mode)}.")
     return current_mode
+
+
+def deduplicate_dep_items(dep_items):
+    if not dep_items:
+        return dep_items
+
+    unique_items = []
+    seen = set()
+
+    for item in dep_items:
+        first_oss = item.oss_items[0] if getattr(item, "oss_items", None) else None
+        oss_name = getattr(first_oss, "name", None) if first_oss else None
+        oss_ver = getattr(first_oss, "version", None) if first_oss else None
+        comment = getattr(first_oss, "comment", None) if first_oss else None
+
+        depends_on = None
+        if getattr(item, "depends_on", None):
+            depends_on = tuple(sorted(item.depends_on))
+
+        key = (getattr(item, "purl", None), oss_name, oss_ver, comment, depends_on)
+        if key in seen:
+            continue
+        seen.add(key)
+        unique_items.append(item)
+
+    return unique_items

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/constant.py

@@ -41,7 +41,7 @@ SUPPORT_PACKAE = {
     SWIFT: 'Package.resolved',
     CARTHAGE: 'Cartfile.resolved',
     GO: 'go.mod',
-    NUGET: ['packages.config', os.path.join('obj', 'project.assets.json')],
+    NUGET: ['packages.config', os.path.join('obj', 'project.assets.json'), 'Directory.Packages.props'],
     HELM: 'Chart.yaml',
     UNITY: os.path.join('Library', 'PackageManager', 'ProjectCache'),
     CARGO: 'Cargo.toml'

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/package_manager/Maven.py

@@ -255,10 +255,11 @@ class Maven(PackageManager):
             dep_key = f"{oss_item.name}({version})"
 
             if self.direct_dep:
-                if dep_key in self.direct_dep_list:
-                    oss_item.comment = 'direct'
-                else:
-                    oss_item.comment = 'transitive'
+                if len(self.direct_dep_list) > 0:
+                    if dep_key in self.direct_dep_list:
+                        oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'transitive'
                 try:
                     if dep_key in self.relation_tree:
                         dep_item.depends_on_raw = self.relation_tree[dep_key]

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/package_manager/Npm.py

@@ -9,6 +9,7 @@ import subprocess
 import json
 import shutil
 import re
+import requests
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
@@ -26,9 +27,11 @@ class Npm(PackageManager):
     input_file_name = 'tmp_npm_license_output.json'
     tmp_custom_json = 'custom.json'
     flag_tmp_node_modules = False
+    _network_available = False
 
     def __init__(self, input_dir, output_dir):
         super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+        self._check_network_available()
 
     def __del__(self):
         if os.path.isfile(os.path.join(self.input_dir, self.input_file_name)):
@@ -174,24 +177,33 @@ class Npm(PackageManager):
             package_path = d['path']
 
             private_pkg = False
-            if _private in d:
-                if d[_private]:
-                    private_pkg = True
+            if _private in d and d[_private]:
+                private_pkg = True
 
-            oss_item.download_location = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
-            dn_loc = f"{self.dn_url}{oss_init_name}"
-            dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
-            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
-            if d[_repository]:
-                dn_loc = d[_repository]
-            elif private_pkg:
-                dn_loc = ''
+            oss_item.download_location = ''
 
-            oss_item.homepage = dn_loc
+            npm_dl_url = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
+            npm_home_url = f"{self.dn_url}{oss_init_name}"
+            dep_item.purl = get_url_to_purl(npm_dl_url, self.package_manager_name)
+            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
 
+            repo_url = d[_repository] if d[_repository] else ''
             if private_pkg:
+                oss_item.homepage = repo_url or ''
                 oss_item.download_location = oss_item.homepage
                 oss_item.comment = 'private'
+            else:
+                npm_url_exists = False
+                if self._network_available is True:
+                    npm_url_exists = self._npm_url_exists(oss_init_name)
+
+                if self._network_available and not npm_url_exists:
+                    oss_item.homepage = repo_url or ""
+                    oss_item.download_location = oss_item.homepage
+                else:
+                    oss_item.homepage = repo_url or npm_home_url
+                    oss_item.download_location = npm_dl_url
+
             if self.package_name == f'{oss_init_name}({oss_item.version})':
                 oss_item.comment = 'root package'
             elif self.direct_dep and len(self.relation_tree) > 0:
@@ -221,6 +233,25 @@ class Npm(PackageManager):
             self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
         return
 
+    def _check_network_available(self) -> bool:
+        test_url = 'https://registry.npmjs.org/'
+        try:
+            resp = requests.head(test_url, timeout=3, allow_redirects=True)
+            self._network_available = resp.status_code < 400
+        except Exception:
+            self._network_available = False
+        return self._network_available
+
+    def _npm_url_exists(self, package_name: str) -> bool:
+        url = f"https://registry.npmjs.org/{package_name}"
+        try:
+            resp = requests.head(url, timeout=3, allow_redirects=True)
+            if resp.status_code == 405:
+                resp = requests.get(url, timeout=3, allow_redirects=True)
+            return resp.status_code < 400
+        except Exception:
+            return False
+
 
 def check_multi_license(license_name, manifest_file_path):
     multi_license_list = []

fosslight_dependency-4.1.27/src/fosslight_dependency/package_manager/Nuget.py

@@ -0,0 +1,368 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import re
+import os
+import subprocess
+from defusedxml.ElementTree import parse, fromstring
+import json
+import requests
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager
+from fosslight_dependency._package_manager import check_license_name, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Nuget(PackageManager):
+    package_manager_name = const.NUGET
+
+    dn_url = "https://nuget.org/packages/"
+    packageReference = False
+    directory_packages_props = 'Directory.Packages.props'
+    nuget_api_url = 'https://api.nuget.org/v3-flatcontainer/'
+    dotnet_ver = []
+    _exclude_dirs = {"test", "tests", "sample", "samples", "example", "examples"}
+
+    def __init__(self, input_dir, output_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+
+        for manifest_i in const.SUPPORT_PACKAE.get(self.package_manager_name):
+            if os.path.exists(os.path.basename(manifest_i)):
+                self.append_input_package_list_file(os.path.basename(manifest_i))
+                if manifest_i != 'packages.config':
+                    self.packageReference = True
+
+    def run_plugin(self):
+        ret = True
+        directory_packages_props_path = os.path.join(self.input_dir, self.directory_packages_props)
+        if not os.path.isfile(directory_packages_props_path):
+            return ret
+
+        logger.info(f"Found {self.directory_packages_props}. Using NuGet CPM flow.")
+        self.packageReference = True
+
+        restore_targets = self._find_restore_targets()
+        if restore_targets:
+            logger.info("Found .sln or .csproj files. Running 'dotnet restore'...")
+            for target_path, target_file in restore_targets:
+                logger.info(f"Restoring: {os.path.relpath(target_file, self.input_dir)}")
+                try:
+                    result = subprocess.run(
+                        ['dotnet', 'restore', target_file, '/p:EnableWindowsTargeting=true'],
+                        cwd=target_path,
+                        capture_output=True,
+                        text=True,
+                        timeout=300
+                    )
+                    if result.returncode == 0:
+                        logger.info(f"Successfully restored {os.path.relpath(target_file, self.input_dir)}")
+                    else:
+                        logger.warning(f"'dotnet restore' failed for {target_file} with return code {result.returncode}")
+                        if result.stderr:
+                            logger.warning(result.stderr)
+                except FileNotFoundError:
+                    logger.error("'dotnet' command not found. Please install .NET SDK.")
+                except subprocess.TimeoutExpired:
+                    logger.warning(f"'dotnet restore' timed out for {target_file}.")
+                except Exception as e:
+                    logger.warning(f"Failed to run 'dotnet restore' for {target_file}: {e}")
+        else:
+            logger.warning("No .sln or .csproj files found to restore.")
+
+        self.project_dirs = []
+        found_projects = False
+
+        for root, dirs, files in os.walk(self.input_dir):
+            rel_root = os.path.relpath(root, self.input_dir)
+            parts = rel_root.split(os.sep) if rel_root != os.curdir else []
+            if any(p.lower() in self._exclude_dirs for p in parts):
+                continue
+            assets_json = os.path.join(root, 'obj', 'project.assets.json')
+            if os.path.isfile(assets_json):
+                found_projects = True
+
+                rel_path = os.path.relpath(assets_json, self.input_dir)
+                logger.info(f"Found project.assets.json at: {rel_path}")
+
+                if rel_path not in self.input_package_list_file:
+                    self.append_input_package_list_file(rel_path)
+
+                project_dir = os.path.dirname(assets_json)
+                if project_dir.endswith(os.sep + 'obj'):
+                    project_dir = project_dir[: -len(os.sep + 'obj')]
+
+                if project_dir and project_dir not in self.project_dirs:
+                    self.project_dirs.append(project_dir)
+
+        if not found_projects:
+            logger.warning(
+                "Directory.Packages.props found and 'dotnet restore' completed, "
+                "but no obj/project.assets.json files were discovered."
+            )
+
+        return ret
+
+    def parse_oss_information(self, f_name):
+        tmp_license_txt_file_name = 'tmp_license.txt'
+        if f_name == self.directory_packages_props:
+            return
+
+        relation_tree = {}
+        direct_dep_list = []
+        if not hasattr(self, 'global_purl_dict'):
+            self.global_purl_dict = {}
+        if not hasattr(self, 'processed_packages'):
+            self.processed_packages = {}
+
+        file_path = os.path.join(self.input_dir, f_name) if not os.path.isabs(f_name) else f_name
+        with open(file_path, 'r', encoding='utf8') as input_fp:
+            package_list = []
+            if self.packageReference:
+                package_list = self.get_package_info_in_packagereference(input_fp, relation_tree, direct_dep_list)
+            else:
+                package_list = self.get_package_list_in_packages_config(input_fp)
+
+        for oss_origin_name, oss_version in package_list:
+            try:
+                pkg_key = f'{oss_origin_name}({oss_version})'
+                if pkg_key in self.processed_packages:
+                    existing_idx = self.processed_packages[pkg_key]
+                    existing_dep_item = self.dep_items[existing_idx]
+
+                    if pkg_key in relation_tree:
+                        new_deps = relation_tree[pkg_key]
+                        if existing_dep_item.depends_on_raw:
+                            existing_deps_set = set(existing_dep_item.depends_on_raw)
+                            new_deps_set = set(new_deps)
+                            merged_deps = sorted(existing_deps_set | new_deps_set)
+                            existing_dep_item.depends_on_raw = merged_deps
+                        else:
+                            existing_dep_item.depends_on_raw = new_deps
+                    if self.direct_dep and self.packageReference:
+                        if oss_origin_name in direct_dep_list:
+                            if 'direct' not in existing_dep_item.oss_items[0].comment:
+                                existing_dep_item.oss_items[0].comment = 'direct'
+                    continue
+
+                dep_item = DependencyItem()
+                oss_item = OssItem()
+                oss_item.name = f'{self.package_manager_name}:{oss_origin_name}'
+                oss_item.version = oss_version
+
+                license_name = ''
+                response = requests.get(f'{self.nuget_api_url.lower()}{oss_origin_name.lower()}/'
+                                        f'{oss_item.version.lower()}/{oss_origin_name.lower()}.nuspec')
+                if response.status_code == 200:
+                    root = fromstring(response.text)
+                    xmlns = ''
+                    m = re.search('{.*}', root.tag)
+                    if m:
+                        xmlns = m.group(0)
+                    nupkg_metadata = root.find(f'{xmlns}metadata')
+
+                    license_name_id = nupkg_metadata.find(f'{xmlns}license')
+                    if license_name_id is not None:
+                        license_name, license_comment = self.check_multi_license(license_name_id.text)
+                        if license_comment != '':
+                            oss_item.comment = license_comment
+                    else:
+                        license_url = nupkg_metadata.find(f'{xmlns}licenseUrl')
+                        if license_url is not None:
+                            url_res = requests.get(license_url.text)
+                            if url_res.status_code == 200:
+                                license_name_with_scanner = check_license_name(url_res.text)
+                                if license_name_with_scanner != "":
+                                    license_name = license_name_with_scanner
+                                else:
+                                    license_name = license_url.text
+                    oss_item.license = license_name
+                    repo_id = nupkg_metadata.find(f'{xmlns}repository')
+                    if repo_id is not None:
+                        oss_item.download_location = repo_id.get("url")
+                    else:
+                        proj_url_id = nupkg_metadata.find(f'{xmlns}projectUrl')
+                        if proj_url_id is not None:
+                            oss_item.download_location = proj_url_id.text
+                    oss_item.homepage = f'{self.dn_url}{oss_origin_name}'
+                    if not oss_item.download_location:
+                        oss_item.download_location = f'{oss_item.homepage}/{oss_item.version}'
+                    else:
+                        if oss_item.download_location.endswith('.git'):
+                            oss_item.download_location = oss_item.download_location[:-4]
+                    dep_item.purl = get_url_to_purl(f'{oss_item.homepage}/{oss_item.version}', self.package_manager_name)
+                else:
+                    oss_item.comment = 'Fail to response for nuget api'
+                    dep_item.purl = f'pkg:nuget/{oss_origin_name}@{oss_item.version}'
+                self.global_purl_dict[f'{oss_origin_name}({oss_item.version})'] = dep_item.purl
+
+                if self.direct_dep and self.packageReference:
+                    if oss_origin_name in direct_dep_list:
+                        oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'transitive'
+
+                    key = f'{oss_origin_name}({oss_item.version})'
+                    if key in relation_tree:
+                        dep_item.depends_on_raw = relation_tree[key]
+
+                dep_item.oss_items.append(oss_item)
+                self.dep_items.append(dep_item)
+                self.processed_packages[pkg_key] = len(self.dep_items) - 1
+
+            except Exception as e:
+                logger.warning(f"Failed to parse oss information: {e}")
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(self.global_purl_dict, self.dep_items)
+
+        if os.path.isfile(tmp_license_txt_file_name):
+            os.remove(tmp_license_txt_file_name)
+
+        return
+
+    def get_package_list_in_packages_config(self, input_fp):
+        package_list = []
+        root = parse(input_fp).getroot()
+        for p in root.findall("package"):
+            package_list.append([p.get("id"), p.get("version")])
+        return package_list
+
+    def get_package_info_in_packagereference(self, input_fp, relation_tree, direct_dep_list):
+        json_f = json.load(input_fp)
+
+        dotnet_ver = self.get_dotnet_ver_list(json_f)
+        package_list = self.get_package_list_in_packages_assets(json_f)
+        self.get_dependency_tree(json_f, relation_tree, dotnet_ver)
+        self.get_direct_dependencies_from_assets_json(json_f, direct_dep_list)
+        self.get_direct_package_in_packagereference(direct_dep_list)
+
+        return package_list
+
+    def get_package_list_in_packages_assets(self, json_f):
+        package_list = []
+        for item in json_f['libraries']:
+            if json_f['libraries'][item]['type'] == 'package':
+                oss_info = item.split('/')
+                package_list.append([oss_info[0], oss_info[1]])
+        return package_list
+
+    def get_dotnet_ver_list(self, json_f):
+        dotnet_ver = []
+        json_project_group = json_f['projectFileDependencyGroups']
+        for ver in json_project_group:
+            dotnet_ver.append(ver)
+        return dotnet_ver
+
+    def get_direct_dependencies_from_assets_json(self, json_f, direct_dep_list):
+        try:
+            json_project_group = json_f.get('projectFileDependencyGroups', {})
+            for _, dependencies in json_project_group.items():
+                if not dependencies:
+                    continue
+                for dep_string in dependencies:
+                    package_name = dep_string.split()[0] if dep_string else ''
+                    if package_name and package_name not in direct_dep_list:
+                        direct_dep_list.append(package_name)
+        except Exception as e:
+            logger.warning(f"Failed to extract direct dependencies from project.assets.json: {e}")
+
+    def get_dependency_tree(self, json_f, relation_tree, dotnet_ver):
+        actual_versions = {}
+        for lib_key in json_f.get('libraries', {}):
+            if '/' in lib_key:
+                lib_name, lib_version = lib_key.split('/', 1)
+                actual_versions[lib_name.lower()] = lib_version
+
+        json_target = json_f['targets']
+        for item in json_target:
+            if item not in dotnet_ver:
+                continue
+            json_item = json_target[item]
+            for pkg in json_item:
+                json_pkg = json_item[pkg]
+                if 'type' not in json_pkg:
+                    continue
+                if 'dependencies' not in json_pkg:
+                    continue
+                if json_pkg['type'] != 'package':
+                    continue
+                oss_info = pkg.split('/')
+                relation_tree[f'{oss_info[0]}({oss_info[1]})'] = []
+                for dep in json_pkg['dependencies']:
+                    oss_name = dep
+                    dep_ver_in_spec = json_pkg['dependencies'][dep]
+                    actual_ver = actual_versions.get(oss_name.lower(), dep_ver_in_spec)
+                    relation_tree[f'{oss_info[0]}({oss_info[1]})'].append(f'{oss_name}({actual_ver})')
+
+    def get_direct_package_in_packagereference(self, direct_dep_list):
+        if hasattr(self, 'project_dirs') and self.project_dirs:
+            for project_dir in self.project_dirs:
+                for f in os.listdir(project_dir):
+                    f_path = os.path.join(project_dir, f)
+                    if os.path.isfile(f_path) and ((f.split('.')[-1] == 'csproj') or (f.split('.')[-1] == 'xproj')):
+                        with open(f_path, 'r', encoding='utf8') as input_fp:
+                            root = parse(input_fp).getroot()
+                        itemgroups = root.findall('ItemGroup')
+                        for itemgroup in itemgroups:
+                            for item in itemgroup.findall('PackageReference'):
+                                pkg_name = item.get('Include')
+                                if pkg_name and pkg_name not in direct_dep_list:
+                                    direct_dep_list.append(pkg_name)
+        else:
+            for f in os.listdir(self.input_dir):
+                f_path = os.path.join(self.input_dir, f)
+                if os.path.isfile(f_path) and ((f.split('.')[-1] == 'csproj') or (f.split('.')[-1] == 'xproj')):
+                    with open(f_path, 'r', encoding='utf8') as input_fp:
+                        root = parse(input_fp).getroot()
+                    itemgroups = root.findall('ItemGroup')
+                    for itemgroup in itemgroups:
+                        for item in itemgroup.findall('PackageReference'):
+                            pkg_name = item.get('Include')
+                            if pkg_name and pkg_name not in direct_dep_list:
+                                direct_dep_list.append(pkg_name)
+
+    def check_multi_license(self, license_name):
+        multi_license = license_name
+        license_comment = ''
+        try:
+            if license_name.startswith('(') and license_name.endswith(')'):
+                license_name = license_name.lstrip('(').rstrip(')')
+                license_comment = license_name
+                multi_license = ','.join(re.split(r'OR|AND', license_name))
+        except Exception as e:
+            logger.warning(f'Fail to parse multi license in npm: {e}')
+
+        return multi_license, license_comment
+
+    def _find_restore_targets(self):
+        sln_files = []
+        csproj_files = []
+
+        for root, dirs, files in os.walk(self.input_dir):
+            rel_root = os.path.relpath(root, self.input_dir)
+            parts = rel_root.split(os.sep) if rel_root != os.curdir else []
+            if any(p.lower() in self._exclude_dirs for p in parts):
+                continue
+
+            depth = len(parts) if parts and parts[0] != '.' else 0
+
+            for f in files:
+                if f.endswith('.sln'):
+                    sln_files.append((depth, root, os.path.join(root, f)))
+                elif f.endswith('.csproj'):
+                    csproj_files.append((depth, root, os.path.join(root, f)))
+
+        result = []
+        if sln_files:
+            result.extend([(d, f) for _, d, f in sln_files])
+
+        if csproj_files:
+            result.extend([(d, f) for _, d, f in csproj_files])
+
+        return result

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/package_manager/Yarn.py

@@ -121,24 +121,33 @@ class Yarn(Npm):
             package_path = d['path']
 
             private_pkg = False
-            if _private in d:
-                if d[_private]:
-                    private_pkg = True
+            if _private in d and d[_private]:
+                private_pkg = True
 
-            oss_item.download_location = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
-            dn_loc = f"{self.dn_url}{oss_init_name}"
-            dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
-            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
-            if d[_repository]:
-                dn_loc = d[_repository]
-            elif private_pkg:
-                dn_loc = ''
+            oss_item.download_location = ''
 
-            oss_item.homepage = dn_loc
+            npm_dl_url = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
+            npm_home_url = f"{self.dn_url}{oss_init_name}"
+            dep_item.purl = get_url_to_purl(npm_dl_url, self.package_manager_name)
+            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
 
+            repo_url = d[_repository] if d[_repository] else ''
             if private_pkg:
+                oss_item.homepage = repo_url or ''
                 oss_item.download_location = oss_item.homepage
                 oss_item.comment = 'private'
+            else:
+                npm_url_exists = False
+                if self._network_available is True:
+                    npm_url_exists = self._npm_url_exists(oss_init_name, oss_item.version)
+
+                if self._network_available and not npm_url_exists:
+                    oss_item.homepage = repo_url or ""
+                    oss_item.download_location = oss_item.homepage
+                else:
+                    oss_item.homepage = repo_url or npm_home_url
+                    oss_item.download_location = npm_dl_url
+
             if self.package_name == f'{oss_init_name}({oss_item.version})':
                 oss_item.comment = 'root package'
             elif self.direct_dep and len(self.relation_tree) > 0:

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency/run_dependency_scanner.py

@@ -60,6 +60,7 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                 manifest_file_name.append(value)
 
     found_manifest_file = []
+    found_manifest_set = set()
     suggested_files = []
     for parent, dirs, files in os.walk(input_dir):
         parent_parts = parent.split(os.sep)
@@ -74,17 +75,30 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                    for exclude_path in abs_path_to_exclude):
                 continue
             if file in manifest_file_name:
-                path_with_filename = os.path.join(parent, file)
-                found_manifest_file.append(path_with_filename)
+                candidate = os.path.join(parent, file)
+                norm_candidate = os.path.normpath(candidate)
+                if norm_candidate not in found_manifest_set:
+                    found_manifest_set.add(norm_candidate)
+                    found_manifest_file.append(candidate)
+            for manifest_f in manifest_file_name:
+                candidate = os.path.join(parent, manifest_f)
+                norm_candidate = os.path.normpath(candidate)
+                if os.path.exists(candidate) and norm_candidate not in found_manifest_set:
+                    found_manifest_set.add(norm_candidate)
+                    found_manifest_file.append(candidate)
             if file in const.SUGGESTED_PACKAGE.keys():
                 suggested_files.append(os.path.join(parent, file))
+
         for dir in dirs:
             for manifest_f in manifest_file_name:
                 manifest_l = manifest_f.split(os.path.sep)
-                if len(manifest_l) > 1:
-                    if manifest_l[0] == dir:
-                        if os.path.exists(os.path.join(parent, manifest_f)):
-                            found_manifest_file.append(os.path.join(parent, manifest_f))
+                if len(manifest_l) > 1 and manifest_l[0] == dir:
+                    candidate = os.path.join(parent, manifest_f)
+                    norm_candidate = os.path.normpath(candidate)
+                    if os.path.exists(candidate) and norm_candidate not in found_manifest_set:
+                        found_manifest_set.add(norm_candidate)
+                        found_manifest_file.append(candidate)
+
         if not recursive:
             if len(found_manifest_file) > 0:
                 input_dir = parent
@@ -95,12 +109,22 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
         f_name = os.path.basename(f_with_path)
         dir_path = os.path.dirname(f_with_path)
         for key, value in const.SUPPORT_PACKAE.items():
-            if isinstance(value, list):
-                if f_name in value:
-                    found_package_manager[key][dir_path].append(f_name)
-            else:
-                if f_name == value:
-                    found_package_manager[key][dir_path].append(f_name)
+            manifest_patterns = value if isinstance(value, list) else [value]
+
+            for pattern in manifest_patterns:
+                if os.path.sep not in pattern:
+                    if f_name == pattern:
+                        if pattern not in found_package_manager[key][dir_path]:
+                            found_package_manager[key][dir_path].append(pattern)
+                else:
+                    rel_dir, rel_file = os.path.split(pattern)
+                    expected_path = os.path.join(dir_path, rel_file)
+
+                    if f_name == rel_file:
+                        candidate = os.path.join(os.path.dirname(dir_path), rel_dir, rel_file) if rel_dir else expected_path
+                        if os.path.normpath(candidate) == os.path.normpath(f_with_path):
+                            if pattern not in found_package_manager[key][dir_path]:
+                                found_package_manager[key][dir_path].append(pattern)
     found_package_manager = {k: dict(v) for k, v in found_package_manager.items()}
 
     # both npm and pnpm are detected, remove npm.
@@ -133,10 +157,12 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
     return ret, found_package_manager, input_dir, suggested_files
 
 
-def print_package_info(success_pm, log_lines):
-    if success_pm:
-        for pm, dir_dict in success_pm.items():
-            log_lines.append(f"- {pm}:")
+def print_package_info(pm, log_lines, status=''):
+    if pm:
+        if status:
+            status = f"[{status}] "
+        for pm, dir_dict in pm.items():
+            log_lines.append(f"- {status} {pm}:")
             for path, files in dir_dict.items():
                 file_list = ', '.join(files)
                 log_lines.append(f"  {path}: {file_list}")
@@ -318,16 +344,14 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
     success_pm = {k: dict(v) for k, v in success_pm.items()}
     fail_pm = {k: dict(v) for k, v in fail_pm.items()}
     if len(found_package_manager.keys()) > 0:
+        log_lines = ["Dependency Analysis Summary"]
         if len(success_pm) > 0:
-            log_lines = ["Success to analyze:"]
-            log_lines = print_package_info(success_pm, log_lines)
-            scan_item.set_cover_comment('\n'.join(log_lines))
+            log_lines = print_package_info(success_pm, log_lines, 'Success')
         if len(fail_pm) > 0:
-            log_lines = ["Fail to analyze:"]
-            log_lines = print_package_info(fail_pm, log_lines)
-            scan_item.set_cover_comment('\n'.join(log_lines))
-            scan_item.set_cover_comment('Check log file(fosslight_log*.txt) '
-                                        'and https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.')
+            log_lines = print_package_info(fail_pm, log_lines, 'Fail')
+            log_lines.append('If analysis fails, see fosslight_log*.txt and the prerequisite guide: '
+                             'https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.')
+        scan_item.set_cover_comment('\n'.join(log_lines))
 
     if ret and graph_path:
         graph_path = os.path.abspath(graph_path)

{fosslight_dependency-4.1.25 → fosslight_dependency-4.1.27}/src/fosslight_dependency.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_dependency
-Version: 4.1.25
+Version: 4.1.27
 Summary: FOSSLight Dependency Scanner
 Home-page: https://github.com/fosslight/fosslight_dependency_scanner
 Download-URL: https://github.com/fosslight/fosslight_dependency_scanner

fosslight_dependency-4.1.25/src/fosslight_dependency/package_manager/Nuget.py

@@ -1,196 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2022 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import re
-import os
-from defusedxml.ElementTree import parse, fromstring
-import json
-import requests
-import fosslight_util.constant as constant
-import fosslight_dependency.constant as const
-from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import check_license_name, get_url_to_purl
-from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
-from fosslight_util.oss_item import OssItem
-
-logger = logging.getLogger(constant.LOGGER_NAME)
-
-
-class Nuget(PackageManager):
-    package_manager_name = const.NUGET
-
-    dn_url = "https://nuget.org/packages/"
-    packageReference = False
-    nuget_api_url = 'https://api.nuget.org/v3-flatcontainer/'
-    dotnet_ver = []
-
-    def __init__(self, input_dir, output_dir):
-        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
-
-        for manifest_i in const.SUPPORT_PACKAE.get(self.package_manager_name):
-            if os.path.isfile(manifest_i):
-                self.append_input_package_list_file(manifest_i)
-                if manifest_i != 'packages.config':
-                    self.packageReference = True
-
-    def parse_oss_information(self, f_name):
-        tmp_license_txt_file_name = 'tmp_license.txt'
-        with open(f_name, 'r', encoding='utf8') as input_fp:
-            purl_dict = {}
-            package_list = []
-            if self.packageReference:
-                package_list = self.get_package_info_in_packagereference(input_fp)
-            else:
-                package_list = self.get_package_list_in_packages_config(input_fp)
-
-        for oss_origin_name, oss_version in package_list:
-            try:
-                dep_item = DependencyItem()
-                oss_item = OssItem()
-                oss_item.name = f'{self.package_manager_name}:{oss_origin_name}'
-                oss_item.version = oss_version
-
-                license_name = ''
-                response = requests.get(f'{self.nuget_api_url.lower()}{oss_origin_name.lower()}/'
-                                        f'{oss_item.version.lower()}/{oss_origin_name.lower()}.nuspec')
-                if response.status_code == 200:
-                    root = fromstring(response.text)
-                    xmlns = ''
-                    m = re.search('{.*}', root.tag)
-                    if m:
-                        xmlns = m.group(0)
-                    nupkg_metadata = root.find(f'{xmlns}metadata')
-
-                    license_name_id = nupkg_metadata.find(f'{xmlns}license')
-                    if license_name_id is not None:
-                        license_name, license_comment = self.check_multi_license(license_name_id.text)
-                        if license_comment != '':
-                            oss_item.comment = license_comment
-                    else:
-                        license_url = nupkg_metadata.find(f'{xmlns}licenseUrl')
-                        if license_url is not None:
-                            url_res = requests.get(license_url.text)
-                            if url_res.status_code == 200:
-                                license_name_with_scanner = check_license_name(url_res.text)
-                                if license_name_with_scanner != "":
-                                    license_name = license_name_with_scanner
-                                else:
-                                    license_name = license_url.text
-                    oss_item.license = license_name
-                    repo_id = nupkg_metadata.find(f'{xmlns}repository')
-                    if repo_id is not None:
-                        oss_item.download_location = repo_id.get("url")
-                    else:
-                        proj_url_id = nupkg_metadata.find(f'{xmlns}projectUrl')
-                        if proj_url_id is not None:
-                            oss_item.download_location = proj_url_id.text
-                    oss_item.homepage = f'{self.dn_url}{oss_origin_name}'
-                    if oss_item.download_location == '':
-                        oss_item.download_location = f'{oss_item.homepage}/{oss_item.version}'
-                    else:
-                        if oss_item.download_location.endswith('.git'):
-                            oss_item.download_location = oss_item.download_location[:-4]
-                    dep_item.purl = get_url_to_purl(f'{oss_item.homepage}/{oss_item.version}', self.package_manager_name)
-                else:
-                    oss_item.comment = 'Fail to response for nuget api'
-                    dep_item.purl = f'pkg:nuget/{oss_origin_name}@{oss_item.version}'
-                purl_dict[f'{oss_origin_name}({oss_item.version})'] = dep_item.purl
-
-                if self.direct_dep and self.packageReference:
-                    if oss_origin_name in self.direct_dep_list:
-                        oss_item.comment = 'direct'
-                    else:
-                        oss_item.comment = 'transitive'
-
-                    if f'{oss_origin_name}({oss_item.version})' in self.relation_tree:
-                        dep_item.depends_on_raw = self.relation_tree[f'{oss_origin_name}({oss_item.version})']
-
-                dep_item.oss_items.append(oss_item)
-                self.dep_items.append(dep_item)
-
-            except Exception as e:
-                logger.warning(f"Failed to parse oss information: {e}")
-        if self.direct_dep:
-            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
-
-        if os.path.isfile(tmp_license_txt_file_name):
-            os.remove(tmp_license_txt_file_name)
-
-        return
-
-    def get_package_list_in_packages_config(self, input_fp):
-        package_list = []
-        root = parse(input_fp).getroot()
-        for p in root.findall("package"):
-            package_list.append([p.get("id"), p.get("version")])
-        return package_list
-
-    def get_package_info_in_packagereference(self, input_fp):
-        json_f = json.load(input_fp)
-
-        self.get_dotnet_ver_list(json_f)
-        package_list = self.get_package_list_in_packages_assets(json_f)
-        self.get_dependency_tree(json_f)
-        self.get_direct_package_in_packagereference()
-
-        return package_list
-
-    def get_package_list_in_packages_assets(self, json_f):
-        package_list = []
-        for item in json_f['libraries']:
-            if json_f['libraries'][item]['type'] == 'package':
-                oss_info = item.split('/')
-                package_list.append([oss_info[0], oss_info[1]])
-        return package_list
-
-    def get_dotnet_ver_list(self, json_f):
-        json_project_group = json_f['projectFileDependencyGroups']
-        for dotnet_ver in json_project_group:
-            self.dotnet_ver.append(dotnet_ver)
-
-    def get_dependency_tree(self, json_f):
-        json_target = json_f['targets']
-        for item in json_target:
-            if item not in self.dotnet_ver:
-                continue
-            json_item = json_target[item]
-            for pkg in json_item:
-                json_pkg = json_item[pkg]
-                if 'type' not in json_pkg:
-                    continue
-                if 'dependencies' not in json_pkg:
-                    continue
-                if json_pkg['type'] != 'package':
-                    continue
-                oss_info = pkg.split('/')
-                self.relation_tree[f'{oss_info[0]}({oss_info[1]})'] = []
-                for dep in json_pkg['dependencies']:
-                    oss_name = dep
-                    oss_ver = json_pkg['dependencies'][dep]
-                    self.relation_tree[f'{oss_info[0]}({oss_info[1]})'].append(f'{oss_name}({oss_ver})')
-
-    def get_direct_package_in_packagereference(self):
-        for f in os.listdir(self.input_dir):
-            if os.path.isfile(f) and ((f.split('.')[-1] == 'csproj') or (f.split('.')[-1] == 'xproj')):
-                with open(f, 'r', encoding='utf8') as input_fp:
-                    root = parse(input_fp).getroot()
-                itemgroups = root.findall('ItemGroup')
-                for itemgroup in itemgroups:
-                    for item in itemgroup.findall('PackageReference'):
-                        self.direct_dep_list.append(item.get('Include'))
-
-    def check_multi_license(self, license_name):
-        multi_license = license_name
-        license_comment = ''
-        try:
-            if license_name.startswith('(') and license_name.endswith(')'):
-                license_name = license_name.lstrip('(').rstrip(')')
-                license_comment = license_name
-                multi_license = ','.join(re.split(r'OR|AND', license_name))
-        except Exception as e:
-            logger.warning(f'Fail to parse multi license in npm: {e}')
-
-        return multi_license, license_comment